scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge pull request #32884 from BerriAI/litellm_internal_staging
Some checks failed
CodeQL / Analyze (actions) (push) Has been cancelled
Details
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
Details
CodeQL / Analyze (python) (push) Has been cancelled
Details
CodSpeed Benchmarks / benchmarks (push) Has been cancelled
Details
Helm unit test / unit-test (push) Has been cancelled
Details
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
Details
GitHub Actions Security Analysis / zizmor (push) Has been cancelled
Details

Browse source 
chore(ci): promote internal staging to main
This commit is contained in:
yuneng-jiang 2026-07-11 17:42:56 -07:00 • committed by GitHub
commit 10d5804b3e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1395 changed files with 38574 additions and 10268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
.github/actions/detect-backend-changes/action.yml vendored Normal file
View file

@ -0,0 +1,48 @@
name: "Detect backend-relevant changes"
description: >-
Classify the pull request's changed files with .circleci/scripts/classify_changes.sh
and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files
changed, so callers can short-circuit expensive steps while the job still completes
successfully and satisfies its required status check. The decision defaults to run for
any non pull_request event or whenever the changed set cannot be resolved, so tests are
never skipped when the classification is uncertain.
outputs:
decision:
description: "run when backend-relevant files changed, otherwise skip"
value: ${{ steps.classify.outputs.decision }}
runs:
using: composite
steps:
- id: classify
shell: bash
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
set -uo pipefail
if [ -z "${BASE_SHA:-}" ]; then
echo "detect-backend-changes: not a pull_request event; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
fi
if ! git fetch --no-tags --depth=1 origin "${BASE_SHA}" >/dev/null 2>&1; then
echo "detect-backend-changes: could not fetch base ${BASE_SHA}; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
fi
changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null)" || {
echo "detect-backend-changes: git diff failed; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
}
if [ -z "${changed}" ]; then
echo "detect-backend-changes: no changed files vs ${BASE_SHA}; skipping job"
echo "decision=skip" >> "${GITHUB_OUTPUT}"
exit 0
fi
echo "detect-backend-changes: changed files vs ${BASE_SHA}:"
printf '%s\n' "${changed}" | sed 's/^/ /'
decision="$(printf '%s\n' "${changed}" | bash .circleci/scripts/classify_changes.sh backend)" || decision="run"
echo "detect-backend-changes: decision=${decision}"
echo "decision=${decision}" >> "${GITHUB_OUTPUT}"

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -45,12 +45,18 @@ jobs:
name: Run tests
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.timeout-minutes }}
outputs:
decision: ${{ steps.changes.outputs.decision }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Detect backend-relevant changes
id: changes
uses: ./.github/actions/detect-backend-changes
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
@ -72,16 +78,19 @@ jobs:
${{ runner.os }}-uv-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
env:
TEST_PATH: ${{ inputs.test-path }}
MAX_FAILURES: ${{ inputs.max-failures }}
@ -114,7 +123,7 @@ jobs:
fi
- name: Save coverage report
if: always()
if: always() && steps.changes.outputs.decision != 'skip'
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
@ -124,7 +133,7 @@ jobs:
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
if: always() && needs.run.outputs.decision != 'skip'
runs-on: ubuntu-latest
permissions:
contents: read

4
.github/workflows/guard-main-branch.yml vendored
View file

@ -31,12 +31,12 @@ jobs:
echo "PR head repo: $HEAD_REPO"
echo "PR head branch: $HEAD_REF"
if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_staging' branch instead."
echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
exit 1
fi
if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
echo "Allowed source branch."
exit 0
fi
echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_staging' instead."
echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
exit 1

12
.github/workflows/test-linting.yml vendored
View file

@ -48,7 +48,7 @@ jobs:
- name: Install dependencies
run: |
uv sync --frozen --group proxy-dev
uv sync --frozen --group proxy-dev --group e2e-dev
# basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
# only after `prisma generate` writes prisma/client.py et al. Without this the
@ -107,6 +107,16 @@ jobs:
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
else
echo "No changed tests/e2e Python files; skipping."
fi
- name: Check for circular imports
run: |
cd litellm

76
.github/workflows/test-litellm-ui-build.yml vendored
View file

@ -36,79 +36,3 @@ jobs:
- name: Build
run: npm run build
frontend-lint:
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
persist-credentials: false
- name: Collect changed files
id: changed
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: "20"
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json --check eslint-metrics.json

92
.github/workflows/test-litellm-ui-lint.yml vendored Normal file
View file

@ -0,0 +1,92 @@
name: UI Lint
permissions:
contents: read
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
jobs:
frontend-lint:
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
persist-credentials: false
- name: Collect changed files
id: changed
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: "20"
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
- name: Check for dead code (knip)
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: npm run knip

7
.github/workflows/test-unit-documentation.yml vendored
View file

@ -32,6 +32,10 @@ jobs:
path: docs/my-website
persist-credentials: false
- name: Detect backend-relevant changes
id: changes
uses: ./.github/actions/detect-backend-changes
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
@ -53,10 +57,12 @@ jobs:
${{ runner.os }}-uv-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
@ -64,6 +70,7 @@ jobs:
# Run the same documentation tests that CircleCI ran (as direct Python scripts)
- name: Run documentation validation tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py

7
.github/workflows/test-unit-proxy-legacy.yml vendored
View file

@ -49,6 +49,10 @@ jobs:
with:
persist-credentials: false
- name: Detect backend-relevant changes
id: changes
uses: ./.github/actions/detect-backend-changes
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
@ -70,16 +74,19 @@ jobs:
${{ runner.os }}-uv-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
if: steps.changes.outputs.decision != 'skip'
env:
TEST_PATH: ${{ matrix.test-group.path }}
run: |

2
CLAUDE.md
View file

@ -19,7 +19,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose for internal contributors; external / OSS contributions target the current daily OSS branch instead, named `litellm_oss_daily_YYYY_MM_DD` (a fresh one is cut each weekday, so use the most recent)
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule

2
CONTRIBUTING.md
View file

@ -322,7 +322,7 @@ npm run build
## Submitting Your PR
1. **Push your branch**: `git push origin your-feature-branch`
2. **Create a PR**: Go to GitHub and create a pull request
2. **Create a PR**: Go to GitHub and open a pull request against the current daily OSS branch, named `litellm_oss_daily_YYYY_MM_DD`. A fresh one is cut each weekday, so pick the most recent from the [branch list](https://github.com/BerriAI/litellm/branches/all?query=litellm_oss_daily). Do not target `main`.
3. **Fill out the PR template**: Provide clear description of changes
4. **Wait for review**: Maintainers will review and provide feedback
5. **Address feedback**: Make requested changes and push updates

13
Makefile
View file

@ -5,7 +5,7 @@
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev lint-checks format \
lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety pre-commit \
@ -27,6 +27,7 @@ help:
@echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@ -54,6 +55,7 @@ UV := uv
UV_RUN := $(UV) run --no-sync
LINT_DEP_INSTALL ?= install-dev
LINT_E2E_DEP_INSTALL ?= lint-install
LINT_DEP_BASE ?= lint-fetch-base
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
@ -111,7 +113,7 @@ lint-fetch-base:
# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
# running proxy need.
lint-install:
$(UV) sync --inexact --frozen --group proxy-dev
$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
@ -164,6 +166,9 @@ lint-ruff-FULL-dev: install-dev
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@ -208,9 +213,9 @@ check-import-safety: $(LINT_DEP_INSTALL)
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety

1
backend/routes/allowlist.py
View file

@ -46,6 +46,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/fallback",
"/fallbacks",
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/cost/",
"/credentials",

10
basedpyright-code-budget.json
View file

@ -57,7 +57,7 @@
"limit": 5900
},
"reportMissingTypeArgument": {
"limit": 15918
"limit": 15903
},
"reportMissingTypeStubs": {
"limit": 41
@ -105,13 +105,13 @@
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40541
"limit": 40539
},
"reportUnknownParameterType": {
"limit": 20418
"limit": 20403
},
"reportUnknownVariableType": {
"limit": 32151
"limit": 32141
},
"reportUnnecessaryCast": {
"limit": 177
@ -123,7 +123,7 @@
"limit": 7
},
"reportUnnecessaryIsInstance": {
"limit": 1212
"limit": 1209
},
"reportUntypedBaseClass": {
"limit": 165

7
helm/litellm-helm/templates/_helpers.tpl
View file

@ -76,10 +76,13 @@ so fall back to "default" (or an explicit override) to avoid a cyclic dependency
{{- end }}
{{/*
Get redis service name
Get redis service name.
The bundled Redis subchart only serves sentinel in "replication" architecture
(it rejects standalone + sentinel outright), and in that mode the sentinel
Service is named "<release>-redis", not "<release>-redis-master".
*/}}
{{- define "litellm.redis.serviceName" -}}
{{- if and (eq .Values.redis.architecture "standalone") .Values.redis.sentinel.enabled -}}
{{- if .Values.redis.sentinel.enabled -}}
{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}
{{- else -}}
{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}

15
helm/litellm-helm/templates/configmap-litellm.yaml
View file

@ -1,9 +1,22 @@
{{- if .Values.proxyConfigMap.create }}
{{- $config := deepCopy .Values.proxy_config }}
{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }}
{{- $generalSettings := (get $config "general_settings") | default dict }}
{{- if not (hasKey $generalSettings "coordination_redis") }}
{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }}
{{- if .Values.redis.sentinel.enabled }}
{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }}
{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }}
{{- end }}
{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }}
{{- $_ := set $config "general_settings" $generalSettings }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{ $config | toYaml | indent 6 }}
{{- end }}

143
helm/litellm-helm/tests/coordination_redis_tests.yaml Normal file
View file

@ -0,0 +1,143 @@
suite: test coordination redis
templates:
- configmap-litellm.yaml
- deployment.yaml
tests:
- it: should not render coordination_redis when redis is disabled
template: configmap-litellm.yaml
set:
redis.enabled: false
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: coordination_redis
- it: should not emit redis env vars when redis is disabled
template: deployment.yaml
set:
redis.enabled: false
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
any: true
- it: should render coordination_redis pointing at the bundled redis when enabled
template: configmap-litellm.yaml
set:
redis.enabled: true
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n"
- matchRegex:
path: data["config.yaml"]
pattern: "master_key: os.environ/PROXY_MASTER_KEY"
- it: should emit redis env vars backing the coordination_redis os.environ refs
template: deployment.yaml
set:
redis.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PORT
value: "6379"
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: RELEASE-NAME-redis
key: redis-password
- it: should not render coordination_redis when coordination is opted out
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.coordination.enabled: false
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: coordination_redis
- it: should keep emitting redis env vars when coordination is opted out
template: deployment.yaml
set:
redis.enabled: true
redis.coordination.enabled: false
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
- it: should not clobber a user supplied coordination_redis block
template: configmap-litellm.yaml
set:
redis.enabled: true
proxy_config.general_settings.coordination_redis:
url: os.environ/COORDINATION_REDIS_URL
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n"
- notMatchRegex:
path: data["config.yaml"]
pattern: "host: os.environ/REDIS_HOST"
- it: should render sentinel_nodes and service_name in sentinel mode
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
asserts:
# The sentinel Service the redis subchart renders is "<release>-redis", and a
# plain client cannot speak the sentinel protocol, so host/port must not appear
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n"
- notMatchRegex:
path: data["config.yaml"]
pattern: "host: os.environ/REDIS_HOST"
- it: should carry a custom sentinel masterSet into service_name
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
redis.sentinel.masterSet: litellm-master
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "service_name: litellm-master"
- it: should point REDIS_HOST at the sentinel service in sentinel mode
template: deployment.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PORT
value: "26379"

22
helm/litellm-helm/values.yaml
View file

@ -331,12 +331,28 @@ postgresql:
# secretKeys:
# userPasswordKey: password
# requires cache: true in config file
# either enable this or pass a secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL
# with cache: true to use existing redis instance
# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend
# tracking, and the pod lock manager. Enabling this deploys the bundled Redis
# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and
# renders a `general_settings.coordination_redis` block into the proxy config.
#
# To point at an existing Redis instead, leave `enabled: false` and pass a
# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy
# falls back to those env vars for coordination. Set `cache: true` in the proxy
# config only if you also want LLM response caching, which is independent of
# coordination
#
# When `redis.sentinel.enabled` is set, the coordination block is rendered with
# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead
# of host/port, because a plain Redis client cannot talk to the sentinel port
redis:
enabled: false
architecture: standalone
coordination:
# Set to false to keep the bundled Redis for response caching only and leave
# `general_settings.coordination_redis` out of the rendered config. A
# `coordination_redis` block you define yourself in `proxy_config` always wins
enabled: true
# Prisma migration job settings
migrationJob:

13
helm/litellm/templates/_helpers.tpl
View file

@ -213,6 +213,10 @@ harmless no-op for the Job and authoritative for the app pods.
*/}}
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{/* These feed the proxy's coordination Redis (cross-pod rate limits, spend
tracking, pod lock manager) via its REDIS_* env fallback. An explicit
`general_settings.coordination_redis` block in proxy_config takes
precedence over anything emitted here. */}}
{{- if $root.Values.redis.host }}
- name: REDIS_HOST
value: {{ $root.Values.redis.host | quote }}
@ -226,10 +230,11 @@ harmless no-op for the Job and authoritative for the app pods.
key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- if $root.Values.redis.cluster }}
{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a
RedisClusterCache when it's set (litellm/caching/caching.py:169-192).
We seed with the single configured endpoint — the cluster client
discovers the remaining nodes from CLUSTER SLOTS at startup. */}}
{{/* The proxy falls back to REDIS_CLUSTER_NODES (JSON) to build a cluster-mode
coordination client when `general_settings.coordination_redis` is absent
and no plain-Redis response cache is configured. We seed with the single
configured endpoint; the cluster client discovers the remaining nodes from
CLUSTER SLOTS at startup. */}}
- name: REDIS_CLUSTER_NODES
value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }}
{{- end }}

109
helm/litellm/tests/redis_env_tests.yaml Normal file
View file

@ -0,0 +1,109 @@
suite: test redis coordination env vars
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
- backend/deployment.yaml
values:
- ./values/required.yaml
tests:
- it: gateway omits redis env vars when no host is configured
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: redis.example.com
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_CLUSTER_NODES
any: true
- it: gateway emits host, port and password when redis is configured
template: gateway/deployment.yaml
set:
redis.host: redis.example.com
redis.port: 6380
redis.passwordSecret.name: redis-secret
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: redis.example.com
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PORT
value: "6380"
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-secret
key: password
- it: backend emits the same redis env vars so both pods coordinate on one redis
template: backend/deployment.yaml
set:
redis.host: redis.example.com
redis.passwordSecret.name: redis-secret
redis.passwordSecret.passwordKey: redis-password
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: redis.example.com
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-secret
key: redis-password
- it: gateway omits REDIS_PASSWORD for an auth-less redis
template: gateway/deployment.yaml
set:
redis.host: redis.example.com
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PASSWORD
any: true
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: redis.example.com
- it: gateway seeds REDIS_CLUSTER_NODES from host and port in cluster mode
template: gateway/deployment.yaml
set:
redis.host: redis.example.com
redis.port: 6380
redis.cluster: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_CLUSTER_NODES
value: '[{"host":"redis.example.com","port":6380}]'
- it: gateway omits REDIS_CLUSTER_NODES when cluster mode is off
template: gateway/deployment.yaml
set:
redis.host: redis.example.com
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_CLUSTER_NODES
any: true

13
helm/litellm/values.yaml
View file

@ -100,7 +100,18 @@ database:
usernameKey: username
passwordKey: password
# Optional Redis (caching, rate limiting). Leave host empty to disable.
# Optional Redis. Leave host empty to disable.
#
# This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend
# tracking, and the pod lock manager. The chart emits REDIS_HOST / REDIS_PORT /
# REDIS_PASSWORD, which the proxy picks up through its coordination Redis env
# fallback. Response caching is separate and off unless you enable it in
# `proxy_config.litellm_settings.cache`.
#
# For full control, define `general_settings.coordination_redis` in
# `proxy_config` (host/port/password/username/url/ssl/startup_nodes/
# sentinel_nodes/sentinel_password/service_name, each accepting os.environ/VAR
# refs). An explicit block overrides these env vars.
#
# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster,
# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260710000000_add_dcr_bridge_to_mcp_server_table/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "dcr_bridge" BOOLEAN;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -339,6 +339,7 @@ model LiteLLM_MCPServerTable {
available_on_public_internet Boolean @default(true)
delegate_auth_to_upstream Boolean @default(false)
oauth_passthrough Boolean @default(false)
dcr_bridge Boolean?
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.75"
version = "0.4.76"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.75"
version = "0.4.76"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

16
litellm/_redis.py
View file

@ -325,8 +325,19 @@ def _get_redis_client_logic(**env_overrides):
value = get_secret(v) # type: ignore
env_overrides[k] = value
environment_kwargs = _redis_kwargs_from_environment()
# An explicitly configured connection target outranks REDIS_URL from the
# environment. Without this, the url branch below strips the caller's
# host/port/password and silently connects to whatever REDIS_URL names.
caller_named_a_target = any(
env_overrides.get(key) is not None for key in ("host", "startup_nodes", "sentinel_nodes")
)
if caller_named_a_target and env_overrides.get("url") is None:
environment_kwargs.pop("url", None)
redis_kwargs = {
**_redis_kwargs_from_environment(),
**environment_kwargs,
**env_overrides,
}
@ -678,9 +689,8 @@ def get_redis_connection_pool(
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
connection_class = async_redis.Connection
if "ssl" in redis_kwargs:
if redis_kwargs.pop("ssl", False):
connection_class = async_redis.SSLConnection
redis_kwargs.pop("ssl", None)
redis_kwargs["connection_class"] = connection_class
return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)

2
litellm/constants.py
View file

@ -715,6 +715,7 @@ openai_compatible_endpoints: List = [
"https://api.clarifai.com/v2/ext/openai/v1",
"https://api.libertai.io/v1",
"https://pinstripes.io/v1",
"https://api.meta.ai/v1",
]
@ -781,6 +782,7 @@ openai_compatible_providers: List = [
"ragflow",
"pinstripes", # Pinstripes - JSON-configured provider
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
]
openai_text_completion_compatible_providers: List = [ # providers that support `/v1/completions`
"together_ai",

6
litellm/cost_calculator.py
View file

@ -760,7 +760,11 @@ def _select_model_name_for_cost_calc(
if custom_pricing is True:
if router_model_id is not None and router_model_id in litellm.model_cost:
entry = litellm.model_cost[router_model_id]
if entry.get("input_cost_per_token") is not None or entry.get("input_cost_per_second") is not None:
if (
entry.get("input_cost_per_token") is not None
or entry.get("input_cost_per_second") is not None
or entry.get("tiered_pricing") is not None
):
return_model = router_model_id
else:
return_model = model

41
litellm/experimental_mcp_client/client.py
View file

@ -382,15 +382,25 @@ class MCPClient:
if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError):
raise root_cause from in_flight_error
async def run_with_session(self, operation: Callable[[ClientSession], Awaitable[TSessionResult]]) -> TSessionResult:
"""Open a session, run the provided coroutine, and clean up."""
async def run_with_session(
self,
operation: Callable[[ClientSession], Awaitable[TSessionResult]],
*,
quiet_on_error: bool = False,
) -> TSessionResult:
"""Open a session, run the provided coroutine, and clean up.
quiet_on_error demotes the failure line to debug for callers that own the exception
(call_tool / list_tools under raise_on_error), so an expected pass-through re-auth does
not emit a warning per call; every other caller keeps the operator-visible warning."""
http_client: Optional[httpx.AsyncClient] = None
try:
self._last_initialize_instructions = None
transport_ctx, http_client = self._create_transport_context()
return await self._execute_session_operation(transport_ctx, operation)
except Exception:
verbose_logger.warning("MCP client run_with_session failed for %s", self.server_url or "stdio")
_log = verbose_logger.debug if quiet_on_error else verbose_logger.warning
_log("MCP client run_with_session failed for %s", self.server_url or "stdio")
raise
finally:
if http_client is not None:
@ -491,7 +501,7 @@ class MCPClient:
return await session.list_tools()
try:
result = await self.run_with_session(_list_tools_operation)
result = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error)
tool_count = len(result.tools)
tool_names = [tool.name for tool in result.tools]
verbose_logger.info(f"MCP client listed {tool_count} tools from {self.server_url or 'stdio'}: {tool_names}")
@ -501,7 +511,13 @@ class MCPClient:
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.exception(
# Mirror call_tool: when the caller opted into raise_on_error it owns the exception and
# logs it at the fitting level (an expected pass-through re-auth 401 is info, not an
# error), so log at debug here to avoid an error-level line + traceback that would trip
# error-rate alerts on that expected signal. The swallow path still logs the full
# exception because nothing downstream will surface the failure.
_log = verbose_logger.debug if raise_on_error else verbose_logger.exception
_log(
f"MCP client list_tools failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
@ -510,7 +526,8 @@ class MCPClient:
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
_log_broken = verbose_logger.debug if raise_on_error else verbose_logger.error
_log_broken(
"MCP client detected broken connection/stream during list_tools - "
"the MCP server may have crashed, disconnected, or timed out"
)
@ -567,7 +584,7 @@ class MCPClient:
)
try:
tool_result = await self.run_with_session(_call_tool_operation)
tool_result = await self.run_with_session(_call_tool_operation, quiet_on_error=raise_on_error)
verbose_logger.info(f"MCP client tool call '{call_tool_request_params.name}' completed successfully")
return tool_result
except asyncio.CancelledError:
@ -580,7 +597,13 @@ class MCPClient:
verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
# When the caller opted into raise_on_error it owns the exception and logs it at the
# level that fits (an expected pass-through re-auth 401 is info, not an operator-actionable
# error), so log at debug here to avoid an error-level line that would trip error-rate
# alerts on that expected signal. The swallow path (raise_on_error=False) still logs at
# error because nothing downstream will surface the failure.
_log = verbose_logger.debug if raise_on_error else verbose_logger.error
_log(
f"MCP client call_tool failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
@ -590,7 +613,7 @@ class MCPClient:
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
_log(
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out."
)

46
litellm/integrations/custom_guardrail.py
View file

@ -1,3 +1,4 @@
import os
import secrets
from datetime import datetime
from typing import (
@ -17,6 +18,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys
from litellm.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.secret_managers.main import str_to_bool
from litellm.types.guardrails import (
DynamicGuardrailParams,
GuardrailEventHooks,
@ -59,6 +61,20 @@ from litellm.exceptions import (
_PRE_CALL_EXECUTED_TOKEN = secrets.token_hex(16)
def _strict_guardrail_modes_enabled() -> bool:
"""Whether guardrail-mode validation raises (default) or logs a warning.
Set `LITELLM_STRICT_GUARDRAIL_MODES=false` to keep the pre-LIT-4226 behavior
for guardrails whose supported_event_hooks list newly includes their
configured mode: log the mismatch and continue instead of raising at boot.
"""
raw = os.environ.get("LITELLM_STRICT_GUARDRAIL_MODES")
if raw is None:
return True
parsed = str_to_bool(raw)
return True if parsed is None else parsed
def get_session_id_from_request_data(request_data: Dict[str, Any]) -> Optional[str]:
"""Extract session_id from request data (litellm_session_id or metadata)."""
session_id = request_data.get("litellm_session_id")
@ -132,7 +148,17 @@ class CustomGuardrail(CustomLogger):
if supported_event_hooks:
## validate event_hook is in supported_event_hooks
self._validate_event_hook(event_hook, supported_event_hooks)
try:
self._validate_event_hook(event_hook, supported_event_hooks)
except ValueError as validation_error:
if _strict_guardrail_modes_enabled():
raise
verbose_logger.warning(
"%s. LITELLM_STRICT_GUARDRAIL_MODES=false; continuing "
"with unsupported event_hook. Set the env var to true "
"(default) to enforce validation and fail at startup.",
validation_error,
)
super().__init__(**kwargs)
def render_violation_message(self, default: str, context: Optional[Dict[str, Any]] = None) -> str:
@ -303,6 +329,18 @@ class CustomGuardrail(CustomLogger):
"""
return None
@classmethod
def get_supported_event_hooks(cls) -> Optional[List[GuardrailEventHooks]]:
"""
Returns the event hooks this guardrail supports, for the UI to render.
Subclasses should override to return their supported hooks list. When a
subclass returns None, the endpoint omits it from the per-provider map
and the UI is expected to fall back to the global `supported_modes`
list client-side.
"""
return None
def _validate_event_hook(
self,
event_hook: Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks], Mode]],
@ -757,6 +795,12 @@ class CustomGuardrail(CustomLogger):
# raw provider JSON so redaction is not duplicated upstream).
clean_guardrail_response = redact_nested_match_and_regex_keys(clean_guardrail_response)
from litellm.litellm_core_utils.sensitive_data_masker import (
mask_credentials_in_payload,
)
clean_guardrail_response = mask_credentials_in_payload(clean_guardrail_response)
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,

29
litellm/integrations/datadog/datadog.py
View file

@ -19,7 +19,7 @@ import os
import time
import traceback
from datetime import datetime as datetimeObj
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Optional, Sequence, Union
import httpx
from httpx import Response
@ -50,6 +50,7 @@ from litellm.types.integrations.base_health_check import IntegrationHealthCheckS
from litellm.types.integrations.datadog import (
DD_ERRORS,
DD_MAX_BATCH_SIZE,
DD_MAX_PAYLOAD_SIZE_BYTES,
DataDogStatus,
DatadogInitParams,
DatadogPayload,
@ -384,8 +385,10 @@ class DataDogLogger(
async def _send_with_413_split(self, batch: List) -> List:
"""
Send a batch, halving any sub-batch that 413s (payload too large) and retrying the
halves, since Datadog enforces a 5MB uncompressed limit per request.
Send a batch, halving any sub-batch that exceeds Datadog's intake limits before
sending, and halving again on a 413 (payload too large) response, since Datadog
enforces a 5MB uncompressed limit per request. The proactive split avoids paying
a serialize + gzip + round trip for a payload the intake is guaranteed to reject.
A 413 surfaces as a raised MaskedHTTPStatusError (httpx raise_for_status), not a
returned response, so both paths are handled. A lone event that still 413s is
@ -398,6 +401,11 @@ class DataDogLogger(
chunk = pending.pop()
if not chunk:
continue
if len(chunk) > 1 and self._exceeds_intake_limits(chunk):
mid = len(chunk) // 2
pending.append(chunk[mid:])
pending.append(chunk[:mid])
continue
try:
response = await self.async_send_compressed_data(chunk)
except Exception as e:
@ -436,6 +444,21 @@ class DataDogLogger(
def _undelivered(chunk: List, pending: List[List]) -> List:
return chunk + [event for remaining in reversed(pending) for event in remaining]
@staticmethod
def _exceeds_intake_limits(chunk: Sequence[DatadogPayload]) -> bool:
"""
True when a chunk would breach Datadog's log intake limits: more than
DD_MAX_BATCH_SIZE events per payload, or a serialized size above
DD_MAX_PAYLOAD_SIZE_BYTES (held under Datadog's 5MB uncompressed cap so
the batch is split before the intake rejects it with a 413).
"""
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
if len(chunk) > DD_MAX_BATCH_SIZE:
return True
payload_size_bytes = len(safe_dumps(chunk).encode("utf-8"))
return payload_size_bytes > DD_MAX_PAYLOAD_SIZE_BYTES
async def flush_queue(self):
if self.flush_lock is None:
return

9
litellm/integrations/otel/README.md
View file

@ -223,6 +223,15 @@ lives in [`plumbing/`](./plumbing):
readers/exporters receive them alongside the server metrics, and one is built
and registered as the global only when none is set (mirroring how V2 owns trace
export).
- [`events.py`](./plumbing/events.py) — GenAI client events. Gated on
`enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call
records the semconv `gen_ai.client.operation.exception` log event at severity
WARN, carrying `exception.type` / `exception.message` / `exception.stacktrace`
and correlated to the failed span through the trace and span ids. The
`LoggerProvider` is resolved like the meter provider, except that an explicit
`NoOpLoggerProvider` global is an operator opt-out that builds no recorder at
all. The deprecated `error.*` span attributes and the `exception` span event
are still stamped by the emitter for backwards compatibility.
### Adapter

11
litellm/integrations/otel/emitter.py
View file

@ -18,6 +18,7 @@ from litellm.integrations.otel.model.payloads import (
ServiceSpanData,
SpanError,
)
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
from litellm.integrations.otel.model.spans import (
@ -77,9 +78,11 @@ class SpanEmitter:
tracer: Tracer,
config: OpenTelemetryV2Config,
mappers: Sequence[AttributeMapper] | None = None,
event_recorder: GenAIEventRecorder | None = None,
) -> None:
self._tracer = tracer
self._config = config
self._event_recorder = event_recorder
# The mapper chain is the sole source of span attributes. When not
# passed in, resolve it from the config so there's one source of truth.
self._mappers: list[AttributeMapper] = (
@ -223,6 +226,14 @@ class SpanEmitter:
ExceptionEvent.NAME,
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
)
if self._event_recorder is not None and role is SpanRole.LLM_CALL:
self._event_recorder.record_operation_exception(
span_context=span.get_span_context(),
error_type=error_type,
message=message,
stack_trace=error.stack_trace,
timestamp_ns=end_time_ns,
)
# On success leave the status UNSET (the semconv default) rather than
# forcing OK — that matches the FastAPI server span and avoids implying a
# span-level health signal litellm doesn't actually evaluate. Only a

29
litellm/integrations/otel/logger.py
View file

@ -6,6 +6,7 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast
from opentelemetry.context import Context, attach, get_current
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Span, Tracer, get_current_span, use_span
@ -40,14 +41,17 @@ from litellm.integrations.otel.model.payloads import (
is_mcp_list_tools,
is_mcp_tool_call,
)
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
from litellm.integrations.otel.plumbing.metrics import (
GenAIMetricRecorder,
create_genai_metrics,
)
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
get_event_logger,
get_meter,
get_tracer,
resolve_logger_provider,
resolve_meter_provider,
)
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
@ -104,7 +108,7 @@ class OpenTelemetryV2(CustomLogger):
config: OpenTelemetryV2Config | None = None,
callback_name: str | None = None,
tracer_provider: TracerProvider | None = None,
logger_provider: Any | None = None, # reserved for OTel logs
logger_provider: LoggerProvider | None = None,
meter_provider: Any | None = None,
**kwargs: Any,
) -> None:
@ -117,7 +121,12 @@ class OpenTelemetryV2(CustomLogger):
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
self._metrics_recorder = self._init_metrics(meter_provider)
self._metric_filter_error_logged = False
self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names))
self._emitter = SpanEmitter(
self.tracer,
self.config,
mappers=resolve_mappers(self.config.mapper_names),
event_recorder=self._init_events(logger_provider),
)
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
self._init_otel_logger_on_litellm_proxy()
@ -136,6 +145,22 @@ class OpenTelemetryV2(CustomLogger):
meter = get_meter(provider, LITELLM_TRACER_NAME)
return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name)
def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None":
"""Create the GenAI event recorder when events are enabled, else ``None``.
``logger_provider`` is an explicit override (tests inject one); otherwise the
provider is resolved from the OTel global so an operator-configured logs
pipeline receives the events, building and registering one only when no
global provider is set. A ``None`` resolution means the operator opted out
of the logs signal, so no recorder is built.
"""
if not self.config.enable_events:
return None
provider = resolve_logger_provider(self.config, logger_provider)
if provider is None:
return None
return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME))
# ====================================================================== #
# Proxy global registration
# ====================================================================== #

13
litellm/integrations/otel/model/semconv.py
View file

@ -177,6 +177,19 @@ class ExceptionEvent:
NAME: Final = "exception"
TYPE: Final = "exception.type"
MESSAGE: Final = "exception.message"
STACKTRACE: Final = "exception.stacktrace"
class GenAIEvent:
"""GenAI semconv event names, from the GenAI registry's *events* section.
``gen_ai.client.operation.exception`` is defined as a log-based event
(severity WARN) carrying the ``exception.*`` trio, correlated to the failed
span via the trace/span ids — the semconv-compliant home for GenAI failure
details, unlike the deprecated ``error.message`` span attribute.
"""
OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception"
class Server:

52
litellm/integrations/otel/plumbing/events.py Normal file
View file

@ -0,0 +1,52 @@
"""GenAI client events: the ``gen_ai.client.operation.exception`` log event.
The GenAI semantic conventions define exception recording for client
operations as a log-based event (severity WARN) carrying the ``exception.*``
attribute trio, correlated to the failed span through the trace/span ids —
not as a span attribute or span event. This module owns building and
emitting that event; the exporter pipeline it rides is built in
:mod:`litellm.integrations.otel.plumbing.providers`.
"""
from dataclasses import dataclass
from opentelemetry._events import Event, EventLogger
from opentelemetry._logs.severity import SeverityNumber
from opentelemetry.trace import SpanContext
from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent
@dataclass(frozen=True, slots=True)
class GenAIEventRecorder:
event_logger: EventLogger
def record_operation_exception(
self,
span_context: SpanContext,
error_type: str,
message: str,
stack_trace: str | None,
timestamp_ns: int | None,
) -> None:
# ``exception.type`` and ``exception.message`` are the semconv-required
# pair and always ride the event; only the recommended stacktrace is
# conditional on the payload carrying one.
stacktrace = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else ()
self.event_logger.emit(
Event(
name=GenAIEvent.OPERATION_EXCEPTION,
timestamp=timestamp_ns,
trace_id=span_context.trace_id,
span_id=span_context.span_id,
trace_flags=span_context.trace_flags,
severity_number=SeverityNumber.WARN,
attributes=dict(
(
(ExceptionEvent.TYPE, error_type),
(ExceptionEvent.MESSAGE, message),
*stacktrace,
)
),
)
)

119
litellm/integrations/otel/plumbing/providers.py
View file

@ -2,9 +2,20 @@
from typing import TYPE_CHECKING, Any, Callable, Iterable
from opentelemetry import baggage, metrics
from opentelemetry import _logs, baggage, metrics
from opentelemetry._events import EventLogger
from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider
from opentelemetry.context import Context
from opentelemetry.metrics import MeterProvider, NoOpMeterProvider
from opentelemetry.sdk._events import EventLoggerProvider
from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider
from opentelemetry.sdk._logs.export import (
BatchLogRecordProcessor,
ConsoleLogExporter,
InMemoryLogExporter,
LogExporter,
SimpleLogRecordProcessor,
)
from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
@ -224,6 +235,112 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
def _otlp_logs_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path.
The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads
``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
for the logs signal (rewriting a sibling signal path when present).
"""
if not endpoint:
return endpoint
endpoint = endpoint.rstrip("/")
if endpoint.endswith("/v1/logs"):
return endpoint
for other_signal in ("/v1/traces", "/v1/metrics"):
if endpoint.endswith(other_signal):
return endpoint[: -len(other_signal)] + "/v1/logs"
return endpoint + "/v1/logs"
def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter:
"""Build a log exporter mirroring the exporter selection of the other signals.
``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
and ``otlp_grpc`` export over OTLP with the configured endpoint/headers;
``in_memory`` buffers for tests. Like GenAI metrics, events ride the
single-destination shorthand fields, not the multi-exporter ``exporters`` list.
"""
kind = (config.exporter or "console").lower()
if kind in ("in_memory", "inmemory", "memory"):
return InMemoryLogExporter()
if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
OTLPLogExporter as HTTPLogExporter,
)
return HTTPLogExporter(
endpoint=_otlp_logs_endpoint(config.endpoint),
headers=parse_headers(config.headers),
)
if kind in ("otlp_grpc", "grpc"):
try:
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
OTLPLogExporter as GRPCLogExporter,
)
except ImportError as exc:
raise ImportError(
"OpenTelemetry OTLP gRPC log exporter is not available. Install "
"`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)."
) from exc
return GRPCLogExporter(endpoint=config.endpoint, headers=parse_headers(config.headers))
return ConsoleLogExporter()
def build_logger_provider(
config: OpenTelemetryV2Config,
log_exporter: LogExporter | None = None,
) -> SDKLoggerProvider:
"""Build the :class:`LoggerProvider` GenAI events export through.
``log_exporter`` is an explicit override (tests inject an
``InMemoryLogExporter``); otherwise the exporter is selected from the config's
exporter kind via :func:`build_log_exporter`. Console and in-memory exporters
get a Simple processor (synchronous export, which tests rely on), everything
else a Batch processor — the same split as span processing.
"""
exporter = log_exporter if log_exporter is not None else build_log_exporter(config)
provider = SDKLoggerProvider(resource=build_resource(config))
use_simple = isinstance(exporter, (ConsoleLogExporter, InMemoryLogExporter))
provider.add_log_record_processor(
SimpleLogRecordProcessor(exporter) if use_simple else BatchLogRecordProcessor(exporter)
)
return provider
def resolve_logger_provider(
config: OpenTelemetryV2Config,
logger_provider: SDKLoggerProvider | None = None,
) -> SDKLoggerProvider | None:
"""Resolve the :class:`LoggerProvider` GenAI events record through, or ``None``
when the operator has opted out of the logs signal.
Same resolution order as :func:`resolve_meter_provider`: an injected provider
wins (DI/tests); an operator-configured SDK global is reused so events ride
their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and
yields ``None``, so no event is ever built. Only the default placeholder
global makes V2 build a provider from the config and publish it as the global.
"""
if logger_provider is not None:
return logger_provider
existing: LoggerProvider = _logs.get_logger_provider()
if isinstance(existing, SDKLoggerProvider):
return existing
if isinstance(existing, NoOpLoggerProvider):
return None
provider = build_logger_provider(config)
_logs.set_logger_provider(provider)
return provider
def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger:
return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version)
def build_meter_provider(
config: OpenTelemetryV2Config,
metric_reader: "MetricReader | None" = None,

20
litellm/integrations/prometheus.py
View file

@ -1618,6 +1618,14 @@ class PrometheusLogger(CustomLogger):
user_id: Optional[str] = None,
user_api_key_org_id: Optional[str] = None,
):
if (
isinstance(self.litellm_remaining_team_budget_metric, NoOpMetric)
and isinstance(self.litellm_remaining_api_key_budget_metric, NoOpMetric)
and isinstance(self.litellm_remaining_user_budget_metric, NoOpMetric)
and isinstance(self.litellm_remaining_org_budget_metric, NoOpMetric)
):
return
_metadata = litellm_params.get("metadata") or {}
_team_spend = _metadata.get("user_api_key_team_spend", None)
_team_max_budget = _metadata.get("user_api_key_team_max_budget", None)
@ -3332,6 +3340,9 @@ class PrometheusLogger(CustomLogger):
- looks up team info from db if not available in metadata
- Set team budget metrics
"""
if isinstance(self.litellm_remaining_team_budget_metric, NoOpMetric):
return
if user_api_team:
team_object = await self._assemble_team_object(
team_id=user_api_team,
@ -3453,6 +3464,9 @@ class PrometheusLogger(CustomLogger):
- Fetches org info via cache (get_org_object)
- Sets org budget metrics
"""
if isinstance(self.litellm_remaining_org_budget_metric, NoOpMetric):
return
if not org_id:
return
@ -3582,6 +3596,9 @@ class PrometheusLogger(CustomLogger):
key_max_budget: Optional[float],
key_spend: Optional[float],
):
if isinstance(self.litellm_remaining_api_key_budget_metric, NoOpMetric):
return
if user_api_key:
user_api_key_dict = await self._assemble_key_object(
user_api_key=user_api_key,
@ -3642,6 +3659,9 @@ class PrometheusLogger(CustomLogger):
- looks up user info from db if not available in metadata
- Set user budget metrics
"""
if isinstance(self.litellm_remaining_user_budget_metric, NoOpMetric):
return
if user_id:
user_object = await self._assemble_user_object(
user_id=user_id,

7
litellm/integrations/rubrik.py
View file

@ -7,7 +7,7 @@ import time
import urllib.parse
import uuid
from collections import Counter
from typing import TYPE_CHECKING, Any, Literal, Optional
from typing import TYPE_CHECKING, Any, List, Literal, Optional
import httpx
from litellm._logging import verbose_logger
@ -52,6 +52,10 @@ class _MalformedToolBlockingResponseError(Exception):
class RubrikLogger(CustomGuardrail, CustomBatchLogger):
@classmethod
def get_supported_event_hooks(cls) -> List[GuardrailEventHooks]:
return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call]
def __init__(
self,
api_key: str | None = None,
@ -69,6 +73,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger):
kwargs["event_hook"] = kwargs.get("event_hook") or GuardrailEventHooks.post_call
if kwargs.get("default_on") is None:
kwargs["default_on"] = True
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
super().__init__(
flush_lock=self.flush_lock,
**kwargs,

15
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -7,6 +7,7 @@ This module has no dependencies on proxy code and can be safely imported at the
import json
import os
import time
from pathlib import Path
from typing import Optional
@ -68,3 +69,17 @@ def get_litellm_gateway_api_key(
if stored_url != expected_base_url.rstrip("/"):
return None
return token_data["key"]
def is_cli_token_fresh(token_data: dict, buffer_hours: float = 0.1) -> bool:
"""Check whether a cached CLI token (as stored in token.json) is still
within its expiration window. Used by `lite auth print-token` to fail
fast, without a network round trip, once the cached token is past
`LITELLM_CLI_JWT_EXPIRATION_HOURS`."""
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
timestamp = token_data.get("timestamp")
if not isinstance(timestamp, (int, float)):
return False
age_hours = (time.time() - timestamp) / 3600
return age_hours < (CLI_JWT_EXPIRATION_HOURS - buffer_hours)

3
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -95,6 +95,9 @@ class ExceptionCheckers:
if "current length is" in _error_str_lowercase and "while limit is" in _error_str_lowercase:
return True
if "maximum input length is" in _error_str_lowercase and "tokens" in _error_str_lowercase:
return True
return False
@staticmethod

225
litellm/litellm_core_utils/fallback_generalizations.py
View file

@ -3,52 +3,69 @@ Declarative fallback generalizations for unknown / newly-released models.
The ``fallback_generalizations`` block in ``model_prices_and_context_window.json``
holds an ordered list of rules. Each rule pairs a single case-insensitive regex
with the metadata to apply when a model name has no exact entry in the cost map.
The metadata is a partial cost-map entry: ``litellm_provider`` drives provider
routing, and the remaining fields (``mode``, ``supports_*``, context window,
pricing, ...) drive ``get_model_info`` / ``supports_*``.
with a ``model_info`` dict, and the structure of ``model_info`` decides which of
two kinds the rule is.
Precedence: rules are evaluated in file order and the first match wins. They are
consulted only after exact and case-insensitive lookups miss, so an exact entry
always takes precedence over a rule.
A ROUTING rule carries exactly one ``model_info`` key, ``litellm_provider``. It is
consumed only by ``get_llm_provider`` bare-id inference: the first routing rule
whose regex matches decides the provider. Routing rules never contribute to model
info.
A CAPABILITY rule carries any ``model_info`` keys except ``litellm_provider``
(``mode``, ``supports_*``, context window, pricing, ...). It is consumed by
``get_model_info`` fallback resolution: the ``model_info`` of ALL capability rules
whose regex matches is unioned in file order, with later rules overriding earlier
ones on key conflicts, and the caller backfills ``litellm_provider`` with the
provider it requested. If no capability rule matches, model-info resolution misses
as if no rules existed.
LEGACY-SCHEMA SHIM (temporary, until the new-schema JSON reaches main): released
proxies fetch this JSON remotely from main, whose block still ships the old schema
where a rule mixes ``litellm_provider`` with capability keys and may inherit a
parent's ``model_info`` via ``extends``. Such a legacy rule is tolerated rather
than skipped: ``extends`` is resolved once at install time (single level, against
raw parents), and the resolved rule acts as BOTH kinds, a routing rule (its
``litellm_provider`` participates in first-hit inference) and a capability rule
(its full ``model_info``, provider included, participates in the union). New-schema
rules never mix the two and never use ``extends``. A rule whose
``litellm_provider`` is not a string is invalid and is warned about and skipped
(a warning rather than a crash, for the same remote-fetch reason).
Rules are only consulted after exact and case-insensitive lookups miss, so an
exact cost-map entry always takes precedence over any rule.
Patterns are matched case-insensitively with ``re.search`` and are not implicitly
anchored: a rule must include ``^`` and ``$`` (as the shipped rules do) to bind to
the whole model name, otherwise it matches as a substring. Keeping anchoring in the
regex makes the rule the single, self-contained source of truth for what it matches.
A rule may set ``extends`` to the ``name`` of another rule to inherit that rule's
``model_info``; the rule's own ``model_info`` overrides the inherited keys, so a
narrow rule (for example a version-gated capability flag) carries only its delta
instead of duplicating the parent's pricing block. Inheritance is resolved once,
at install time, against each rule's raw (unresolved) ``model_info``; it is a
single level (a parent that itself extends is not chained).
anchored: a rule must include ``^`` and ``$`` to bind to the whole model name,
otherwise it matches as a substring. Keeping anchoring in the regex makes the rule
the single, self-contained source of truth for what it matches.
Any other keys on a rule (for example a free-text ``description`` documenting what
the regex matches) are ignored by the engine and exist only for the reader.
The compiled-regex list is built once and cached. ``match_fallback_generalization``
is O(number of rules); callers must only invoke it on a cache miss.
Rules are compiled and classified once, at install time. The match functions are
O(number of rules); callers must only invoke them on a cache miss.
"""
import re
from typing import Optional
from dataclasses import dataclass
from typing import Optional, Union
from litellm._logging import verbose_logger
NAME_FIELD = "name"
PATTERN_FIELD = "pattern"
MODEL_INFO_FIELD = "model_info"
EXTENDS_FIELD = "extends"
PROVIDER_KEY = "litellm_provider"
LEGACY_EXTENDS_FIELD = "extends"
def _resolve_extends(rules: list) -> list:
"""Expand ``extends`` inheritance so each rule's ``model_info`` is self-contained.
def _resolve_legacy_extends(rules: list) -> list:
"""Expand legacy ``extends`` inheritance so each rule's ``model_info`` is self-contained.
A rule with ``extends: <name>`` is rewritten with ``model_info`` set to the parent's
``model_info`` overlaid by its own. Resolution is single-level and uses each rule's
raw ``model_info`` as the parent source. Non-dict rules and dangling parents are
passed through unchanged.
Compatibility shim for the old remote schema: single level, resolved against each
parent's raw ``model_info``, with the child's own keys winning on conflict. Non-dict
rules and dangling parents pass through unchanged; new-schema rules carry no
``extends`` and are untouched.
"""
base_by_name = {
rule[NAME_FIELD]: rule[MODEL_INFO_FIELD]
@ -58,84 +75,138 @@ def _resolve_extends(rules: list) -> list:
and isinstance(rule.get(MODEL_INFO_FIELD), dict)
}
def resolved(rule: dict) -> dict:
parent_name = rule.get(EXTENDS_FIELD)
def resolved(rule: object) -> object:
if not isinstance(rule, dict):
return rule
parent_name = rule.get(LEGACY_EXTENDS_FIELD)
own_info = rule.get(MODEL_INFO_FIELD)
parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None
if parent_info is None or not isinstance(own_info, dict):
return rule
return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}}
return [resolved(rule) if isinstance(rule, dict) else rule for rule in rules]
return [resolved(rule) for rule in rules]
@dataclass(frozen=True, slots=True)
class _RoutingRule:
pattern: re.Pattern
provider: str
@dataclass(frozen=True, slots=True)
class _CapabilityRule:
pattern: re.Pattern
model_info: dict
_CompiledRule = Union[_RoutingRule, _CapabilityRule]
def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:
if not isinstance(rule, dict):
return ()
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get(NAME_FIELD, pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
return ()
try:
compiled = re.compile(pattern, re.IGNORECASE)
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return ()
if PROVIDER_KEY not in model_info:
return (_CapabilityRule(pattern=compiled, model_info=model_info),)
provider = model_info[PROVIDER_KEY]
if not isinstance(provider, str):
verbose_logger.warning(
"LiteLLM: skipping invalid fallback generalization rule %s: '%s' in '%s' must be a string.",
rule.get(NAME_FIELD, pattern),
PROVIDER_KEY,
MODEL_INFO_FIELD,
)
return ()
if len(model_info) == 1:
return (_RoutingRule(pattern=compiled, provider=provider),)
return (
_RoutingRule(pattern=compiled, provider=provider),
_CapabilityRule(pattern=compiled, model_info=model_info),
)
class _FallbackGeneralizations:
"""Holds the active rule list and its lazily-compiled regex cache."""
"""Holds the raw rule list and its install-time-compiled routing and capability rules."""
def __init__(self) -> None:
self.rules: list[dict] = []
self._compiled: Optional[list[tuple[re.Pattern, dict]]] = None
self.rules: list = []
self.routing_rules: tuple = ()
self.capability_rules: tuple = ()
def set_rules(self, rules: Optional[list[dict]]) -> None:
self.rules = rules if isinstance(rules, list) else []
self._compiled = None
def set_rules(self, rules: Optional[list]) -> None:
installed = rules if isinstance(rules, list) else []
compiled = tuple(kind for rule in _resolve_legacy_extends(installed) for kind in _compile_rule(rule))
self.rules = installed
self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule))
self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule))
def _compile(self) -> list[tuple[re.Pattern, dict]]:
compiled: list[tuple[re.Pattern, dict]] = []
for rule in self.rules:
if not isinstance(rule, dict):
continue
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get("name", pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
continue
try:
compiled.append((re.compile(pattern, re.IGNORECASE), model_info))
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return compiled
def match(self, model: str) -> Optional[dict]:
def match_routing(self, model: str) -> Optional[str]:
if not model:
return None
if self._compiled is None:
self._compiled = self._compile()
for pattern, model_info in self._compiled:
if pattern.search(model) is not None:
return dict(model_info)
return None
return next(
(rule.provider for rule in self.routing_rules if rule.pattern.search(model) is not None),
None,
)
def match_capabilities(self, model: str) -> Optional[dict]:
if not model:
return None
matched = tuple(rule.model_info for rule in self.capability_rules if rule.pattern.search(model) is not None)
if not matched:
return None
return {key: value for model_info in matched for key, value in model_info.items()}
_registry = _FallbackGeneralizations()
def set_fallback_generalizations(rules: Optional[list[dict]]) -> None:
"""Install the active rule list and invalidate the compiled-regex cache.
def set_fallback_generalizations(rules: Optional[list]) -> None:
"""Install the active rule list, compiling and classifying each rule.
``extends`` inheritance is resolved here, once, before the rules are stored.
Called once when the model cost map is loaded (and again on any reload).
Legacy ``extends`` inheritance is resolved here, once, before classification;
a legacy rule mixing ``litellm_provider`` with capability keys installs as both
kinds. Malformed and invalid-regex rules are warned about and skipped. Called
once when the model cost map is loaded (and again on any reload).
"""
_registry.set_rules(_resolve_extends(rules) if isinstance(rules, list) else rules)
_registry.set_rules(rules)
def get_fallback_generalization_rules() -> list[dict]:
def get_fallback_generalization_rules() -> list:
"""Return the raw rule list (read-only view for callers/tests)."""
return _registry.rules
def match_fallback_generalization(model: str) -> Optional[dict]:
"""Return the ``model_info`` of the first rule whose regex matches ``model``.
def match_routing_generalization(model: str) -> Optional[str]:
"""Return the provider of the first routing rule whose regex matches ``model``.
O(number of rules). Only call this once exact lookups have missed.
"""
return _registry.match(model)
return _registry.match_routing(model)
def match_capability_generalizations(model: str) -> Optional[dict]:
"""Return the union of the ``model_info`` of every capability rule matching ``model``.
Later rules override earlier ones on key conflicts. Returns ``None`` when no
capability rule matches. O(number of rules); only call once exact lookups have missed.
"""
return _registry.match_capabilities(model)

11
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -4,7 +4,7 @@ from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_fallback_generalization,
match_routing_generalization,
)
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.secret_managers.main import get_secret, get_secret_str
@ -346,6 +346,9 @@ def get_llm_provider(
elif endpoint == "https://pinstripes.io/v1":
custom_llm_provider = "pinstripes"
dynamic_api_key = get_secret_str("PINSTRIPES_API_KEY")
elif endpoint == "https://api.meta.ai/v1":
custom_llm_provider = "meta"
dynamic_api_key = get_secret_str("META_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))
@ -471,12 +474,10 @@ def get_llm_provider(
custom_llm_provider = "sap"
# Last resort for an otherwise-unknown model: a declarative
# fallback-generalization rule (e.g. routes future claude-* to anthropic).
# fallback-generalization routing rule (e.g. routes future claude-* to anthropic).
# Exact provider matches above always win; this only runs on a miss.
if not custom_llm_provider:
generalization = match_fallback_generalization(model)
if generalization is not None:
custom_llm_provider = generalization.get("litellm_provider") or None
custom_llm_provider = match_routing_generalization(model)
if not custom_llm_provider:
if litellm.suppress_debug_info is False:

139
litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py Normal file
View file

@ -0,0 +1,139 @@
"""
Provider-neutral graduated tiered pricing calculation.
Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
reservation logic so neither has to depend on the other.
"""
from typing import List, Optional, Union
def _coerce_cost_per_token(value: Union[float, int, str, None]) -> float:
"""
Coerce a per-token cost into a float.
Model cost values loaded from YAML config may arrive as strings (e.g.
scientific notation like "4e-07"), which would break arithmetic.
"""
if value is None:
return 0.0
if isinstance(value, str):
try:
return float(value)
except ValueError:
return 0.0
return float(value)
def calculate_tiered_cost(
tokens: int,
tiered_pricing: List[dict],
cost_key: str,
fallback_cost_key: Optional[str] = None,
) -> float:
"""
Calculate cost for a given number of tokens based on a true tiered pricing structure.
This function iterates through sorted pricing tiers, calculates the cost for the
number of tokens that fall into each tier's range, and sums them up to get the total cost.
Args:
tokens (int): The total number of tokens to calculate the cost for.
tiered_pricing (List[dict]): A list of dictionaries, where each dictionary
represents a pricing tier.
cost_key (str): The key in the tier dictionary that holds the per-token cost
(e.g., 'input_cost_per_token').
fallback_cost_key (Optional[str], optional): A fallback key to use if the
primary `cost_key` is not found in a tier. Defaults to None.
Returns:
float: The total calculated cost for the given tokens.
Example:
>>> tiered_pricing = [
... {"range": [0, 100000], "input_cost_per_token": 0.0001},
... {"range": [100000, 500000], "input_cost_per_token": 0.00005},
... ]
Calculating cost for 150,000 tokens:
(100,000 * 0.0001) + (50,000 * 0.00005) = $12.5
"""
if not tiered_pricing or tokens <= 0:
return 0.0
total_cost = 0.0
tokens_processed = 0
sorted_tiers = sorted(tiered_pricing, key=lambda x: x.get("range", [0, 0])[0])
for tier in sorted_tiers:
if tokens_processed >= tokens:
break
tier_range = tier.get("range", [])
if len(tier_range) != 2:
continue
range_start, range_end = tier_range
if tokens <= range_start:
continue
tier_start = max(range_start, tokens_processed)
tier_end = min(range_end, tokens)
if tier_end > tier_start:
tokens_in_tier = tier_end - tier_start
cost_per_token = tier.get(cost_key) or tier.get(fallback_cost_key, 0)
total_cost += tokens_in_tier * _coerce_cost_per_token(cost_per_token)
tokens_processed = tier_end
# After loop, check if any tokens remain (i.e., tokens > highest tier's end range)
# and charge them at the last tier's rate.
if tokens_processed < tokens and sorted_tiers:
last_tier = sorted_tiers[-1]
remaining_tokens = tokens - tokens_processed
cost_per_token = last_tier.get(cost_key) or last_tier.get(fallback_cost_key, 0)
total_cost += remaining_tokens * _coerce_cost_per_token(cost_per_token)
return total_cost
def select_tier_for_input(
tiered_pricing: List[dict],
input_tokens: int,
) -> Optional[dict]:
"""
Select the pricing tier for a request based on its total input token count.
Alibaba Model Studio (Dashscope) tiered pricing is all-or-nothing: the tier is
chosen by the total input tokens of a single request and every token in the
request (input and output) is billed at that one tier's rate, rather than
graduated income-tax-style slicing. A tier matches when
``range_start < input_tokens <= range_end`` (so a request of exactly
``range_end`` tokens stays in the lower tier, matching the official
``0 < Token <= 32K`` phrasing). Requests above the highest declared range fall
back to the last (most expensive) tier.
"""
if not tiered_pricing or input_tokens <= 0:
return None
sorted_tiers = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
valid_tiers = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
if not valid_tiers:
return None
matching = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
if matching:
return matching[0]
return valid_tiers[-1]
def tier_rate(
tier: dict,
cost_key: str,
fallback_cost_key: Optional[str] = None,
) -> float:
"""Read a per-token rate from a tier, coercing YAML string costs to float."""
raw = tier.get(cost_key) or tier.get(fallback_cost_key, 0)
return _coerce_cost_per_token(raw)

35
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,6 +1,8 @@
from collections.abc import Mapping
from typing import Any, Dict, List, Optional, Set
from pydantic import BaseModel
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
@ -153,6 +155,39 @@ def mask_sensitive_structure(data: object) -> object:
return _error_masker.mask(data)
def mask_credentials_in_payload(data: object) -> object:
"""Return a copy of ``data`` where string values under sensitive-named keys
are masked but every other value (``None``, ``int``, ``float``, ``bool``,
``bytes``, ``datetime``, tuples, sets, typed objects) is preserved by
identity, and dicts/lists are rebuilt structurally.
Use this for logging payloads that carry response data through to
SpendLogs / OTel / Langfuse, where :meth:`SensitiveDataMasker.mask`'s
config-dump semantics (``None`` -> ``"None"``, tuples stringified,
objects flattened via ``__dict__``) would silently distort the record.
Sensitive-key detection is delegated to the shared
:class:`SensitiveDataMasker` so pattern updates stay in one place.
"""
return _walk_payload(data, key_is_sensitive=False, depth=0)
def _walk_payload(node: object, key_is_sensitive: bool, depth: int) -> object:
if depth >= DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER:
return node
if isinstance(node, Mapping):
return {k: _walk_payload(v, _default_masker.is_sensitive_key(k), depth + 1) for k, v in node.items()}
if isinstance(node, list):
return [_walk_payload(item, key_is_sensitive, depth + 1) for item in node]
if isinstance(node, tuple):
return tuple(_walk_payload(item, key_is_sensitive, depth + 1) for item in node)
if isinstance(node, BaseModel):
return _walk_payload(node.model_dump(), key_is_sensitive, depth)
if key_is_sensitive and isinstance(node, str) and node:
return _default_masker._mask_value(node)
return node
def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]:
"""Return a new dict with values masked for keys listed in ``sensitive_fields``.

46
litellm/llms/anthropic/chat/transformation.py
View file

@ -266,6 +266,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def custom_llm_provider(self) -> Optional[str]:
return "anthropic"
@property
def _resolved_provider(self) -> str:
return self.custom_llm_provider or "anthropic"
@classmethod
def get_config(cls, *, model: Optional[str] = None):
config = super().get_config()
@ -335,23 +339,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return any(v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7"))
@staticmethod
def _supports_effort_level(model: str, level: str) -> bool:
def _supports_effort_level(model: str, level: str, custom_llm_provider: str) -> bool:
"""Check ``supports_{level}_reasoning_effort`` in the model map."""
return AnthropicConfig._supports_model_capability(model, f"supports_{level}_reasoning_effort")
return AnthropicConfig._supports_model_capability(
model, f"supports_{level}_reasoning_effort", custom_llm_provider
)
@staticmethod
def _validate_effort_for_model(model: str, effort: Optional[str]) -> Optional[str]:
def _validate_effort_for_model(model: str, effort: Optional[str], custom_llm_provider: str) -> Optional[str]:
"""Return ``None`` if ``effort`` is allowed on ``model``, else an error message."""
if effort == "max" and not (
AnthropicConfig._is_adaptive_thinking_model(model) or AnthropicConfig._supports_effort_level(model, "max")
AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider)
or AnthropicConfig._supports_effort_level(model, "max", custom_llm_provider)
):
return f"effort='max' is not supported by this model. Got model: {model}"
if effort == "xhigh" and not AnthropicConfig._supports_effort_level(model, "xhigh"):
if effort == "xhigh" and not AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider):
return f"effort='xhigh' is not supported by this model. Got model: {model}"
return None
@staticmethod
def _model_supports_effort_param(model: str) -> bool:
def _model_supports_effort_param(model: str, custom_llm_provider: str) -> bool:
"""Whether the model accepts ``output_config.effort`` at all.
A model qualifies if its map entry advertises ``supports_output_config``
@ -359,10 +366,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
signals: e.g. Claude Opus 4.5 supports ``output_config`` without
advertising a non-default (max/xhigh) effort level.
"""
if AnthropicConfig._supports_model_capability(model, "supports_output_config"):
if AnthropicConfig._supports_model_capability(model, "supports_output_config", custom_llm_provider):
return True
return any(
AnthropicConfig._supports_effort_level(model, level)
AnthropicConfig._supports_effort_level(model, level, custom_llm_provider)
for level in ("low", "minimal", "medium", "high", "xhigh", "max")
)
@ -451,7 +458,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if (
"claude-3-7-sonnet" in model
or AnthropicConfig._is_adaptive_thinking_model(model)
or AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider)
or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
@ -1159,11 +1166,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def _map_reasoning_effort(
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
model: str,
custom_llm_provider: str,
llm_provider: str = "anthropic",
) -> Optional[AnthropicThinkingParam]:
"""Capability probes read the cost map under ``custom_llm_provider``; ``llm_provider`` only tags raised exceptions."""
if reasoning_effort is None or reasoning_effort == "none":
return None
if AnthropicConfig._is_adaptive_thinking_model(model):
if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
return AnthropicThinkingParam(
type="adaptive",
)
@ -1471,20 +1480,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=effort_value,
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
custom_llm_provider=self._resolved_provider,
llm_provider=self._resolved_provider,
)
if mapped_thinking is None:
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
else:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model):
if AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(effort_value)
if mapped_effort is None:
AnthropicConfig._raise_invalid_reasoning_effort(
model=model,
value=effort_value,
llm_provider=self.custom_llm_provider or "anthropic",
llm_provider=self._resolved_provider,
)
optional_params["output_config"] = {"effort": mapped_effort}
elif param == "web_search_options" and isinstance(value, dict):
@ -1813,7 +1823,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages = anthropic_messages_pt(
model=model,
messages=messages,
llm_provider=self.custom_llm_provider or "anthropic",
llm_provider=self._resolved_provider,
)
except Exception as e:
raise AnthropicError(
@ -1902,7 +1912,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_config = optional_params.get("output_config")
if not output_config or not isinstance(output_config, dict):
return
if litellm.drop_params is True and not self._model_supports_effort_param(model):
if litellm.drop_params is True and not self._model_supports_effort_param(model, self._resolved_provider):
litellm.verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
model,
@ -1916,14 +1926,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
raise litellm.exceptions.BadRequestError(
message=(f"Invalid effort value: {effort!r}. Must be one of: 'high', 'medium', 'low', 'xhigh', 'max'"),
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
llm_provider=self._resolved_provider,
)
gate_error = self._validate_effort_for_model(model, effort)
gate_error = self._validate_effort_for_model(model, effort, self._resolved_provider)
if gate_error is not None:
raise litellm.exceptions.BadRequestError(
message=gate_error,
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
llm_provider=self._resolved_provider,
)
data["output_config"] = output_config

70
litellm/llms/anthropic/common_utils.py
View file

@ -289,6 +289,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
status_code=400,
)
@staticmethod
def _strip_version_suffix(model: str) -> str:
at = model.rfind("@")
if at > 0:
return model[:at]
return model
@staticmethod
def _model_map_lookup_candidates(model: str) -> List[str]:
"""Model-map keys to try for ``model``: the id itself, the same id with a
@ -324,6 +331,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
_DATED_RELEASE_SUFFIX_RE.sub("", cand),
_DOTTED_VERSION_RE.sub(r"\1-\2", cand),
_strip_bedrock_id_suffixes(cand),
AnthropicModelInfo._strip_version_suffix(cand),
)
)
return list(dict.fromkeys((*primary, *normalized)))
@ -352,18 +360,43 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return value if isinstance(value, bool) else None
@staticmethod
def _supports_model_capability(model: str, key: str) -> bool:
"""Check a boolean capability ``key`` in the model map.
def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> Optional[bool]:
"""Resolve boolean capability ``key`` for ``model`` under the caller's provider.
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
Returns the flag when the provider-aware lookup resolves ``model`` to an
entry (or fallback rule) that sets it explicitly, and ``None`` when the
model does not resolve under that provider or the resolved entry has no
opinion on ``key``.
"""
from litellm.utils import _get_model_info_helper
try:
resolved_model, resolved_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider
)
value = _get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider).get(key)
except Exception: # noqa: BLE001 # _get_model_info_helper raises bare Exception for unmapped models
return None
return value if isinstance(value, bool) else None
@staticmethod
def _supports_model_capability(model: str, key: str, custom_llm_provider: str) -> bool:
"""Check a boolean capability ``key`` in the model map under the caller's provider.
The provider-aware lookup is authoritative when it resolves an explicit flag,
so ``key: false`` on the provider-namespaced entry wins over every fallback.
Otherwise ``_supports_factory``'s provider-level fallbacks and the raw
model-map walk remain as backstops for alias forms the lookup misses.
"""
from litellm.utils import _supports_factory
resolved = AnthropicModelInfo._get_provider_resolved_capability(model, key, custom_llm_provider)
if resolved is not None:
return resolved
try:
if _supports_factory(
model=model,
custom_llm_provider="anthropic",
custom_llm_provider=custom_llm_provider,
key=key,
):
return True
@ -372,17 +405,24 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return AnthropicModelInfo._get_model_capability(model, key) is True
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:
def _is_adaptive_thinking_model(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` uses adaptive thinking (``output_config.effort``).
The model cost map is authoritative: an explicit ``supports_adaptive_thinking``
entry, or a ``fallback_generalizations`` rule for unknown Claude models. The
version gate (>= 4.6, including provider-prefixed Bedrock/Vertex ids that map to
no exact entry) lives entirely in that declarative rule, not here.
entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations``
rule for unknown Claude models. The version gate (>= 4.6, including
provider-prefixed Bedrock/Vertex ids that map to no exact entry) lives entirely
in that declarative rule, not here.
"""
return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking")
return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider)
def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool:
def is_effort_used(
self,
optional_params: Optional[dict],
model: Optional[str] = None,
*,
custom_llm_provider: str,
) -> bool:
"""
Check if effort parameter is being used and requires a beta header.
@ -394,7 +434,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return False
# Claude 4.6+ models use output_config as a stable API feature — no beta header needed
if model and self._is_adaptive_thinking_model(model):
if model and self._is_adaptive_thinking_model(model, custom_llm_provider):
return False
# Check if reasoning_effort is provided for Claude Opus 4.5
@ -475,6 +515,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
prompt_caching_set: bool = False,
file_id_used: bool = False,
mcp_server_used: bool = False,
*,
custom_llm_provider: str,
) -> List[str]:
"""
Get list of common beta headers based on the features that are active.
@ -487,7 +529,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
betas = []
# Detect features
effort_used = self.is_effort_used(optional_params, model)
effort_used = self.is_effort_used(optional_params, model, custom_llm_provider=custom_llm_provider)
if effort_used:
betas.append(ANTHROPIC_EFFORT_BETA_HEADER) # effort-2025-11-24
@ -643,7 +685,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
tool_search_used = self.is_tool_search_used(tools=tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
input_examples_used = self.is_input_examples_used(tools=tools)
effort_used = self.is_effort_used(optional_params=optional_params, model=model)
effort_used = self.is_effort_used(optional_params=optional_params, model=model, custom_llm_provider="anthropic")
code_execution_tool_used = self.is_code_execution_tool_used(tools=tools)
container_with_skills_used = self.is_container_with_skills_used(optional_params=optional_params)
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(

161
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -3,6 +3,7 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
import httpx
from litellm.constants import (
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
@ -32,8 +33,22 @@ from ...common_utils import (
DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01"
DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING = (
"Dropping adaptive `thinking`/`output_config.effort` for model=%s: the model "
"does not support extended thinking, or max_tokens is too small to fit the "
"minimum thinking budget."
)
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
@property
def custom_llm_provider(self) -> Optional[str]:
return "anthropic"
@property
def _resolved_provider(self) -> str:
return self.custom_llm_provider or "anthropic"
def get_supported_anthropic_messages_params(self, model: str) -> list:
return [
"messages",
@ -174,7 +189,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return headers, api_base
@staticmethod
def _translate_reasoning_effort_to_anthropic(model: str, optional_params: Dict) -> None:
def _translate_reasoning_effort_to_anthropic(model: str, optional_params: Dict, custom_llm_provider: str) -> None:
"""Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
Caller-supplied ``thinking`` / ``output_config`` win over the alias.
@ -191,7 +206,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return
try:
mapped_thinking = AnthropicConfig._map_reasoning_effort(reasoning_effort=reasoning_effort, model=model)
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=reasoning_effort,
model=model,
custom_llm_provider=custom_llm_provider,
)
except _BadRequestError as e:
raise AnthropicError(message=str(e.message), status_code=400)
@ -201,7 +220,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return
optional_params.setdefault("thinking", mapped_thinking)
if AnthropicModelInfo._is_adaptive_thinking_model(model):
if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
raise AnthropicError(
@ -212,7 +231,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
),
status_code=400,
)
gate_error = AnthropicConfig._validate_effort_for_model(model, mapped_effort)
gate_error = AnthropicConfig._validate_effort_for_model(model, mapped_effort, custom_llm_provider)
if gate_error is not None:
raise AnthropicError(message=gate_error, status_code=400)
existing_output_config = optional_params.get("output_config")
@ -222,13 +241,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params["output_config"] = existing_output_config
@staticmethod
def _translate_legacy_thinking_for_adaptive_model(model: str, optional_params: Dict) -> None:
def _translate_legacy_thinking_for_adaptive_model(
model: str, optional_params: Dict, custom_llm_provider: str
) -> None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
Caller-provided ``output_config.effort`` is never overridden.
"""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if not AnthropicModelInfo._is_adaptive_thinking_model(model):
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
thinking = optional_params.get("thinking")
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
@ -236,7 +257,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
budget = int(thinking.get("budget_tokens") or 0)
if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and (
AnthropicConfig._supports_effort_level(model, "xhigh")
AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider)
):
effort = "xhigh"
elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET:
@ -253,6 +274,123 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
existing_output_config.setdefault("effort", effort)
optional_params["output_config"] = existing_output_config
@staticmethod
def _translate_adaptive_effort_for_non_adaptive_model(
model: str, optional_params: Dict, max_tokens: Optional[int], custom_llm_provider: str
) -> None:
"""Translate the 4.6+ adaptive-thinking interface (``thinking.type=adaptive``
and/or ``output_config.effort``) down to what an older Anthropic model
supports. Clients like Claude Code send this interface unconditionally, so
without translation it reaches a pre-4.6 model and Anthropic rejects it with
"This model does not support the effort parameter".
The reshape is silent, matching how the messages path already strips
unsupported ``output_config`` for older models (bedrock invoke, issue
#22797): the goal is to keep the request working, not to fail it.
``thinking.type=adaptive`` and ``output_config.effort`` are independent
capabilities. Adaptive thinking needs ``supports_adaptive_thinking`` (4.6+);
``output_config.effort`` needs ``supports_output_config``, which some
non-adaptive models (e.g. Claude Opus 4.5) advertise on its own. So the two
are handled separately:
- Adaptive-thinking models (4.6+): both are native, left untouched.
- ``supports_output_config`` but non-adaptive (Opus 4.5): keep
``output_config.effort`` (native), only drop the unsupported adaptive
``thinking`` block. When adaptive thinking is being dropped and the
effort level itself isn't supported by the model (e.g. ``xhigh``/``max``
on Opus 4.5, which only accepts low/medium/high, while ``xhigh`` is
Claude Code's default), fall through to the legacy translation below
instead of forwarding a level Anthropic would reject. Effort-only
requests are always left untouched: provider subclasses own their level
normalization (bedrock clamps ``xhigh`` to the model's ceiling after
this base transform runs).
- Thinking-capable but neither (``supports_reasoning``, e.g. Haiku/Sonnet
4.5): map effort to legacy ``thinking={type: enabled, budget_tokens}`` via
``AnthropicConfig._map_reasoning_effort``, capped below ``max_tokens``
(Anthropic requires ``max_tokens > budget_tokens``) and dropped when
``max_tokens`` can't fit even the minimum budget.
- No reasoning support: ``thinking`` is dropped.
For the last two, only the consumed ``effort`` key is removed from
``output_config``; any residual (e.g. ``format``) is left for provider
subclasses to handle.
"""
from litellm.exceptions import BadRequestError as _BadRequestError
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
return
output_config = optional_params.get("output_config")
thinking = optional_params.get("thinking")
effort = output_config.get("effort") if isinstance(output_config, dict) else None
adaptive_thinking = isinstance(thinking, dict) and thinking.get("type") == "adaptive"
if effort is None and not adaptive_thinking:
return
# Models that natively accept `output_config.effort` but are not adaptive (Claude Opus 4.5).
# Keep the native effort and only drop the adaptive `thinking` block, which these models
# reject. Effort-only requests pass through so provider subclasses (bedrock/vertex) keep
# owning level clamping; an adaptive request only stays here when its effort level is one
# the model supports, otherwise it falls through to the legacy budget translation below.
if AnthropicConfig._model_supports_effort_param(model, custom_llm_provider) and (
not adaptive_thinking
or AnthropicConfig._validate_effort_for_model(model, effort, custom_llm_provider) is None
):
if adaptive_thinking:
optional_params.pop("thinking", None)
return
supports_thinking = AnthropicModelInfo._supports_model_capability(
model, "supports_reasoning", custom_llm_provider
)
try:
legacy_thinking = (
AnthropicConfig._map_reasoning_effort(
reasoning_effort=effort or "medium",
model=model,
custom_llm_provider=custom_llm_provider,
)
if supports_thinking
else None
)
except _BadRequestError as e:
raise AnthropicError(message=str(e.message), status_code=400)
capped_thinking = (
AnthropicMessagesConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens)
if legacy_thinking is not None
else None
)
if capped_thinking is not None:
optional_params["thinking"] = capped_thinking
else:
verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING, model)
optional_params.pop("thinking", None)
if isinstance(output_config, dict) and "effort" in output_config:
residual = {k: v for k, v in output_config.items() if k != "effort"}
if residual:
optional_params["output_config"] = residual
else:
optional_params.pop("output_config", None)
@staticmethod
def _cap_thinking_budget_to_max_tokens(thinking: Dict, max_tokens: Optional[int]) -> Optional[Dict]:
"""Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic
requires ``max_tokens > budget_tokens``). Returns the (possibly capped)
thinking dict, or ``None`` when ``max_tokens`` is too small to fit even the
minimum thinking budget and thinking should be dropped."""
budget = thinking.get("budget_tokens")
if max_tokens is None or not isinstance(budget, int):
return thinking
if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS:
return None
if budget < max_tokens:
return thinking
return {**thinking, "budget_tokens": max_tokens - 1}
def transform_anthropic_messages_request(
self,
model: str,
@ -277,11 +415,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
)
self._translate_legacy_thinking_for_adaptive_model(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
)
self._translate_adaptive_effort_for_non_adaptive_model(
model=model,
optional_params=anthropic_messages_optional_request_params,
max_tokens=max_tokens,
custom_llm_provider=self._resolved_provider,
)
system_param = anthropic_messages_optional_request_params.get("system")

4
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -21,6 +21,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
and Azure endpoint format.
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "azure_ai"
def should_strip_billing_metadata(self) -> bool:
return True

9
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -423,6 +423,7 @@ class AmazonConverseConfig(BaseConfig):
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=reasoning_effort,
model=model,
custom_llm_provider="bedrock",
llm_provider="bedrock_converse",
)
if mapped_thinking is None:
@ -430,7 +431,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params.pop("output_config", None)
else:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model):
if AnthropicConfig._is_adaptive_thinking_model(model, "bedrock"):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
AnthropicConfig._raise_invalid_reasoning_effort(
@ -465,7 +466,7 @@ class AmazonConverseConfig(BaseConfig):
model=model,
llm_provider="bedrock_converse",
)
error = AnthropicConfig._validate_effort_for_model(model=model, effort=effort)
error = AnthropicConfig._validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock")
if error is not None:
raise litellm.exceptions.BadRequestError(
message=error,
@ -1279,7 +1280,7 @@ class AmazonConverseConfig(BaseConfig):
if anthropic_output_config is not None and isinstance(anthropic_output_config, dict):
if base_model.startswith("anthropic"):
if litellm.drop_params is True and not AnthropicConfig._model_supports_effort_param(model):
if litellm.drop_params is True and not AnthropicConfig._model_supports_effort_param(model, "bedrock"):
litellm.verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
model,
@ -1422,7 +1423,7 @@ class AmazonConverseConfig(BaseConfig):
if (
isinstance(output_config, dict)
and output_config.get("effort") is not None
and not AnthropicConfig._is_adaptive_thinking_model(model)
and not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock")
):
from litellm.types.llms.anthropic import (
ANTHROPIC_EFFORT_BETA_HEADER,

5
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -115,7 +115,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
keeps working. Non-adaptive models and models without a ceiling are
left untouched.
"""
if not AnthropicConfig._is_adaptive_thinking_model(model):
if not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock"):
return
effort = params.get("reasoning_effort")
if not isinstance(effort, str):
@ -228,7 +228,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
custom_llm_provider="bedrock",
key="supports_output_config",
)
or AnthropicConfig._model_supports_effort_param(model)
or AnthropicConfig._model_supports_effort_param(model, "bedrock")
):
if anthropic_request.pop("output_config", None) is not None:
verbose_logger.warning(
@ -269,6 +269,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
prompt_caching_set=False,
file_id_used=self.is_file_id_used(messages),
mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
custom_llm_provider="bedrock",
)
beta_set.update(auto_betas)

4
litellm/llms/bedrock/claude_platform/transformation.py
View file

@ -54,7 +54,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
tool_search_used=self.is_tool_search_used(tools=optional_params.get("tools")),
programmatic_tool_calling_used=self.is_programmatic_tool_calling_used(tools=optional_params.get("tools")),
input_examples_used=self.is_input_examples_used(tools=optional_params.get("tools")),
effort_used=self.is_effort_used(optional_params=optional_params, model=model),
effort_used=self.is_effort_used(
optional_params=optional_params, model=model, custom_llm_provider="anthropic"
),
user_anthropic_beta_headers=self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
),

65
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -77,6 +77,10 @@ class AmazonAnthropicClaudeMessagesConfig(
DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
def __init__(self, **kwargs):
@ -93,26 +97,48 @@ class AmazonAnthropicClaudeMessagesConfig(
return [{"type": "text", "text": value}]
return [value]
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None:
"""Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on
some Claude aliases; Anthropic Messages carries that content in the
top-level ``system`` field. Move any such entries into ``system`` before
the Invoke request is built."""
@staticmethod
def _is_system_role_message(message: Any) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None:
"""Bedrock Invoke validates ``role: "system"`` entries inside ``messages``
per model. Models carrying ``supports_mid_conversation_system`` in the
cost map (the Opus 4.8 family) only reject a leading run ("messages.0:
use the top-level 'system' parameter for the initial system prompt") and
accept mid-conversation entries (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) in place, where they
MUST stay: hoisting one mutates the ``system`` prefix and invalidates the
prompt cache for the entire message history. Older Claude models (Opus
4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position
("role 'system' is not supported on this model"), so without the flag
every system entry is hoisted into the top-level ``system`` field.
Billing-header system blocks are stripped from the top-level ``system``
field regardless of whether anything was hoisted."""
messages = anthropic_messages_request.get("messages")
if not isinstance(messages, list):
return
system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"]
if not system_role_messages:
return
anthropic_messages_request["messages"] = [
m for m in messages if not (isinstance(m, dict) and m.get("role") == "system")
]
if _supports_factory(
model=model,
custom_llm_provider="bedrock",
key="supports_mid_conversation_system",
):
leading_count = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
len(messages),
)
hoisted = messages[:leading_count]
remaining = messages[leading_count:]
else:
hoisted = [m for m in messages if self._is_system_role_message(m)]
remaining = [m for m in messages if not self._is_system_role_message(m)]
if hoisted:
anthropic_messages_request["messages"] = remaining
system_content = [
block
for source in (
anthropic_messages_request.get("system"),
*(m.get("content") for m in system_role_messages),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
]
@ -247,7 +273,7 @@ class AmazonAnthropicClaudeMessagesConfig(
Returns:
True if the model supports extended thinking on Bedrock
"""
if AnthropicModelInfo._is_adaptive_thinking_model(model):
if AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"):
return True
model_lower = model.lower()
@ -297,7 +323,7 @@ class AmazonAnthropicClaudeMessagesConfig(
if not self._supports_extended_thinking_on_bedrock(model):
return False
is_adaptive_thinking_model = AnthropicModelInfo._is_adaptive_thinking_model(model)
is_adaptive_thinking_model = AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock")
thinking = anthropic_messages_request.get("thinking")
if isinstance(thinking, dict):
@ -574,6 +600,7 @@ class AmazonAnthropicClaudeMessagesConfig(
mcp_server_used=anthropic_model_info.is_mcp_server_used(
anthropic_messages_optional_request_params.get("mcp_servers")
),
custom_llm_provider="bedrock",
)
beta_set.update(auto_betas)
@ -640,7 +667,7 @@ class AmazonAnthropicClaudeMessagesConfig(
path degrades ``xhigh`` -> ``max`` rather than 400-ing. Non-adaptive models
and models without a ceiling are left untouched.
"""
if not AnthropicModelInfo._is_adaptive_thinking_model(model):
if not AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"):
return
effort = optional_params.get("reasoning_effort")
if not isinstance(effort, str):
@ -669,7 +696,7 @@ class AmazonAnthropicClaudeMessagesConfig(
litellm_params=litellm_params,
headers=headers,
)
self._normalize_system_role_messages_for_bedrock(anthropic_messages_request)
self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model)
#########################################################
############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###
#########################################################
@ -728,7 +755,7 @@ class AmazonAnthropicClaudeMessagesConfig(
custom_llm_provider="bedrock",
key="supports_output_config",
)
or AnthropicConfig._model_supports_effort_param(model)
or AnthropicConfig._model_supports_effort_param(model, "bedrock")
):
if anthropic_messages_request.pop("output_config", None) is not None:
verbose_logger.warning(
@ -765,7 +792,7 @@ class AmazonAnthropicClaudeMessagesConfig(
if (
litellm.drop_params is True
and "output_config" in anthropic_messages_request
and not AnthropicConfig._model_supports_effort_param(model)
and not AnthropicConfig._model_supports_effort_param(model, "bedrock")
):
verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,

83
litellm/llms/dashscope/cost_calculator.py
View file

@ -7,6 +7,7 @@ Handles tiered pricing and prompt caching scenarios.
from dataclasses import dataclass
from typing import List, Optional, Tuple
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import calculate_tiered_cost
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import get_model_info
@ -42,80 +43,6 @@ def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
return TokenBreakdown(text_tokens, cached_tokens, completion_tokens, reasoning_tokens)
def _calculate_tiered_cost(
tokens: int,
tiered_pricing: List[dict],
cost_key: str,
fallback_cost_key: Optional[str] = None,
) -> float:
"""
Calculate cost for a given number of tokens based on a true tiered pricing structure.
This function iterates through sorted pricing tiers, calculates the cost for the
number of tokens that fall into each tier's range, and sums them up to get the total cost.
Args:
tokens (int): The total number of tokens to calculate the cost for.
tiered_pricing (List[dict]): A list of dictionaries, where each dictionary
represents a pricing tier.
cost_key (str): The key in the tier dictionary that holds the per-token cost
(e.g., 'input_cost_per_token').
fallback_cost_key (Optional[str], optional): A fallback key to use if the
primary `cost_key` is not found in a tier. Defaults to None.
Returns:
float: The total calculated cost for the given tokens.
Example:
>>> tiered_pricing = [
... {"range": [0, 100000], "input_cost_per_token": 0.0001},
... {"range": [100000, 500000], "input_cost_per_token": 0.00005},
... ]
Calculating cost for 150,000 tokens:
(100,000 * 0.0001) + (50,000 * 0.00005) = $12.5
"""
if not tiered_pricing or tokens <= 0:
return 0.0
total_cost = 0.0
tokens_processed = 0
sorted_tiers = sorted(tiered_pricing, key=lambda x: x.get("range", [0, 0])[0])
for tier in sorted_tiers:
if tokens_processed >= tokens:
break
tier_range = tier.get("range", [])
if len(tier_range) != 2:
continue
range_start, range_end = tier_range
if tokens <= range_start:
continue
tier_start = max(range_start, tokens_processed)
tier_end = min(range_end, tokens)
if tier_end > tier_start:
tokens_in_tier = tier_end - tier_start
cost_per_token = tier.get(cost_key) or tier.get(fallback_cost_key, 0)
total_cost += tokens_in_tier * cost_per_token
tokens_processed = tier_end
# After loop, check if any tokens remain (i.e., tokens > highest tier's end range)
# and charge them at the last tier's rate.
if tokens_processed < tokens and sorted_tiers:
last_tier = sorted_tiers[-1]
remaining_tokens = tokens - tokens_processed
cost_per_token = last_tier.get(cost_key) or last_tier.get(fallback_cost_key, 0)
total_cost += remaining_tokens * cost_per_token
return total_cost
def _calculate_prompt_cost(
breakdown: TokenBreakdown,
model_info: ModelInfo,
@ -123,12 +50,12 @@ def _calculate_prompt_cost(
) -> float:
"""Calculate total prompt cost including cached tokens."""
if tiered_pricing:
text_cost = _calculate_tiered_cost(
text_cost = calculate_tiered_cost(
tokens=breakdown.text_tokens,
tiered_pricing=tiered_pricing,
cost_key="input_cost_per_token",
)
cache_cost = _calculate_tiered_cost(
cache_cost = calculate_tiered_cost(
tokens=breakdown.cached_tokens,
tiered_pricing=tiered_pricing,
cost_key="cache_read_input_token_cost",
@ -155,12 +82,12 @@ def _calculate_completion_cost(
) -> float:
"""Calculate total completion cost including reasoning tokens."""
if tiered_pricing:
completion_cost = _calculate_tiered_cost(
completion_cost = calculate_tiered_cost(
tokens=breakdown.completion_tokens,
tiered_pricing=tiered_pricing,
cost_key="output_cost_per_token",
)
reasoning_cost = _calculate_tiered_cost(
reasoning_cost = calculate_tiered_cost(
tokens=breakdown.reasoning_tokens,
tiered_pricing=tiered_pricing,
cost_key="output_cost_per_reasoning_token",

7
litellm/llms/databricks/chat/transformation.py
View file

@ -181,6 +181,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@property
def custom_llm_provider(self) -> Optional[str]:
return "databricks"
@classmethod
def get_config(cls):
return super().get_config()
@ -372,6 +376,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=reasoning_effort_value,
model=model,
custom_llm_provider="databricks",
llm_provider="databricks",
)
if mapped_thinking is None:
@ -379,7 +384,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
optional_params.pop("output_config", None)
else:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model):
if AnthropicConfig._is_adaptive_thinking_model(model, "databricks"):
mapped_effort: Optional[str] = None
if isinstance(reasoning_effort_value, str):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort_value)

4
litellm/llms/github_copilot/messages/transformation.py
View file

@ -25,6 +25,10 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig):
super().__init__()
self.authenticator = Authenticator()
@property
def custom_llm_provider(self) -> Optional[str]:
return "github_copilot"
def handles_web_search_natively(self) -> bool:
"""
Copilot's /v1/messages endpoint does not execute ``web_search`` tools, so

6
litellm/llms/openai_like/dynamic_config.py
View file

@ -91,7 +91,7 @@ def create_config_class(provider: SimpleProviderConfig):
def get_supported_openai_params(self, model: str) -> list:
"""Get supported OpenAI params, excluding tool-related params for models
that don't support function calling."""
from litellm.utils import supports_function_calling
from litellm.utils import supports_function_calling, supports_reasoning
supported_params = super().get_supported_openai_params(model=model)
@ -113,6 +113,10 @@ def create_config_class(provider: SimpleProviderConfig):
f"function calling — removed tool-related params from supported params."
)
_supports_reasoning = supports_reasoning(model=model, custom_llm_provider=provider.slug)
if _supports_reasoning and "reasoning_effort" not in supported_params:
supported_params.append("reasoning_effort")
return supported_params
def map_openai_params(

69
litellm/llms/openai_like/messages/transformation.py
View file

@ -1,8 +1,11 @@
from typing import Any, Optional
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
from litellm.secret_managers.main import get_secret_str
DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01"
@ -67,3 +70,69 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
if base.endswith("/v1"):
base = base[: -len("/v1")]
return f"{base}/v1/messages"
class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig):
"""
Provider-level native Anthropic Messages passthrough for JSON-configured
OpenAI-compatible providers whose ``supported_endpoints`` in providers.json
includes ``"/v1/messages"``. Resolves the api key and api base from the
provider's configured env vars, then forwards the Anthropic payload
untranslated like ``OpenAILikeAnthropicMessagesConfig``.
"""
def __init__(self, provider: SimpleProviderConfig):
super().__init__()
self._provider = provider
@property
def custom_llm_provider(self) -> Optional[str]:
return self._provider.slug
def should_strip_billing_metadata(self) -> bool:
return True
def _resolve_api_key(self, api_key: Optional[str]) -> Optional[str]:
return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key
def _resolve_api_base(self, api_base: Optional[str]) -> str:
env_api_base = get_secret_str(self._provider.api_base_env) if self._provider.api_base_env else None
return api_base or env_api_base or self._provider.base_url
def validate_anthropic_messages_environment(
self,
headers: dict[str, str],
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> tuple[dict[str, str], Optional[str]]:
return super().validate_anthropic_messages_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=self._resolve_api_key(api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
return super().get_complete_url(
api_base=self._resolve_api_base(api_base),
api_key=api_key,
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)

7
litellm/llms/openai_like/providers.json
View file

@ -168,6 +168,13 @@
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"meta": {
"base_url": "https://api.meta.ai/v1",
"api_key_env": "META_API_KEY",
"api_base_env": "META_API_BASE",
"base_class": "openai_gpt",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"pinstripes": {
"base_url": "https://pinstripes.io/v1",
"api_key_env": "PINSTRIPES_API_KEY",

4
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -17,6 +17,10 @@ from ..output_params_utils import sanitize_vertex_anthropic_output_params
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
@property
def custom_llm_provider(self) -> Optional[str]:
return "vertex_ai"
def should_strip_billing_metadata(self) -> bool:
return True

2
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py
View file

@ -26,7 +26,7 @@ def _model_accepts_output_config_effort(model: str) -> bool:
"""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
return AnthropicConfig._model_supports_effort_param(model)
return AnthropicConfig._model_supports_effort_param(model, "vertex_ai")
def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None:

1
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -112,6 +112,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
prompt_caching_set=self.is_cache_control_set(messages),
file_id_used=self.is_file_id_used(messages),
mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
custom_llm_provider="vertex_ai",
)
beta_set = set(auto_betas)

643
litellm/model_prices_and_context_window_backup.json
View file

@ -1359,6 +1359,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1393,6 +1394,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1427,6 +1429,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1461,6 +1464,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1481,6 +1485,7 @@
"anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1516,6 +1521,7 @@
"global.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1551,6 +1557,7 @@
"us.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1586,6 +1593,7 @@
"eu.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1621,6 +1629,43 @@
"au.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 5.5e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true
},
"jp.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1703,6 +1748,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1737,6 +1783,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1771,6 +1818,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1805,6 +1853,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1839,6 +1888,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1873,6 +1923,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -6012,6 +6063,522 @@
"supports_vision": true,
"supports_web_search": true
},
"azure/gpt-5.6": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.6-sol": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_priority": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 2e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_priority": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 2e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token_above_272k_tokens_priority": 9e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.6-terra": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.6-luna": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_above_272k_tokens": 2e-07,
"cache_read_input_token_cost_priority": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-07,
"input_cost_per_token": 1e-06,
"input_cost_per_token_above_272k_tokens": 2e-06,
"input_cost_per_token_priority": 2e-06,
"input_cost_per_token_above_272k_tokens_priority": 4e-06,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"output_cost_per_token_above_272k_tokens": 9e-06,
"output_cost_per_token_priority": 1.2e-05,
"output_cost_per_token_above_272k_tokens_priority": 1.8e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/us/gpt-5.6": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"cache_read_input_token_cost_priority": 1.375e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"input_cost_per_token_priority": 1.375e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"output_cost_per_token_priority": 8.25e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/us/gpt-5.6-sol": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"cache_read_input_token_cost_priority": 1.375e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"input_cost_per_token_priority": 1.375e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"output_cost_per_token_priority": 8.25e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/us/gpt-5.6-terra": {
"cache_read_input_token_cost": 2.75e-07,
"cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
"cache_read_input_token_cost_priority": 6.875e-07,
"input_cost_per_token": 2.75e-06,
"input_cost_per_token_above_272k_tokens": 5.5e-06,
"input_cost_per_token_priority": 6.875e-06,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token_above_272k_tokens": 2.475e-05,
"output_cost_per_token_priority": 4.125e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/us/gpt-5.6-luna": {
"cache_read_input_token_cost": 1.1e-07,
"cache_read_input_token_cost_above_272k_tokens": 2.2e-07,
"cache_read_input_token_cost_priority": 2.75e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_above_272k_tokens": 2.2e-06,
"input_cost_per_token_priority": 2.75e-06,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"output_cost_per_token_above_272k_tokens": 9.9e-06,
"output_cost_per_token_priority": 1.65e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/eu/gpt-5.6": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"cache_read_input_token_cost_priority": 1.375e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"input_cost_per_token_priority": 1.375e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"output_cost_per_token_priority": 8.25e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/eu/gpt-5.6-sol": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"cache_read_input_token_cost_priority": 1.375e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"input_cost_per_token_priority": 1.375e-05,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"output_cost_per_token_priority": 8.25e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/eu/gpt-5.6-terra": {
"cache_read_input_token_cost": 2.75e-07,
"cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
"cache_read_input_token_cost_priority": 6.875e-07,
"input_cost_per_token": 2.75e-06,
"input_cost_per_token_above_272k_tokens": 5.5e-06,
"input_cost_per_token_priority": 6.875e-06,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token_above_272k_tokens": 2.475e-05,
"output_cost_per_token_priority": 4.125e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/eu/gpt-5.6-luna": {
"cache_read_input_token_cost": 1.1e-07,
"cache_read_input_token_cost_above_272k_tokens": 2.2e-07,
"cache_read_input_token_cost_priority": 2.75e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_above_272k_tokens": 2.2e-06,
"input_cost_per_token_priority": 2.75e-06,
"litellm_provider": "azure",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"output_cost_per_token_above_272k_tokens": 9.9e-06,
"output_cost_per_token_priority": 1.65e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure/gpt-5.5": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
@ -25501,6 +26068,42 @@
"supports_function_calling": true,
"supports_tool_choice": false
},
"meta/muse-spark-1.1": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "meta",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.25e-06,
"source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_minimal_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"meta_llama/Llama-3.3-70B-Instruct": {
"litellm_provider": "meta_llama",
"max_input_tokens": 128000,
@ -44447,20 +45050,26 @@
"fallback_generalizations": {
"rules": [
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"name": "bedrock-claude-ids",
"pattern": "anthropic\\.claude-",
"description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
"model_info": {
"supports_adaptive_thinking": true
"litellm_provider": "bedrock"
}
},
{
"name": "anthropic-claude",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"name": "anthropic-claude-ids",
"pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$",
"description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.",
"model_info": {
"litellm_provider": "anthropic"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
@ -44476,6 +45085,22 @@
"supports_pdf_input": true,
"supports_system_messages": true
}
},
{
"name": "claude-adaptive-thinking",
"pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.6 or higher, in any id shape that contains claude-<family>-: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
"model_info": {
"supports_mid_conversation_system": true
}
}
]
}

1
litellm/models/mcp_server.py
View file

@ -96,6 +96,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
available_on_public_internet: bool = True
delegate_auth_to_upstream: bool = False
oauth_passthrough: bool = False
dcr_bridge: Optional[bool] = None
is_byok: bool = False
byok_description: List[str] = Field(default_factory=list)
byok_api_key_help_url: Optional[str] = None

282
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,12 +1,23 @@
import re
from datetime import datetime, timezone
from typing import Dict, List, Optional, Set, Tuple, cast
from fastapi import HTTPException
from starlette.datastructures import Headers
from starlette.requests import Request
from starlette.types import Scope
from typing_extensions import assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (
BridgeEnvelopeAdmitted,
BridgeEnvelopeInvalid,
NotBridgeEnvelope,
envelope_keys_from_master_key,
is_bridge_envelope_shaped,
resolve_bridge_envelope,
)
from litellm.proxy._types import (
UI_TEAM_ID,
LiteLLM_TeamTable,
@ -17,12 +28,17 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.auth.user_api_key_auth import (
_run_centralized_common_checks,
user_api_key_auth,
)
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
from litellm.repositories.table_repositories import (
AgentsRepository,
MCPServerRepository,
)
from litellm.types.mcp_server.mcp_server_manager import MCPServer
def _parse_mcp_server_names_from_path(path: str, mcp_servers_header: Optional[List[str]] = None) -> Optional[List[str]]:
@ -226,6 +242,29 @@ class MCPRequestHandler:
client_ip=IPAddressUtils.get_mcp_client_ip(request),
):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
(
bridge_delegate_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
)
is not None
and oauth2_headers
and is_bridge_envelope_shaped(oauth2_headers["Authorization"])
):
# A single DCR-bridge oauth_delegate target carrying an envelope-shaped
# Authorization: open the envelope, admit under its recovered identity, and
# inject the inner upstream token for egress. A non-envelope bearer on the same
# server is NOT admitted here — it falls through to the oauth2 arm, which 401s.
validated_user_api_key_auth, mcp_server_auth_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=bridge_delegate_target,
authorization_value=oauth2_headers["Authorization"],
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=request_route,
)
elif oauth2_headers:
# Authorization on a non-delegated server: the bearer must be a real
# LiteLLM credential, so a failed validation is a genuine 401/403 and
@ -432,6 +471,247 @@ class MCPRequestHandler:
return False
return True
@staticmethod
def _single_dcr_bridge_delegate_target(
path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str]
) -> Optional[MCPServer]:
"""The one DCR-bridge ``oauth_delegate`` server this request targets, or ``None``.
Returns the server only when EXACTLY ONE target resolves and it is both
``is_oauth_delegate`` and ``is_dcr_bridge``. Fails closed (``None``) on a
multi-target request, an unresolved target, or a non-matching server, so the
envelope admission arm never fires for an aggregate scope or a server that did not
opt into the bridge. Mirrors :meth:`_target_servers_are_true_passthrough`.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
target_names = MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers)
if len(target_names) != 1:
return None
server = global_mcp_server_manager.get_mcp_server_by_name(target_names[0], client_ip=client_ip)
if server is None or not server.is_oauth_delegate or not server.is_dcr_bridge:
return None
# Egress resolves the injected per-server token only by alias / server_name; a server with
# neither cannot receive the forwarded token, so fail closed rather than admit-and-drop.
if not (server.server_name or server.alias):
return None
return server
@staticmethod
async def _admit_dcr_bridge_delegate(
server: MCPServer,
authorization_value: str,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
request: Request,
route: str,
) -> Tuple[UserAPIKeyAuth, Optional[Dict[str, Dict[str, str]]]]:
"""Open the bridge envelope and admit the caller under the live key it references.
The envelope's signature proves the user authenticated when it was minted, but
authorization is resolved fresh here rather than trusted from the envelope: the
sealed ``key_hash`` reloads the current ``UserAPIKeyAuth`` record, and the admitted
identity then runs through the standard pipeline's centralized policy gate, so the
key's present restrictions and revocation state gate the request instead of a
snapshot frozen at mint time. The inner upstream token is injected under the
server's per-server auth-header key so egress forwards it via the
``PassthroughConfig`` override; the envelope ``Authorization`` the leak-defense
strips never reaches the upstream. A new headers dict is returned rather than
mutating the input. Fails closed with a 401 on an invalid or expired envelope, or
when the referenced key is missing, blocked, or expired, its owner is
SCIM-deactivated, or the centralized policy gate rejects it (blocked team or
project, org or budget limits).
The sealed token is keyed alias-first, matching the order egress resolves
(``lookup_mcp_server_auth_in_headers`` tries ``alias`` before ``server_name``). Keying
under ``server_name`` would leave a caller-supplied ``x-mcp-{alias}-authorization`` at the
higher-priority alias slot, pairing the admitted identity with an attacker's upstream
credential; the alias-keyed injection overwrites any such caller value.
"""
from litellm.proxy.proxy_server import master_key
if not master_key:
raise HTTPException(status_code=500, detail="Server misconfigured: master_key is not set")
await MCPRequestHandler._run_pre_db_read_auth_checks(request=request, route=route)
keys = envelope_keys_from_master_key(master_key)
result = resolve_bridge_envelope(authorization_value, keys, datetime.now(timezone.utc), server.server_id)
match result:
case BridgeEnvelopeAdmitted():
header_key = server.alias or server.server_name
if header_key is None:
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
admitted = await MCPRequestHandler._reload_admitted_key(result.identity.key_hash)
await MCPRequestHandler._enforce_admitted_live_policy(admitted=admitted, request=request, route=route)
injected = {header_key: {"Authorization": result.upstream_authorization.get_secret_value()}}
new_headers = {**(mcp_server_auth_headers or {}), **injected}
return admitted, new_headers
case BridgeEnvelopeInvalid() | NotBridgeEnvelope():
raise HTTPException(status_code=401, detail="Invalid or expired credential")
case _:
assert_never(result)
@staticmethod
async def _run_pre_db_read_auth_checks(request: Request, route: str) -> None:
"""Run the proxy-wide gates ``user_api_key_auth`` applies before any key lookup: the
request-size and body-safety limits, the IP allowlist, and the ``general_settings``
route allowlist. The envelope arm bypasses ``user_api_key_auth`` (it opens the envelope
and reloads the identity itself), so without this a caller blocked by IP or hitting a
proxy route the allowlist forbids would be admitted through an envelope where the same
principal presented on the normal MCP admission path would be rejected. Runs before the
envelope crypto so a disallowed caller is turned away before any work, mirroring the
standard pipeline's pre-DB ordering. Violations raise the gate's own status (an IP or
route block is a 403, an oversized body its own limit error)."""
from litellm.proxy.auth.auth_utils import pre_db_read_auth_checks
await pre_db_read_auth_checks(
request=request,
request_data=await _read_request_body(request=request),
route=route,
)
@staticmethod
async def _reload_admitted_key(key_hash: str) -> UserAPIKeyAuth:
"""Reload the live key record an admitted envelope references and re-check live policy.
Resolving the current ``UserAPIKeyAuth`` (cache first, then DB) is what stops the
envelope from carrying frozen authority: the key's present team/org/object-permission
restrictions ride on the returned object, and a key that has since been deleted,
blocked, or expired fails closed with a 401 here rather than being admitted as an
unrestricted identity. ``get_key_object`` raises for a hash with no key row; a
blocked or expired row is rejected explicitly because ``get_key_object`` resolves a
row without applying those checks (the main ``user_api_key_auth`` pipeline enforces
them downstream, which this admission path bypasses). The owner's SCIM state is the
other builder-inline check mirrored here, so IdP offboarding revokes every envelope
minted under the user's keys rather than leaving them live until expiry. Team,
project, org, and budget state are NOT re-checked here; the caller runs the admitted
identity through ``_enforce_admitted_live_policy`` for those.
"""
from litellm.proxy.auth.auth_checks import get_key_object
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
if prisma_client is None:
raise HTTPException(status_code=500, detail="Server misconfigured: no database connection")
try:
key_object = await get_key_object(
hashed_token=key_hash,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
)
except (ProxyException, HTTPException):
raise HTTPException(status_code=401, detail="Invalid or expired credential") from None
except Exception as e: # noqa: BLE001 # a DB outage during reload is a retryable 503, not an opaque 500
MCPRequestHandler._raise_503_if_db_unavailable(e)
raise
if not MCPRequestHandler._admitted_key_is_active(key_object):
raise HTTPException(status_code=401, detail="Invalid or expired credential")
await MCPRequestHandler._reject_if_admitted_owner_scim_deactivated(key_object)
return key_object
@staticmethod
def _raise_503_if_db_unavailable(e: Exception) -> None:
"""Raise a retryable 503 when ``e`` means the auth database is unreachable, else return so the
caller applies its own fail-closed mapping. A DB outage must not masquerade as an auth failure
(401) or surface as an opaque 500; the caller retries. Mirrors ``UserAPIKeyAuthExceptionHandler``,
which renders a service-unavailable database error as 503 on the standard pipeline."""
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise HTTPException(
status_code=503,
detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.",
) from None
@staticmethod
async def _reject_if_admitted_owner_scim_deactivated(key_object: UserAPIKeyAuth) -> None:
"""Fail closed with a 401 when the key's owning user was deactivated via SCIM.
The standard pipeline enforces this inline in ``_user_api_key_auth_builder`` rather
than in ``common_checks``, so the centralized policy gate does not cover it; without
this mirror, IdP offboarding would leave the user's already-minted envelopes live
until expiry. A failed user lookup skips the gate (fail-open), matching the builder:
this is the one deliberately fail-open check in an otherwise fail-closed arm, so a
transient DB outage during this lookup admits the request rather than rejecting it,
keeping parity with how the standard pipeline treats the same lookup failure."""
if key_object.user_id is None:
return
from litellm.proxy.auth.auth_checks import get_user_object
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
try:
user_object = await get_user_object(
user_id=key_object.user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
)
except Exception as e: # noqa: BLE001 # mirror the builder's fail-open user lookup; DB errors are of any type
verbose_logger.debug(f"bridge admission: user lookup failed, skipping SCIM gate: {e}")
user_object = None
if user_object is None or not isinstance(user_object.metadata, dict):
return
if user_object.metadata.get("scim_active") is False:
raise HTTPException(status_code=401, detail="Invalid or expired credential")
@staticmethod
async def _enforce_admitted_live_policy(admitted: UserAPIKeyAuth, request: Request, route: str) -> None:
"""Run the standard pipeline's authorization checks over the admitted identity.
Mirrors the ``user_api_key_auth`` wrapper between the builder and its return: clear the
request-scoped ``budget_reservation`` on the reloaded identity, run the route gate
(``RouteChecks.should_call_route``) to enforce the identity's ``allowed_routes`` and any
disabled/admin-only route, then run ``_run_centralized_common_checks`` (the same gate every
builder path funnels through) for team-block, project-block, org, and budget. The route gate
closes a bypass: a key barred from MCP routes could otherwise mint an envelope at the token
endpoint (not itself an MCP route) and replay it against MCP, because the centralized checks
treat MCP as an inference route and never re-check ``allowed_routes``.
Failures surface with the status the standard pipeline would give them, mirroring
``UserAPIKeyAuthExceptionHandler``: a disallowed route is the route gate's own 403, an
over-budget identity is a 429, a sub-check that raised its own ``HTTPException``/
``ProxyException`` keeps that status, a transient database outage is a retryable 503, and
only a genuinely unresolvable failure (a blocked team/project raises a bare ``Exception``,
same as the standard pipeline's fallback) becomes the fail-closed 401. Collapsing every
failure to 401 was misleading: it told an over-budget but validly-authenticated caller their
credential was invalid, which on a DCR client reads as broken auth and can trigger a
pointless re-authorize loop that cannot fix a budget problem, and it masked a DB outage as an
auth error."""
from litellm.proxy.auth.route_checks import RouteChecks
admitted.budget_reservation = None
try:
RouteChecks.should_call_route(route=route, valid_token=admitted, request=request)
await _run_centralized_common_checks(
user_api_key_auth_obj=admitted,
request=request,
request_data=await _read_request_body(request=request),
route=route,
)
except (HTTPException, ProxyException):
raise
except litellm.BudgetExceededError as e:
raise HTTPException(status_code=getattr(e, "status_code", 429), detail=str(e)) from None
except Exception as e: # noqa: BLE001 # untyped gate failure: retryable 503 for a DB outage, else fail closed 401
MCPRequestHandler._raise_503_if_db_unavailable(e)
raise HTTPException(status_code=401, detail="Invalid or expired credential") from None
@staticmethod
def _admitted_key_is_active(key_object: UserAPIKeyAuth) -> bool:
"""False when the referenced key is blocked or past its expiry, so a revoked key
cannot be admitted through its still-unexpired envelope. Mirrors the active-key gate
the bridge token endpoint applies at mint time."""
if key_object.blocked is True:
return False
expires = key_object.expires
if expires is None:
return True
expiry = expires if isinstance(expires, datetime) else datetime.fromisoformat(expires)
if expiry.tzinfo is None or expiry.tzinfo.utcoffset(expiry) is None:
expiry = expiry.replace(tzinfo=timezone.utc)
return expiry >= datetime.now(timezone.utc)
@staticmethod
def _resolve_target_server_names(path: str, mcp_servers_header: Optional[List[str]]) -> List[str]:
"""

57
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -52,6 +52,7 @@ _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
"token_url",
"registration_url",
"oauth2_flow",
"dcr_bridge",
"token_exchange_endpoint",
"audience",
"subject_token_type",
@ -75,6 +76,34 @@ _TOKEN_EXCHANGE_COLUMN_FIELDS: frozenset = frozenset(
}
)
# The client-forwarded token modes share one stored-credential shape: the admin-declared upstream
# OAuth app (client_id/client_secret) plus the same authorize relay, and neither mints anything the
# gateway keeps. So a switch WITHIN this class must preserve the stored app, unlike a cross-class
# switch (e.g. an oauth2 row whose client may be DCR-minted and is not reusable elsewhere).
_CLIENT_FORWARDED_AUTH_TYPES: frozenset = frozenset({"true_passthrough", "oauth_delegate"})
# Minted token material that must never survive a client rotation on a persisted row.
_MINTED_TOKEN_CREDENTIAL_FIELDS: frozenset = frozenset({"access_token", "refresh_token", "expires_in"})
def _credential_auth_class(auth_type: Optional[str]) -> Optional[str]:
"""Collapse the client-forwarded modes to one credential class; every other auth_type is its own
class. Used so credential handling keys off whether the stored-credential shape actually changed,
not off a raw auth_type inequality that treats true_passthrough<->oauth_delegate as a full reset."""
if auth_type in _CLIENT_FORWARDED_AUTH_TYPES:
return "client_forwarded"
return auth_type
def _drop_stale_minted_on_client_rotation(merged: Dict[str, Any], new_creds: Dict[str, Any]) -> Dict[str, Any]:
"""When the update rotates the client, drop stale minted token keys it did not itself set, so an old
app's access/refresh token never rides forward under the new client. A no-op when no client key changed."""
if "client_id" not in new_creds and "client_secret" not in new_creds:
return merged
return {
key: value for key, value in merged.items() if key not in _MINTED_TOKEN_CREDENTIAL_FIELDS or key in new_creds
}
def _is_global_env_var_scope(scope: Any) -> bool:
"""``scope="user"`` entries are placeholders the user fills in; everything
@ -678,7 +707,9 @@ async def update_mcp_server(
existing = await MCPServerRepository(prisma_client).table.find_unique(where={"server_id": data.server_id})
auth_type_changed = bool(
data.auth_type and existing and existing.auth_type is not None and existing.auth_type != data.auth_type
data.auth_type
and existing
and _credential_auth_class(existing.auth_type) != _credential_auth_class(data.auth_type)
)
# Clear stale credentials when auth_type changes but no new credentials provided
@ -711,11 +742,12 @@ async def update_mcp_server(
# would wipe encrypted secrets that the UI cannot display back.
if "credentials" in data_dict and data_dict["credentials"] is not None:
if existing and existing.credentials:
# Only merge when auth_type is unchanged. Switching auth types
# (e.g. oauth2 → api_key) should replace credentials entirely
# to avoid stale secrets from the previous auth type lingering.
auth_type_unchanged = data.auth_type is None or data.auth_type == existing.auth_type
if auth_type_unchanged:
# Only merge when the credential CLASS is unchanged. A cross-class switch
# (e.g. oauth2 → api_key, or oauth2 → true_passthrough) replaces credentials
# entirely to avoid stale secrets from the previous class lingering; a switch
# within the client-forwarded class (true_passthrough ↔ oauth_delegate) keeps
# the same declared app and so must merge, not replace.
if not auth_type_changed:
existing_creds = (
json.loads(existing.credentials)
if isinstance(existing.credentials, str)
@ -726,8 +758,9 @@ async def update_mcp_server(
if isinstance(data_dict["credentials"], str)
else dict(data_dict["credentials"])
)
# New values override existing; existing keys not in update are preserved
merged = {**existing_creds, **new_creds}
# New values override existing; existing keys not in update are preserved. A client
# rotation additionally drops the previous app's stale minted token keys.
merged = _drop_stale_minted_on_client_rotation({**existing_creds, **new_creds}, new_creds)
# Migrate-on-write for legacy rows: token-exchange settings the
# old blob shape carried move to their dedicated columns (unless
# the caller set the column this update, or the row already has
@ -746,6 +779,14 @@ async def update_mcp_server(
# Add audit fields
data_dict["updated_by"] = touched_by
# prisma-python rejects a raw ``None`` for a ``Json?`` field ("value is required but not set"); the
# clear paths above use ``None`` as the merge-skip sentinel, so translate it here to ``Json(None)``,
# which writes SQL null and reads back as ``None``. Done at the edge so the merge guards stay simple.
if "credentials" in data_dict and data_dict["credentials"] is None:
from prisma import Json # noqa: PLC0415 # local import: prisma may be ungenerated at module load in some tools
data_dict["credentials"] = Json(None)
updated_mcp_server = await MCPServerRepository(prisma_client).table.update(
where={"server_id": data.server_id},
data=data_dict, # type: ignore

256
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -471,7 +471,8 @@ def _raise_if_not_oauth2(mcp_server: MCPServer) -> None:
through: the caller owns the upstream token, and this relayed flow is how a browser obtains
one against the upstream IdP (the admin UI's browser-only Authorize uses it). The minted
token is upstream-audienced and held by the caller; the gateway persists nothing for these
modes (DCR persistence is opt-in and never enabled on this path).
modes (``_persist_dcr_client_registration`` skips them unconditionally, so even the admin
Authorize path with ``persist_credentials`` enabled writes nothing to the server row).
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load
_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
@ -498,23 +499,86 @@ def _raise_unless_oauth2_discovery_server(
mcp_server_name: Optional[str],
description: str,
) -> None:
"""404 a NAMED discovery request unless it resolves to an oauth2 server.
"""404 a NAMED discovery request unless it resolves to an oauth2 or DCR-bridge server.
A named server that is unknown (or hidden from the caller) and one that exists
but is non-oauth2 both return the same 404, so the well-known discovery paths
cannot be used to enumerate non-OAuth server names. Root discovery (no name) is
unaffected, and pass-through servers are resolved by the caller before this runs.
DCR-bridge servers are admitted because they serve the gateway's own authorization
server metadata (the register, authorize, and token relays).
"""
if mcp_server_name is None:
return
if mcp_server is not None and mcp_server.auth_type == MCPAuth.oauth2:
return
if mcp_server is not None and mcp_server.is_dcr_bridge:
return
raise HTTPException(
status_code=404,
detail=f"MCP server '{mcp_server_name}' is {description}",
)
def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
"""True when a DCR-bridge server relays client registration to the upstream authorization
server instead of short-circuiting to an admin-configured OAuth client. In the relay arm the
upstream holds each client's own registration, so the authorize and token relays pass the
client's ``client_id`` and ``redirect_uri`` through verbatim and the authorization code
returns directly to the client's redirect URI without transiting the gateway. Gateway-side
redirect trust and the ``/callback`` state relay therefore only apply to the short-circuit
arm, where the upstream only knows the gateway's own callback."""
return mcp_server.is_dcr_bridge and bool(mcp_server.registration_url) and not mcp_server.client_id
def _require_s256_pkce(
code_challenge: Optional[str],
code_challenge_method: Optional[str],
) -> Tuple[str, str]:
"""DCR-bridge servers serve unauthenticated public OAuth clients, so the PKCE downgrade
paths (no challenge, or a non-S256 method; RFC 7636 defaults a missing method to ``plain``)
are rejected at the gateway instead of relying on upstream enforcement. Returns the
validated pair so callers get non-optional values."""
if code_challenge and code_challenge_method == "S256":
return code_challenge, code_challenge_method
raise HTTPException(
status_code=400,
detail=(
"This server requires PKCE: send code_challenge with "
"code_challenge_method=S256 on the authorization request"
),
)
def _redirect_to_upstream_authorize(
*,
mcp_server: MCPServer,
client_id: str,
redirect_uri: str,
state: str,
code_challenge: str,
code_challenge_method: str,
response_type: Optional[str],
scope: Optional[str],
) -> RedirectResponse:
"""The bridge relay arm's authorize redirect: every client-supplied parameter passes through
to the upstream authorize endpoint verbatim, no relay state cookie is set, and the upstream
enforces its own registered redirect binding for the client."""
scope_value = scope or (" ".join(mcp_server.scopes) if mcp_server.scopes else None)
passthrough_params = {
"client_id": client_id,
"redirect_uri": redirect_uri,
"state": state,
"response_type": response_type or "code",
"code_challenge": code_challenge,
"code_challenge_method": code_challenge_method,
**({"scope": scope_value} if scope_value else {}),
}
parsed_auth_url = urlparse(mcp_server.authorization_url or "")
merged_params = {**dict(parse_qsl(parsed_auth_url.query)), **passthrough_params}
return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params))))
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -530,6 +594,24 @@ async def authorize_with_server(
if mcp_server.authorization_url is None:
raise HTTPException(status_code=400, detail="MCP server authorization url is not set")
if mcp_server.is_dcr_bridge:
# Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated,
# now-non-optional pair to the upstream authorize; the short-circuit arm keeps
# calling this for its enforcement side effect, then falls through to the gateway
# /callback flow below, which reads the original code_challenge names.
bridge_challenge, bridge_method = _require_s256_pkce(code_challenge, code_challenge_method)
if _dcr_bridge_relays_client_registration(mcp_server):
return _redirect_to_upstream_authorize(
mcp_server=mcp_server,
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
code_challenge=bridge_challenge,
code_challenge_method=bridge_method,
response_type=response_type,
scope=scope,
)
# Trusted redirect_uri: same-origin, loopback, or ops-allowlisted.
# The URI is encrypted into the OAuth state and decoded on
# /callback to redirect the user back; a non-trusted URI would be
@ -625,11 +707,21 @@ async def exchange_token_with_server(
status_code=400,
detail="code is required for authorization_code grant",
)
bridge_token_relay = _dcr_bridge_relays_client_registration(mcp_server)
if bridge_token_relay and not redirect_uri:
raise HTTPException(
status_code=400,
detail=(
"redirect_uri is required for the authorization_code grant on this server; "
"send the same redirect_uri used on the authorization request"
),
)
proxy_base_url = get_request_base_url(request)
resolved_redirect_uri = redirect_uri if bridge_token_relay else f"{proxy_base_url}/callback"
token_data = {
"grant_type": "authorization_code",
"code": code,
"redirect_uri": f"{proxy_base_url}/callback",
"redirect_uri": resolved_redirect_uri,
**client_auth.body,
}
if code_verifier:
@ -647,7 +739,17 @@ async def exchange_token_with_server(
detail="MCP upstream token endpoint returned no response",
)
response.raise_for_status()
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
if "invalid_target" in exc.response.text:
verbose_logger.warning(
"MCP server %s: the upstream authorization server rejected the token request with "
"invalid_target; it may require RFC 8707 resource indicators, which the gateway "
"does not send yet (tracked as LIT-4339)",
mcp_server.server_id,
)
raise
token_response = response.json()
access_token = token_response["access_token"]
@ -718,6 +820,22 @@ class _PersistedDcrCredentials(BaseModel):
client_id: Optional[str] = None
client_secret: Optional[str] = None
token_endpoint_auth_method: Optional[str] = None
redirect_uris: Optional[list[str]] = None
def _redirect_uri_not_registered(credentials: _PersistedDcrCredentials, current_redirect_uri: str) -> bool:
"""Whether a persisted DCR client is positively known NOT to cover the current callback.
A DCR client is bound to the redirect_uris it was registered with; if the proxy's
resolved public origin has since changed, every authorize built for it will be
rejected by the IdP. Clients persisted before ``redirect_uris`` was recorded (and
admin-configured clients, which never get a recording) return False so they are
grandfathered rather than re-registered, because re-minting a client_id orphans
every user's refresh tokens for that server."""
recorded = credentials.redirect_uris
if not recorded:
return False
return current_redirect_uri not in recorded
def _get_persisted_dcr_credentials(credentials: object) -> Optional[_PersistedDcrCredentials]:
@ -784,11 +902,23 @@ async def _get_persisted_mcp_server_with_dcr_client_id(
return persisted_mcp_server, credentials
async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> bool:
async def _reuse_persisted_dcr_client_if_available(
mcp_server: MCPServer, current_redirect_uri: Optional[str] = None
) -> bool:
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
return False
persisted_mcp_server, credentials = persisted
if current_redirect_uri is not None and _redirect_uri_not_registered(credentials, current_redirect_uri):
verbose_logger.debug(
"register_client_with_server: not reusing persisted DCR client for server_id=%s; its registered "
"redirect_uris=%s do not include the current callback %s. The operator-facing warning for this "
"re-registration event is emitted once by _persisted_dcr_redirect_uri_is_stale.",
mcp_server.server_id,
credentials.redirect_uris,
current_redirect_uri,
)
return False
if not _apply_persisted_dcr_credentials(mcp_server, credentials):
return False
@ -807,11 +937,36 @@ async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> boo
return bool(mcp_server.client_id)
DcrRegistrationPersistenceResult = Literal["persisted", "reused", "failed"]
async def _persisted_dcr_redirect_uri_is_stale(mcp_server: MCPServer, current_redirect_uri: str) -> bool:
"""Whether the server's persisted DCR client is bound to redirect_uris that no longer
cover the current proxy callback, meaning authorize is guaranteed to fail IdP-side.
Consulted when the in-memory server already carries a hydrated client_id, which
otherwise short-circuits registration before any redirect check can run. Servers
without a persisted DCR recording (admin-configured client_id, or registered before
redirect_uris were recorded) are never reported stale."""
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
return False
_, credentials = persisted
if not _redirect_uri_not_registered(credentials, current_redirect_uri):
return False
verbose_logger.warning(
"register_client_with_server: persisted DCR client for server_id=%s is registered with redirect_uris=%s "
"which do not include the current callback %s (proxy origin changed); registering a replacement client. "
"Users previously signed in to this server will need to re-authenticate.",
mcp_server.server_id,
credentials.redirect_uris,
current_redirect_uri,
)
return True
DcrRegistrationPersistenceResult = Literal["persisted", "reused", "skipped", "failed"]
async def _persist_dcr_client_registration(
mcp_server: MCPServer, registration_response: object
mcp_server: MCPServer, registration_response: object, current_redirect_uri: str
) -> DcrRegistrationPersistenceResult:
"""Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row.
@ -821,7 +976,23 @@ async def _persist_dcr_client_registration(
full re-authorization instead of a silent refresh. Mirrors the ``encrypt_credentials``
write that ``client_credentials`` and token exchange already use. Failures are logged,
never raised: registration still returns to the caller even when persistence fails.
The client-forwarded token modes (``true_passthrough`` / ``oauth_delegate``) are skipped
unconditionally: the caller holds the upstream token and the gateway must hold no OAuth
client identity for these servers. Persisting here would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing, making a
fresh pass-through server read as gateway-authorized.
``redirect_uris`` records what the client is bound to so a later origin change can be
detected as a positive mismatch and trigger re-registration instead of stranding the
server on IdP-side redirect_uri rejections. ``client_secret`` and
``token_endpoint_auth_method`` are written explicitly (None when absent) because
``update_mcp_server`` merges credential blobs: a re-registered public client must not
inherit the previous client's secret or auth method.
"""
if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
return "skipped"
try:
registration = _DcrClientRegistration.model_validate(registration_response)
except ValidationError as exc:
@ -833,17 +1004,16 @@ async def _persist_dcr_client_registration(
)
return "failed"
if await _reuse_persisted_dcr_client_if_available(mcp_server):
if await _reuse_persisted_dcr_client_if_available(mcp_server, current_redirect_uri=current_redirect_uri):
return "reused"
credentials: MCPCredentials = {
"client_id": registration.client_id,
**({"client_secret": registration.client_secret} if registration.client_secret is not None else {}),
**(
{"token_endpoint_auth_method": "client_secret_basic"}
if registration.token_endpoint_auth_method == "client_secret_basic"
else {}
"client_secret": registration.client_secret,
"token_endpoint_auth_method": (
"client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
),
"redirect_uris": [current_redirect_uri],
}
from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415
@ -878,6 +1048,21 @@ async def _persist_dcr_client_registration(
return "failed"
_MAX_UPSTREAM_ERROR_CHARS = 500
def _safe_upstream_error_detail(response: httpx.Response) -> str:
"""Bounded plaintext summary of an upstream registration failure for the client.
RFC 7591 error bodies are small JSON objects (``error`` / ``error_description``); relaying the
text lets the client read the real reason instead of a bare 500, and the length bound keeps a
hostile or oversized upstream body from bloating the gateway response."""
body = response.text
if not body:
return response.reason_phrase or "upstream registration failed"
return body[:_MAX_UPSTREAM_ERROR_CHARS]
async def register_client_with_server(
request: Request,
mcp_server: MCPServer,
@ -887,19 +1072,28 @@ async def register_client_with_server(
token_endpoint_auth_method: Optional[str],
fallback_client_id: Optional[str] = None,
persist_credentials: bool = False,
client_redirect_uris: Optional[list] = None,
):
_raise_if_not_oauth2(mcp_server)
request_base_url = get_request_base_url(request)
current_redirect_uri = f"{request_base_url}/callback"
dummy_return = {
"client_id": fallback_client_id or mcp_server.server_name,
"client_secret": "dummy",
"redirect_uris": [f"{request_base_url}/callback"],
"redirect_uris": [current_redirect_uri],
}
if mcp_server.client_id:
if mcp_server.client_id and not (
persist_credentials
and mcp_server.registration_url
and await _persisted_dcr_redirect_uri_is_stale(mcp_server, current_redirect_uri)
):
return dummy_return
if await _reuse_persisted_dcr_client_if_available(mcp_server):
if await _reuse_persisted_dcr_client_if_available(
mcp_server,
current_redirect_uri=current_redirect_uri if persist_credentials else None,
):
return dummy_return
if mcp_server.authorization_url is None:
@ -908,12 +1102,19 @@ async def register_client_with_server(
if mcp_server.registration_url is None:
return dummy_return
bridge_relay = _dcr_bridge_relays_client_registration(mcp_server)
if bridge_relay and not client_redirect_uris:
raise HTTPException(
status_code=400,
detail="redirect_uris is required to register a client with this server",
)
register_data = {
"client_name": client_name,
"redirect_uris": [f"{request_base_url}/callback"],
"grant_types": grant_types or [],
"response_types": response_types or [],
"token_endpoint_auth_method": token_endpoint_auth_method or "",
"redirect_uris": client_redirect_uris if bridge_relay else [current_redirect_uri],
"grant_types": grant_types or (["authorization_code", "refresh_token"] if bridge_relay else []),
"response_types": response_types or (["code"] if bridge_relay else []),
"token_endpoint_auth_method": token_endpoint_auth_method or ("none" if bridge_relay else ""),
}
headers = {
"Content-Type": "application/json",
@ -931,12 +1132,14 @@ async def register_client_with_server(
status_code=502,
detail="MCP upstream registration endpoint returned no response",
)
if bridge_relay and response.status_code >= 400:
raise HTTPException(status_code=response.status_code, detail=_safe_upstream_error_detail(response))
response.raise_for_status()
token_response = response.json()
if persist_credentials:
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response)
if persist_credentials and not bridge_relay:
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response, current_redirect_uri)
if persistence_result == "reused":
return dummy_return
@ -1359,6 +1562,13 @@ async def _build_oauth_protected_resource_response(
else:
resource_url = f"{request_base_url}/mcp"
if mcp_server is not None and mcp_server_name and mcp_server.is_dcr_bridge:
return {
"authorization_servers": [f"{request_base_url}/{mcp_server_name}"],
"resource": resource_url,
"scopes_supported": (mcp_server.scopes if mcp_server.scopes else []),
}
# Pass-through branch: proxy the upstream's own metadata so discovery
# directs the client at the real IdP (Okta, Keycloak, …) instead of us.
if mcp_server is not None and (
@ -1688,6 +1898,7 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
response_types=data.get("response_types", []),
token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""),
fallback_client_id=resolved.server_name or resolved.name,
client_redirect_uris=data.get("redirect_uris"),
)
return dummy_return
@ -1702,4 +1913,5 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
response_types=data.get("response_types", []),
token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""),
fallback_client_id=mcp_server_name,
client_redirect_uris=data.get("redirect_uris"),
)

75
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1044,6 +1044,24 @@ class MCPServerManager:
"browser sign-in, including delegate_auth_to_upstream)."
)
config_dcr_bridge = server_config.get("dcr_bridge", None)
if config_dcr_bridge is not None and not isinstance(config_dcr_bridge, bool):
raise ValueError(
f"Invalid config for MCP server '{server_name or server_id}': dcr_bridge "
f"must be a boolean (got {config_dcr_bridge!r})."
)
if config_dcr_bridge and auth_type not in (
MCPAuth.true_passthrough,
MCPAuth.oauth_delegate,
):
raise ValueError(
f"Invalid config for MCP server '{server_name or server_id}': dcr_bridge is only "
f"supported for auth_type true_passthrough or oauth_delegate (got {auth_type!r}). "
"The DCR bridge serves gateway-hosted OAuth discovery for the client-forwarded "
"token modes; interactive oauth2 servers already run the gateway "
"authorization-code flow."
)
new_server = MCPServer(
server_id=server_id,
name=name_for_prefix,
@ -1079,6 +1097,7 @@ class MCPServerManager:
available_on_public_internet=bool(server_config.get("available_on_public_internet", True)),
delegate_auth_to_upstream=bool(server_config.get("delegate_auth_to_upstream", False)),
oauth_passthrough=bool(server_config.get("oauth_passthrough", False)),
dcr_bridge=config_dcr_bridge,
# AWS SigV4 fields
aws_access_key_id=server_config.get("aws_access_key_id", None),
aws_secret_access_key=server_config.get("aws_secret_access_key", None),
@ -1454,6 +1473,7 @@ class MCPServerManager:
available_on_public_internet=bool(getattr(mcp_server, "available_on_public_internet", True)),
delegate_auth_to_upstream=bool(getattr(mcp_server, "delegate_auth_to_upstream", False)),
oauth_passthrough=bool(getattr(mcp_server, "oauth_passthrough", False)),
dcr_bridge=getattr(mcp_server, "dcr_bridge", None),
created_at=getattr(mcp_server, "created_at", None),
updated_at=getattr(mcp_server, "updated_at", None),
tool_name_to_display_name=_deserialize_json_dict(getattr(mcp_server, "tool_name_to_display_name", None)),
@ -2610,10 +2630,16 @@ class MCPServerManager:
return prefixed_or_original_tools
except MCPUpstreamAuthError:
except MCPUpstreamAuthError as upstream_auth_error:
# Pass-through 401 must surface to single-server routes so the
# client triggers the upstream OAuth flow. The multi-server
# aggregator catches this explicitly to keep absorbing.
if server.is_dcr_bridge and upstream_auth_error.www_authenticate is not None:
raise MCPUpstreamAuthError(
status_code=upstream_auth_error.status_code,
www_authenticate=None,
server_name=upstream_auth_error.server_name,
) from upstream_auth_error
raise
except HTTPException as e:
# A v2 resolver auth challenge (token_exchange's RFC 9728 401, authorization_code's
@ -2623,9 +2649,10 @@ class MCPServerManager:
# Non-auth HTTP errors stay absorbed so one misconfigured server can't blank the listing.
if e.status_code in (401, 403):
headers = e.headers or {}
challenge_header = headers.get("WWW-Authenticate") or headers.get("www-authenticate")
raise MCPUpstreamAuthError(
status_code=e.status_code,
www_authenticate=headers.get("WWW-Authenticate") or headers.get("www-authenticate"),
www_authenticate=None if server.is_dcr_bridge else challenge_header,
server_name=server.name,
) from e
verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}")
@ -3966,10 +3993,50 @@ class MCPServerManager:
tool_call_coro = _obo_call_tool_limited()
else:
# Scoped to the two client-forwarded token modes this stack introduced; legacy
# oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not
# added here even though the list path still relays for it.
relays_upstream_auth = mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate
server_label = mcp_server.name or mcp_server.server_name or mcp_server.alias or ""
async def _call_tool_via_client(client, params):
async with self._limit_outbound_concurrency(mcp_server):
return await client.call_tool(params, host_progress_callback=host_progress_callback)
if not relays_upstream_auth:
return await client.call_tool(params, host_progress_callback=host_progress_callback)
# The client-forwarded modes carry the caller's own upstream token, so an upstream
# 401 (expired/invalid token) is the caller's to resolve: relay it as
# MCPUpstreamAuthError so single-server REST callers turn it into a 401 +
# WWW-Authenticate and re-run the upstream OAuth flow. Only 401 is a re-auth signal
# (mirrors the list path and MCPUpstreamAuthError's contract); a 403 is a genuine
# authorization failure that re-auth won't fix, so it takes the non-auth branch and
# stays a visible warning. raise_on_error only re-raises transport failures
# (tool-level isError results are still returned normally); a non-auth failure keeps
# the same isError degradation the default path produces.
try:
return await client.call_tool(
params, host_progress_callback=host_progress_callback, raise_on_error=True
)
except Exception as e:
auth_info = _extract_upstream_auth_failure(e)
if auth_info is None or auth_info[0] != 401:
# A genuine (non-auth or 403-forbidden) upstream/transport failure.
# raise_on_error demoted the client-layer log to debug, so surface it here at
# warning level to keep the outage visible; the caller still gets the graceful
# isError result the default masking path would have produced. Log the
# exception type only, never str(e), which for an httpx error embeds the
# upstream URL (a credential can hide in it).
verbose_logger.warning(
"Pass-through MCP tool call failed against %s (non-auth, %s)",
server_label,
type(e).__name__,
)
return client.error_tool_result(e)
_, www_authenticate = auth_info
raise MCPUpstreamAuthError(
status_code=401,
www_authenticate=www_authenticate,
server_name=server_label,
) from e
tool_call_coro = _call_tool_via_client(client, call_tool_params)
@ -4800,6 +4867,7 @@ class MCPServerManager:
token_url=server.token_url,
registration_url=server.registration_url,
oauth2_flow=server.oauth2_flow,
dcr_bridge=server.dcr_bridge,
token_exchange_endpoint=server.token_exchange_endpoint,
audience=server.audience,
subject_token_type=server.subject_token_type,
@ -4916,6 +4984,7 @@ class MCPServerManager:
available_on_public_internet=server.available_on_public_internet,
delegate_auth_to_upstream=server.delegate_auth_to_upstream,
oauth_passthrough=getattr(server, "oauth_passthrough", False),
dcr_bridge=server.dcr_bridge,
is_byok=server.is_byok,
byok_description=server.byok_description,
byok_api_key_help_url=server.byok_api_key_help_url,

2
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -129,7 +129,7 @@ def get_request_base_url(request: Request) -> str:
if x_forwarded_port and ":" not in netloc:
netloc = f"{netloc}:{x_forwarded_port}"
return urlunparse((scheme, netloc, parsed.path, "", "", ""))
return urlunparse((scheme, _strip_default_port(scheme, netloc), parsed.path, "", "", ""))
def validate_loopback_redirect_uri(redirect_uri: str) -> None:

169
litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py Normal file
View file

@ -0,0 +1,169 @@
"""Producer and consumer helpers for the DCR-bridge ``oauth_delegate`` envelope.
A DCR-bridge ``oauth_delegate`` client presents ONE bearer that is a litellm-signed
envelope (see :mod:`.envelope`) carrying both a litellm identity and the upstream OAuth
token. The gateway token endpoint mints it (producer) at OAuth issuance, and at the MCP
admission edge the gateway derives the envelope keys from the proxy ``master_key``, opens
it, admits the request under the recovered identity, and forwards the inner upstream token
to the upstream MCP server (consumer). This module is the pure surface for both sides; the
token-endpoint and admission wiring live in their respective call sites.
"""
import hashlib
from datetime import datetime
from functools import lru_cache
from typing import Literal, TypeAlias
from pydantic import BaseModel, ConfigDict, SecretStr
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (
EnvelopeIdentity,
EnvelopeKeys,
EnvelopeMintError,
OpenedEnvelope,
SealedEnvelope,
UpstreamTokenGrant,
is_envelope,
mint_envelope,
open_envelope,
)
_SIGNING_KEY_DOMAIN = b"litellm-mcp-bridge:envelope-signing:"
_ENCRYPTION_KEY_DOMAIN = b"litellm-mcp-bridge:envelope-encryption:"
# scrypt work factors (RFC 7914). n=2**15 with r=8/p=1 costs ~50ms and ~32MB per derivation, which
# makes offline guessing of a candidate master key memory-hard rather than a bare hash comparison.
_SCRYPT_N = 2**15
_SCRYPT_R = 8
_SCRYPT_P = 1
# scrypt's working-set is ~128 * N * r * p bytes; cap at twice that so the maxmem ceiling scales
# with every work factor and a future p or r bump does not trip "memory limit exceeded".
_SCRYPT_MAXMEM = 128 * _SCRYPT_N * _SCRYPT_R * _SCRYPT_P * 2
_DERIVED_KEY_BYTES = 32
@lru_cache(maxsize=8)
def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys:
"""Derive the envelope signing and encryption keys from the proxy master key.
A memory-hard scrypt KDF (RFC 7914) over two distinct domain-label salts yields two
independent 256-bit subkeys from the one secret, so the producer (mint) and consumer
(open) agree on keys without persisting any. scrypt is used rather than a bare hash or
HMAC so that a captured envelope is not a cheap offline oracle for the master key: each
candidate guess costs a full memory-hard derivation, which is what protects a deployment
whose master key is weaker than it should be. The result is cached (the master key is
fixed for a process), so the KDF runs once per key and adds nothing to the per-request
admission path. The derivation is deterministic; rotating ``master_key`` invalidates
every outstanding envelope, which is the intended behavior for a signing-key change.
"""
signing = hashlib.scrypt(
master_key.encode(),
salt=_SIGNING_KEY_DOMAIN,
n=_SCRYPT_N,
r=_SCRYPT_R,
p=_SCRYPT_P,
maxmem=_SCRYPT_MAXMEM,
dklen=_DERIVED_KEY_BYTES,
).hex()
encryption = hashlib.scrypt(
master_key.encode(),
salt=_ENCRYPTION_KEY_DOMAIN,
n=_SCRYPT_N,
r=_SCRYPT_R,
p=_SCRYPT_P,
maxmem=_SCRYPT_MAXMEM,
dklen=_DERIVED_KEY_BYTES,
).hex()
return EnvelopeKeys(signing_key=SecretStr(signing), encryption_key=SecretStr(encryption))
def build_bridge_token_response(
identity: EnvelopeIdentity,
grant: UpstreamTokenGrant,
keys: EnvelopeKeys,
now: datetime,
) -> SealedEnvelope | EnvelopeMintError:
"""Seal ``grant`` for ``identity`` into the client-held bearer the token endpoint returns.
The producer mirror of :func:`resolve_bridge_envelope`: a thin, pure wrapper over
:func:`mint_envelope` that returns the sealed envelope, or the mint error as a value
(an oversized grant) for the caller to map onto an OAuth error response.
"""
return mint_envelope(identity, grant, keys, now)
class NotBridgeEnvelope(BaseModel):
"""The bearer is not an envelope; admission continues on its normal path."""
model_config = ConfigDict(frozen=True)
tag: Literal["not_bridge_envelope"] = "not_bridge_envelope"
class BridgeEnvelopeAdmitted(BaseModel):
"""A valid envelope: the identity to admit under and the full upstream ``Authorization``
value (``token_type access_token``) to forward to the upstream MCP server."""
model_config = ConfigDict(frozen=True)
tag: Literal["admitted"] = "admitted"
identity: EnvelopeIdentity
upstream_authorization: SecretStr
class BridgeEnvelopeInvalid(BaseModel):
"""The bearer is envelope-shaped but did not open (expired, tampered, wrong key);
admission must fail closed rather than fall through to normal validation."""
model_config = ConfigDict(frozen=True)
tag: Literal["invalid"] = "invalid"
BridgeEnvelopeResult: TypeAlias = NotBridgeEnvelope | BridgeEnvelopeAdmitted | BridgeEnvelopeInvalid
def _strip_bearer(value: str) -> str:
parts = value.split(None, 1)
if len(parts) == 2 and parts[0].lower() == "bearer":
return parts[1]
return value
def is_bridge_envelope_shaped(authorization_value: str) -> bool:
"""Cheap, keyless test that an ``Authorization`` value carries an envelope (optional
``Bearer`` scheme stripped). The admission edge engages the bridge arm only for an
envelope, so a plain upstream bearer falls through to normal oauth2 admission."""
return is_envelope(_strip_bearer(authorization_value))
def resolve_bridge_envelope(
authorization_value: str,
keys: EnvelopeKeys,
now: datetime,
expected_server_id: str,
) -> BridgeEnvelopeResult:
"""Classify an ``Authorization`` value presented to a bridge ``oauth_delegate`` server.
Strips an optional ``Bearer`` scheme, then returns ``NotBridgeEnvelope`` for a
non-envelope bearer (normal admission continues), ``BridgeEnvelopeAdmitted`` with the
recovered identity and the upstream ``Authorization`` value to forward for a valid
envelope, and ``BridgeEnvelopeInvalid`` for an envelope-shaped bearer that will not
open. Never raises: it is total over hostile input via :func:`open_envelope`.
``expected_server_id`` is the ``server_id`` of the MCP server the request targets; an
opened envelope whose sealed ``server_id`` does not match is rejected as
``BridgeEnvelopeInvalid``. Binding here (rather than leaving it to the caller) prevents
replaying an envelope minted for one server against another, which would forward the
first server's upstream credential across a server boundary. ``server_id`` is not a
secret (the caller targets that server), so a plain equality check is sufficient and,
unlike ``hmac.compare_digest`` on ``str``, does not raise on a non-ASCII server_id.
"""
candidate = _strip_bearer(authorization_value)
if not is_envelope(candidate):
return NotBridgeEnvelope()
opened = open_envelope(candidate, keys, now)
if not isinstance(opened, OpenedEnvelope):
return BridgeEnvelopeInvalid()
if opened.identity.server_id != expected_server_id:
return BridgeEnvelopeInvalid()
grant = opened.grant
upstream_authorization = f"{grant.token_type} {grant.access_token.get_secret_value()}"
return BridgeEnvelopeAdmitted(identity=opened.identity, upstream_authorization=SecretStr(upstream_authorization))

366
litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py Normal file
View file

@ -0,0 +1,366 @@
"""Client-held sealed envelope for the oauth_delegate DCR bridge.
A DCR-bridge client holds ONE bearer that must carry BOTH a litellm identity and the
upstream OAuth grant, with zero server-side storage. The gateway token endpoint mints a
litellm-signed envelope (:func:`mint_envelope`); the MCP edge validates it, recovers the
identity claims and the inner upstream grant (:func:`open_envelope`), and forwards the
inner access token upstream. This module is pure and unwired: it imports nothing from
endpoint or edge code, reads no proxy globals, and takes all key material and the clock
as explicit parameters.
Wire shape: ``llm_env_`` + an HS256 JWT (same signing approach as the BYOK session
bearer in ``byok_oauth_endpoints.py``). Registered claims are ``iss``/``iat``/``exp``;
custom claims are ``server_id``, ``key_hash``, and ``grant``, where ``grant`` is the
upstream token grant serialized to JSON, encrypted with the repo's symmetric
encryption helpers (``encrypt_value``/``decrypt_value`` from
``encrypt_decrypt_utils`` — the same family ``encrypt_value_helper`` applies to
persisted DCR credentials), and base64url-encoded, so the inner token never appears
in plaintext anywhere in the envelope.
Failures are values: :func:`open_envelope` returns one of the frozen
``EnvelopeOpenError`` variants (discriminated on ``tag``) for invalid, expired,
tampered, or undecryptable input, and :func:`mint_envelope` returns
``EnvelopeTooLarge`` for oversized grants. Error values carry tags and sizes only,
never token material.
The pydantic input models reject programmer errors at construction (e.g. a
non-positive ``expires_in`` or an empty required field). :func:`open_envelope` is
additionally total over hostile, attacker-controlled input: it never raises, only
returns an ``EnvelopeOpenError``. :func:`mint_envelope` operates on a
gateway-supplied grant (an upstream IdP's UTF-8 JSON token response), so it does not
defend against non-UTF-8 field content that cannot survive JSON parsing; its only
value-typed failure is ``EnvelopeTooLarge``.
"""
from __future__ import annotations
import base64
from datetime import datetime, timedelta
from typing import Literal, TypeAlias
import jwt
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value
ENVELOPE_PREFIX = "llm_env_"
"""Marker prefix on every serialized envelope so the edge can cheaply tell an envelope
from a raw upstream token before doing any cryptography."""
ENVELOPE_ISSUER = "litellm-mcp-bridge"
"""``iss`` claim stamped into every envelope and required back on open."""
MAX_ENVELOPE_TTL_SECONDS = 3600
"""Hard ceiling on envelope lifetime. ``exp`` is ``min(upstream expires_in, this cap)``
(the cap alone when the upstream omits ``expires_in``), matching the 1h lifetime of the
BYOK session bearer this module's signing approach is borrowed from: a client-held
credential should never outlive a bounded window even when the upstream token does."""
MAX_ENVELOPE_BYTES = 12288
"""Size cap on the final serialized envelope (prefix + JWT, in bytes). Upstream JWTs
commonly run 2-4KB; base64 plus encryption overhead roughly doubles that inside the
envelope, and common proxy/server header limits sit around 16KB total. 12288 leaves
comfortable headroom for a large upstream token while keeping the envelope safely
transmittable as a single Authorization header. Oversized grants are rejected with a
typed error, never truncated."""
_ENVELOPE_JWT_ALGORITHM = "HS256"
class EnvelopeIdentity(BaseModel):
"""The litellm identity the envelope binds the inner grant to.
``key_hash`` is the hashed litellm key that authorized the mint, never a raw
credential (and the edge rejects a bare hash presented as a bearer). Admission
reloads the live key record by it, so the key's current team/org/object-permission
restrictions and its revocation state are enforced at use time rather than frozen at
mint time. ``server_id`` binds the envelope to one MCP server so it cannot be replayed
across a server boundary.
"""
model_config = ConfigDict(frozen=True)
server_id: str = Field(min_length=1)
key_hash: str = Field(min_length=1)
class UpstreamTokenGrant(BaseModel):
"""The upstream OAuth token response fields sealed inside the envelope.
``expires_in`` must be positive when present; a non-positive value is a programmer
error rejected at construction. Token fields are ``SecretStr`` so reprs never leak
them.
"""
model_config = ConfigDict(frozen=True)
access_token: SecretStr = Field(min_length=1)
token_type: str = Field(min_length=1)
refresh_token: SecretStr | None = None
scope: str | None = None
expires_in: int | None = Field(default=None, gt=0)
class EnvelopeKeys(BaseModel):
"""Injected key material: the HS256 signing key and the symmetric encryption key.
``signing_key`` must be at least 32 bytes: HS256's HMAC-SHA256 has a 256-bit
security level, RFC 7518 requires a key of at least that size, and a shorter key
makes PyJWT emit ``InsecureKeyLengthWarning``.
"""
model_config = ConfigDict(frozen=True)
signing_key: SecretStr = Field(min_length=32)
encryption_key: SecretStr = Field(min_length=1)
class SealedEnvelope(BaseModel):
"""A minted envelope: the client-held bearer value and when it expires."""
model_config = ConfigDict(frozen=True)
token: SecretStr
expires_at: datetime
class OpenedEnvelope(BaseModel):
"""A validated envelope: the identity it was minted for and the recovered grant."""
model_config = ConfigDict(frozen=True)
identity: EnvelopeIdentity
grant: UpstreamTokenGrant
class EnvelopeTooLarge(BaseModel):
"""The serialized envelope exceeded ``MAX_ENVELOPE_BYTES``; carries sizes only."""
model_config = ConfigDict(frozen=True)
tag: Literal["envelope_too_large"] = "envelope_too_large"
size_bytes: int
max_bytes: int
EnvelopeMintError: TypeAlias = EnvelopeTooLarge
class NotAnEnvelope(BaseModel):
"""The candidate does not carry the envelope prefix."""
model_config = ConfigDict(frozen=True)
tag: Literal["not_an_envelope"] = "not_an_envelope"
class BadSignature(BaseModel):
"""The JWT signature does not verify under the provided signing key."""
model_config = ConfigDict(frozen=True)
tag: Literal["bad_signature"] = "bad_signature"
class Expired(BaseModel):
"""The envelope's ``exp`` is not in the future relative to the provided ``now``."""
model_config = ConfigDict(frozen=True)
tag: Literal["expired"] = "expired"
class MalformedPayload(BaseModel):
"""The token is not a well-formed envelope: undecodable JWT, wrong issuer, missing
or mistyped claims, or a decrypted grant that fails validation."""
model_config = ConfigDict(frozen=True)
tag: Literal["malformed_payload"] = "malformed_payload"
class DecryptFailed(BaseModel):
"""The signed ``grant`` blob could not be decrypted under the provided key."""
model_config = ConfigDict(frozen=True)
tag: Literal["decrypt_failed"] = "decrypt_failed"
EnvelopeOpenError: TypeAlias = NotAnEnvelope | BadSignature | Expired | MalformedPayload | DecryptFailed
class _EnvelopeClaims(BaseModel):
"""Decoded-claims boundary that pins the exact shape :func:`mint_envelope` emits.
``server_id``/``key_hash`` mirror the ``min_length`` constraints of
:class:`EnvelopeIdentity` so any claim set that validates here also constructs an
identity, keeping :func:`open_envelope` raise-free: a correctly signed JWT with an
empty identity claim fails here and maps to ``MalformedPayload``.
``strict`` rejects coerced types (``exp: "123"``, ``exp: 123.0``) rather than opening
on them, and ``extra="forbid"`` rejects any claim the gateway never mints (a hostile
``nbf``/``aud``/... rides along on a re-signed token). Since PyJWT's own ``iat``/
``nbf``/``exp`` validators are disabled at decode (they raise on hostile claim types
and, for ``iat``/``nbf``, compare against the wall clock rather than the injected
``now``), this model is the sole, total type gate for every registered claim.
"""
model_config = ConfigDict(frozen=True, strict=True, extra="forbid")
iss: str
iat: int
exp: int
server_id: str = Field(min_length=1)
key_hash: str = Field(min_length=1)
grant: str = Field(min_length=1)
class _GrantWire(BaseModel):
model_config = ConfigDict(frozen=True)
access_token: str
token_type: str
refresh_token: str | None = None
scope: str | None = None
expires_in: int | None = None
def is_envelope(candidate: str) -> bool:
"""Cheap prefix check so the edge can route envelopes vs raw tokens without crypto."""
return candidate.startswith(ENVELOPE_PREFIX)
def mint_envelope(
identity: EnvelopeIdentity,
grant: UpstreamTokenGrant,
keys: EnvelopeKeys,
now: datetime,
) -> SealedEnvelope | EnvelopeMintError:
"""Seal ``grant`` for ``identity`` into a client-held envelope.
``exp`` is ``min(grant.expires_in, MAX_ENVELOPE_TTL_SECONDS)`` seconds from ``now``
(the cap alone when ``expires_in`` is absent). Returns ``EnvelopeTooLarge`` when the
serialized envelope exceeds ``MAX_ENVELOPE_BYTES``.
"""
expires_at = now + timedelta(seconds=_envelope_ttl_seconds(grant.expires_in))
claims = _EnvelopeClaims(
iss=ENVELOPE_ISSUER,
iat=int(now.timestamp()),
exp=int(expires_at.timestamp()),
server_id=identity.server_id,
key_hash=identity.key_hash,
grant=_encrypt_grant_blob(_grant_plaintext(grant), keys.encryption_key),
)
token = ENVELOPE_PREFIX + jwt.encode(
claims.model_dump(),
keys.signing_key.get_secret_value(),
algorithm=_ENVELOPE_JWT_ALGORITHM,
)
size_bytes = len(token.encode("utf-8"))
if size_bytes > MAX_ENVELOPE_BYTES:
return EnvelopeTooLarge(size_bytes=size_bytes, max_bytes=MAX_ENVELOPE_BYTES)
return SealedEnvelope(token=SecretStr(token), expires_at=expires_at)
def open_envelope(
candidate: str,
keys: EnvelopeKeys,
now: datetime,
) -> OpenedEnvelope | EnvelopeOpenError:
"""Validate ``candidate`` and recover the identity and inner grant.
Never raises for bad input: every invalid, expired, tampered, or undecryptable
candidate maps to a distinct ``EnvelopeOpenError`` variant. The recovered
``grant.expires_in`` is the value the upstream reported at mint time and is not
re-derived, so it is stale by up to the envelope's lifetime; callers that need a
live remaining lifetime should use ``now`` against the upstream, not this field.
"""
if not is_envelope(candidate):
return NotAnEnvelope()
# UTF-8 byte length is never below character length, so a character count already over the
# cap rejects an oversize candidate in O(1) without encoding it; the exact byte check then
# runs only on candidates already bounded to <= MAX_ENVELOPE_BYTES characters.
if len(candidate) > MAX_ENVELOPE_BYTES:
return MalformedPayload()
if len(candidate.encode("utf-8", "surrogatepass")) > MAX_ENVELOPE_BYTES:
return MalformedPayload()
claims = _decode_claims(candidate.removeprefix(ENVELOPE_PREFIX), keys.signing_key)
if not isinstance(claims, _EnvelopeClaims):
return claims
if now.timestamp() >= claims.exp:
return Expired()
grant = _decrypt_grant(claims.grant, keys.encryption_key)
if not isinstance(grant, UpstreamTokenGrant):
return grant
return OpenedEnvelope(
identity=EnvelopeIdentity(server_id=claims.server_id, key_hash=claims.key_hash),
grant=grant,
)
def _envelope_ttl_seconds(upstream_expires_in: int | None) -> int:
if upstream_expires_in is None:
return MAX_ENVELOPE_TTL_SECONDS
return min(upstream_expires_in, MAX_ENVELOPE_TTL_SECONDS)
def _grant_plaintext(grant: UpstreamTokenGrant) -> str:
wire = _GrantWire(
access_token=grant.access_token.get_secret_value(),
token_type=grant.token_type,
refresh_token=None if grant.refresh_token is None else grant.refresh_token.get_secret_value(),
scope=grant.scope,
expires_in=grant.expires_in,
)
return wire.model_dump_json(exclude_none=True)
def _decode_claims(
compact: str,
signing_key: SecretStr,
) -> _EnvelopeClaims | BadSignature | MalformedPayload:
"""Verify the HS256 signature and shape of an attacker-controlled compact JWT.
``compact`` is fully hostile and bounded to ``MAX_ENVELOPE_BYTES`` by the caller.
PyJWT's ``iat``/``nbf``/``exp`` validators are disabled: they raise on hostile claim
types and, for ``iat``/``nbf``, compare against the wall clock rather than the
injected ``now`` (``exp`` is checked by the caller against ``now``). Apart from a
signature mismatch (``BadSignature``), every decode failure is ``MalformedPayload``:
a non-UTF-8 candidate surfaces as ``UnicodeEncodeError`` (a ``ValueError``), a
non-string registered claim such as ``iss`` as a ``TypeError`` from PyJWT's claim
validators, and a wrong issuer or structurally invalid token as an
``InvalidTokenError``. ``_EnvelopeClaims`` is the total type gate for the payload.
"""
try:
payload = jwt.decode(
compact,
signing_key.get_secret_value(),
algorithms=[_ENVELOPE_JWT_ALGORITHM],
issuer=ENVELOPE_ISSUER,
options={
"verify_exp": False,
"verify_iat": False,
"verify_nbf": False,
"require": ["iss", "iat", "exp"],
},
)
except jwt.InvalidSignatureError:
return BadSignature()
except (jwt.InvalidTokenError, ValueError, TypeError):
return MalformedPayload()
try:
return _EnvelopeClaims.model_validate(payload)
except ValidationError:
return MalformedPayload()
def _encrypt_grant_blob(plaintext: str, encryption_key: SecretStr) -> str:
ciphertext = bytes(encrypt_value(value=plaintext, signing_key=encryption_key.get_secret_value()))
return base64.urlsafe_b64encode(ciphertext).decode("ascii")
def _decrypt_grant(
blob: str,
encryption_key: SecretStr,
) -> UpstreamTokenGrant | DecryptFailed | MalformedPayload:
from nacl.exceptions import CryptoError
try:
plaintext = decrypt_value(
value=base64.urlsafe_b64decode(blob),
signing_key=encryption_key.get_secret_value(),
)
except (CryptoError, ValueError):
return DecryptFailed()
try:
return UpstreamTokenGrant.model_validate_json(plaintext)
except ValidationError:
return MalformedPayload()

166
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -115,6 +115,93 @@ if MCP_AVAILABLE:
if isinstance(logging_error, BaseException):
verbose_logger.warning("MCP tool call logging failed (continuing): %s", logging_error)
def _relay_upstream_auth_http_exception(e: MCPUpstreamAuthError, request: Request) -> HTTPException:
"""Convert a client-forwarded pass-through upstream 401 into an HTTPException that preserves the
upstream WWW-Authenticate, so a standards-compliant MCP client can run the upstream OAuth flow
instead of the generic 500 the endpoint catch-all would return."""
return e.to_http_exception(
base_url=get_request_base_url(request),
request_path=request.scope.get("_original_path") or request.url.path,
)
async def _handle_virtual_mcp_tool(
request: Request,
data: Dict[str, Any],
tool_name: str,
user_api_key_dict: UserAPIKeyAuth,
) -> Any:
"""Handle the virtual ``mcp_tool_search`` / ``mcp_tool_call`` REST tools (gated on
``mcp_tool_search_enabled``). Kept out of ``call_tool_rest_api`` so that endpoint stays a single
dispatch. An upstream 401 raised by the virtual ``mcp_tool_call`` propagates unhandled to the
caller's ``except MCPUpstreamAuthError`` relay, the same as the direct call path."""
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_SEARCH_TOOL_NAME,
coerce_top_k,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.proxy_server import general_settings, proxy_config, proxy_logging_obj
if not getattr(getattr(user_api_key_dict, "object_permission", None), "mcp_tool_search_enabled", False):
raise HTTPException(
status_code=403,
detail={"error": "forbidden", "message": f"{tool_name} requires mcp_tool_search_enabled on the key"},
)
tool_arguments = data.get("arguments") or {}
rest_client_ip = IPAddressUtils.get_mcp_client_ip(request)
(
virtual_mcp_auth_header,
virtual_mcp_server_auth_headers,
virtual_raw_headers,
) = _extract_mcp_headers_from_request(request, MCPRequestHandler)
virtual_oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(request.headers)
if tool_name == MCP_TOOL_SEARCH_TOOL_NAME:
return await handle_mcp_tool_search(
query=tool_arguments.get("query", ""),
top_k=coerce_top_k(tool_arguments.get("top_k", 5)),
user_api_key_dict=user_api_key_dict,
client_ip=rest_client_ip,
mcp_auth_header=virtual_mcp_auth_header,
mcp_server_auth_headers=virtual_mcp_server_auth_headers,
oauth2_headers=virtual_oauth2_headers,
raw_headers=virtual_raw_headers,
)
# MCP_TOOL_CALL_TOOL_NAME: run the same pre-call pipeline as the normal path so the tool
# execution is spend-logged and guardrail-checked.
(_, virtual_logging_obj) = await ProxyBaseLLMRequestProcessing(data=data).common_processing_pre_call_logic(
request=request,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config,
route_type=CallTypes.call_mcp_tool.value,
proxy_logging_obj=proxy_logging_obj,
general_settings=general_settings,
)
_tool_start_time = datetime.now()
result = await handle_mcp_tool_call(
tool_name=tool_arguments.get("tool_name", ""),
arguments=tool_arguments.get("arguments") or {},
user_api_key_dict=user_api_key_dict,
client_ip=rest_client_ip,
mcp_auth_header=virtual_mcp_auth_header,
mcp_server_auth_headers=virtual_mcp_server_auth_headers,
oauth2_headers=virtual_oauth2_headers,
raw_headers=virtual_raw_headers,
litellm_logging_obj=virtual_logging_obj,
)
await _safe_fire_mcp_tool_call_logging(
virtual_logging_obj,
result,
_tool_start_time,
datetime.now(),
user_api_key_auth=user_api_key_dict,
request_data=data,
)
return result
def _get_server_auth_header(
server,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
@ -821,77 +908,10 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_CALL_TOOL_NAME,
MCP_TOOL_SEARCH_TOOL_NAME,
coerce_top_k,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
if not getattr(
getattr(user_api_key_dict, "object_permission", None),
"mcp_tool_search_enabled",
False,
):
raise HTTPException(
status_code=403,
detail={
"error": "forbidden",
"message": f"{tool_name} requires mcp_tool_search_enabled on the key",
},
)
rest_client_ip = IPAddressUtils.get_mcp_client_ip(request)
(
virtual_mcp_auth_header,
virtual_mcp_server_auth_headers,
virtual_raw_headers,
) = _extract_mcp_headers_from_request(request, MCPRequestHandler)
virtual_oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(request.headers)
if tool_name == MCP_TOOL_SEARCH_TOOL_NAME:
return await handle_mcp_tool_search(
query=tool_arguments.get("query", ""),
top_k=coerce_top_k(tool_arguments.get("top_k", 5)),
user_api_key_dict=user_api_key_dict,
client_ip=rest_client_ip,
mcp_auth_header=virtual_mcp_auth_header,
mcp_server_auth_headers=virtual_mcp_server_auth_headers,
oauth2_headers=virtual_oauth2_headers,
raw_headers=virtual_raw_headers,
)
else: # MCP_TOOL_CALL_TOOL_NAME
# Run the same pre-call pipeline as the normal call path so the
# tool execution is spend-logged and guardrail-checked.
(
_,
virtual_logging_obj,
) = await ProxyBaseLLMRequestProcessing(data=data).common_processing_pre_call_logic(
request=request,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config,
route_type=CallTypes.call_mcp_tool.value,
proxy_logging_obj=proxy_logging_obj,
general_settings=general_settings,
)
_tool_start_time = datetime.now()
result = await handle_mcp_tool_call(
tool_name=tool_arguments.get("tool_name", ""),
arguments=tool_arguments.get("arguments") or {},
user_api_key_dict=user_api_key_dict,
client_ip=rest_client_ip,
mcp_auth_header=virtual_mcp_auth_header,
mcp_server_auth_headers=virtual_mcp_server_auth_headers,
oauth2_headers=virtual_oauth2_headers,
raw_headers=virtual_raw_headers,
litellm_logging_obj=virtual_logging_obj,
)
await _safe_fire_mcp_tool_call_logging(
virtual_logging_obj,
result,
_tool_start_time,
datetime.now(),
user_api_key_auth=user_api_key_dict,
request_data=data,
)
return result
return await _handle_virtual_mcp_tool(request, data, tool_name, user_api_key_dict)
# Validate required parameters early
server_id = data.get("server_id")
@ -1020,8 +1040,16 @@ if MCP_AVAILABLE:
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except MCPUpstreamAuthError as e:
# A client-forwarded pass-through upstream 401 from either the direct or the virtual call
# branch. Relay it as a 401 + WWW-Authenticate so the MCP client can re-run upstream OAuth,
# and log at info: an expected caller-must-reauth signal, not an operator-actionable error.
verbose_logger.info(f"MCP tool call relaying upstream HTTP {e.status_code}")
raise _relay_upstream_auth_http_exception(e, request)
except HTTPException as e:
# Re-raise HTTPException as-is to preserve status code and detail
# Locally generated denials (tool/server permission, IP filtering, BYOK) stay at error level
# so restriction probing keeps full monitoring visibility; the relayed upstream 401 above is
# the only status demoted to info.
verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}")
raise e
except Exception as e:

54
litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py
View file

@ -7,6 +7,8 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints.
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.exceptions import ContextWindowExceededError
from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR
if TYPE_CHECKING:
@ -15,6 +17,36 @@ if TYPE_CHECKING:
from litellm.router import Router
class SemanticToolFilterContextWindowError(Exception):
"""Raised when the embedding model exceeds its context window, so semantic filtering cannot run."""
def __init__(self, embedding_model: str, stage: str, original_error: str):
self.embedding_model = embedding_model
self.stage = stage
self.original_error = original_error
super().__init__(
f"MCP semantic tool filtering could not run: embedding model '{embedding_model}' "
f"exceeded its context window while embedding {stage}. "
f"The request was blocked instead of silently passing all tools through. "
f"Switch to an embedding model with a larger context window, or disable "
f"semantic tool filtering."
)
def _is_context_window_error(error: Optional[BaseException], max_depth: int = 5) -> bool:
"""Detect a context-window overflow anywhere in an exception's cause chain."""
current = error
for _ in range(max_depth):
if current is None:
return False
if isinstance(current, ContextWindowExceededError):
return True
if ExceptionCheckers.is_error_str_context_window_exceeded(str(current)):
return True
current = current.__cause__ or current.__context__
return False
class SemanticMCPToolFilter:
"""Filters MCP tools using semantic similarity to reduce context window size."""
@ -42,6 +74,7 @@ class SemanticMCPToolFilter:
self.embedding_model = embedding_model
self.router_instance = litellm_router_instance
self.tool_router: Optional["SemanticRouter"] = None
self.context_window_error: Optional[str] = None
self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts
async def build_router_from_mcp_registry(self) -> None:
@ -111,6 +144,7 @@ class SemanticMCPToolFilter:
return
try:
self.context_window_error = None
# Convert tools to routes
routes = []
self._tool_map = {}
@ -143,6 +177,9 @@ class SemanticMCPToolFilter:
except Exception as e:
verbose_logger.error(f"Failed to build semantic router: {e}")
self.tool_router = None
if _is_context_window_error(e):
self.context_window_error = str(e)
return
raise
async def filter_tools(
@ -169,6 +206,13 @@ class SemanticMCPToolFilter:
if not available_tools:
return available_tools
if self.context_window_error is not None:
raise SemanticToolFilterContextWindowError(
embedding_model=self.embedding_model,
stage="the MCP tool descriptions during semantic router build",
original_error=self.context_window_error,
)
if not query or not query.strip():
return available_tools
@ -189,6 +233,16 @@ class SemanticMCPToolFilter:
return self._get_tools_by_names(matched_tool_names, available_tools)
except Exception as e:
if _is_context_window_error(e):
verbose_logger.error(
f"Semantic tool filter embedding exceeded its context window: {e}",
exc_info=True,
)
raise SemanticToolFilterContextWindowError(
embedding_model=self.embedding_model,
stage="the user query",
original_error=str(e),
) from e
verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True)
return available_tools

35
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -1006,6 +1006,22 @@ if MCP_AVAILABLE:
content=[TextContent(text=f"Error: {str(e.detail)}", type="text")],
isError=True,
)
except MCPUpstreamAuthError as e:
# The MCP session manager serializes handler exceptions as JSON-RPC errors, so a
# mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST
# call path and the connect-time preemptive check do. Return an explicit isError
# naming the upstream status (at info level, not a traceback) so the client still
# learns it must re-authenticate upstream and expected pass-through 401s don't spam.
verbose_logger.info(f"Upstream auth failure calling MCP tool: HTTP {e.status_code}")
return CallToolResult(
content=[
TextContent(
text=f"Error: upstream authentication required (HTTP {e.status_code})",
type="text",
)
],
isError=True,
)
except Exception as e:
verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}")
return CallToolResult(
@ -2943,6 +2959,14 @@ if MCP_AVAILABLE:
raw_headers=raw_headers,
**kwargs,
)
except MCPUpstreamAuthError:
# A client-forwarded pass-through upstream 401 is an expected caller-must-reauth signal, so
# re-raise it without post_call_failure_hook, which fires the proxy's llm_exceptions alert.
# mcp_server_tool_call then downgrades it to an informational isError result for the
# streamable client. Note: this function is @client-decorated, so the decorator's standard
# failure logging still records the event (spend log / OTel); only the extra alert sink is
# skipped here.
raise
except Exception as e:
traceback_str = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
from litellm.proxy.proxy_server import proxy_logging_obj
@ -3676,6 +3700,17 @@ if MCP_AVAILABLE:
and not _scope_has_authorization_header(scope)
and not _client_has_per_server_auth_header(server, mcp_server_auth_headers)
):
if server.is_dcr_bridge:
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={
"www-authenticate": _get_passthrough_www_authenticate(
scope=scope,
server_name=server_name,
)
},
)
upstream_status, upstream_www_authenticate = await _probe_upstream_auth(server.url or "", "")
if upstream_status == 401 and upstream_www_authenticate:
raise HTTPException(

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vffq7buvlg04.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/122djf0bncn-8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0axk76owb7jv..js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sqw622fcvsv4.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/14g~hmf3h_efw.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0cmepm.jkel-i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0onea0n77pqw1.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/02-u6qtmsnqn0.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/05q6y.kb.q2s..js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0rehsq9xe1kde.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0nk7-_~gcxbz0.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0~r95y0t-0dlp.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0kc37~1yrtr2p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/04s-iyzsr4cq~.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/16zj68af4snfa.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/003_1s9xbht43.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0tmaomqtwbi33.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/02-2~p5k.ielz.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0op63kdo3uwng.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/112_-0alpxot8.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/12yfh0_n50ojz.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/06w8_.601z7_i.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0nht59ws0elww.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/18187o3gb9vc5.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0._ir~nvcseg7.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

55
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}

14
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/09qhs_ev_bxr8.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6iga_s7xld1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0hpxif-db_y5-.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/15j3hwz2dxrik.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"N7WCdfNd30Hp6HEF5tFIL"}

0
litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_clientMiddlewareManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/N7WCdfNd30Hp6HEF5tFIL/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/0-.xiiczht-vh.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0-0c4mv4-mc9n.js
View file

File diff suppressed because one or more lines are too long

48
litellm/proxy/_experimental/out/_next/static/chunks/0-px9-g~2oyp5.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0-s2am3eulbyd.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0-~nw1zmks9_4.js Normal file
View file

File diff suppressed because one or more lines are too long

3
litellm/proxy/_experimental/out/_next/static/chunks/0._ir~nvcseg7.js Normal file
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/0.cm9osit06~i.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0.w8~sa9q0n_s.js
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/_next/static/chunks/0y5ybr0-5yc_z.js → litellm/proxy/_experimental/out/_next/static/chunks/0.xp85h9ki~9t.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0.zblsr85hcyn.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/16-7emebrt-xf.js → litellm/proxy/_experimental/out/_next/static/chunks/00-xblkiz3o8~.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/003_1s9xbht43.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00fcsizkvc4hx.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00pr1xqcusy8a.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/00tczcrtv5upo.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/010e8lif45kbo.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/016u~n51r0h1k.js Normal file
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more