scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge remote-tracking branch 'upstream/litellm_internal_staging' into deepkeep-as-internal

Browse source
This commit is contained in:
Yaniv Israel 2026-07-05 13:06:21 +03:00
commit 08f732d93c
1292 changed files with 43189 additions and 13842 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
.circleci/config.yml
View file

@ -5,6 +5,16 @@ orbs:
win: circleci/windows@5.0 # Add Windows orb
commands:
skip_if_unrelated_changes:
parameters:
category:
type: enum
enum: ["backend", "client"]
default: "backend"
steps:
- run:
name: "Skip job when no << parameters.category >>-relevant files changed"
command: bash .circleci/scripts/path_filter.sh << parameters.category >>
setup_google_dns:
steps:
- run:
@ -282,6 +292,7 @@ jobs:
parallelism: 4
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -354,6 +365,7 @@ jobs:
parallelism: 4
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -427,6 +439,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -480,6 +493,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -545,6 +559,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -584,6 +599,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -624,6 +640,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -656,6 +673,7 @@ jobs:
FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -705,6 +723,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -755,6 +774,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -787,6 +807,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -832,6 +853,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -877,6 +899,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -918,6 +941,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -963,6 +987,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1007,6 +1032,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -1045,6 +1071,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1089,6 +1116,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1132,6 +1160,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1163,6 +1192,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1205,6 +1235,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1248,6 +1279,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1291,6 +1323,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1321,6 +1354,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1366,6 +1400,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1407,6 +1442,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -1459,6 +1495,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1482,6 +1519,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1507,6 +1545,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1531,6 +1570,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns
@ -1606,6 +1646,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1698,6 +1739,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns
@ -1746,13 +1788,13 @@ jobs:
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
-e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1787,6 +1829,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1832,13 +1875,13 @@ jobs:
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1869,6 +1912,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1911,14 +1955,14 @@ jobs:
-e COHERE_API_KEY=$COHERE_API_KEY \
-e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \
-e GCS_FLUSH_INTERVAL="1" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1960,13 +2004,13 @@ jobs:
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE="bad-license" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
--port 4000
- run:
name: Start outputting logs for second container
@ -2000,6 +2044,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2041,13 +2086,13 @@ jobs:
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e PROXY_BATCH_WRITE_AT=2 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2085,6 +2130,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2117,13 +2163,13 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Run Docker container 2
command: |
@ -2139,13 +2185,13 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4001 \
--detailed_debug
--port 4001
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2180,6 +2226,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2201,19 +2248,20 @@ jobs:
# the OTEL test - should get this as a trace
command: |
docker run -d \
--restart on-failure \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2252,6 +2300,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
# Remove Docker CLI installation since it's already available in machine executor
- install_uv
@ -2289,13 +2338,13 @@ jobs:
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e GCS_FLUSH_INTERVAL="1" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2333,6 +2382,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2365,14 +2415,14 @@ jobs:
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2471,6 +2521,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2499,13 +2550,13 @@ jobs:
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2537,6 +2588,7 @@ jobs:
- *python312_image
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: .
# Check file locations
@ -2567,6 +2619,8 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
@ -2609,6 +2663,8 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
@ -2629,7 +2685,7 @@ jobs:
cd ui/litellm-dashboard
CI=true npm run test -- --run \
--pool forks --poolOptions.forks.maxForks=8
--pool forks --poolOptions.forks.maxForks=6
e2e_ui_testing:
docker:
@ -2654,6 +2710,8 @@ jobs:
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_uv
- restore_cache:
@ -2791,6 +2849,8 @@ jobs:
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_uv
- restore_cache:
@ -2892,6 +2952,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- run:
name: Build Docker image
@ -2917,6 +2978,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns

27
.circleci/scripts/classify_changes.sh Executable file
View file

@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: classify_changes.sh <backend|client>}"
has_client=false
has_backend=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
ui/*) has_client=true ;;
docs/* | *.md | *.mdx) : ;;
*) has_backend=true ;;
esac
done
case "$category" in
backend)
[ "$has_backend" = true ] && echo run || echo skip
;;
client)
{ [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
;;
*)
echo run
;;
esac

40
.circleci/scripts/path_filter.sh Executable file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: path_filter.sh <backend|client>}"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
run_full() {
echo "path-filter[$category]: running job ($1)"
exit 0
}
[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
candidate_bases="main litellm_internal_staging litellm_oss_staging"
merge_base=""
for base in $candidate_bases; do
git fetch --quiet origin "$base" 2>/dev/null || continue
candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue
[ -n "$candidate" ] || continue
if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then
merge_base="$candidate"
fi
done
[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases"
changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed"
[ -n "$changed" ] || run_full "no files changed vs $merge_base"
echo "path-filter[$category]: changed files vs ${merge_base}:"
printf '%s\n' "$changed" | sed 's/^/ /' || true
decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed"
if [ "$decision" = run ]; then
run_full "$category-relevant changes detected"
fi
echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful"
circleci-agent step halt

2
.github/pull_request_template.md vendored
View file

@ -4,7 +4,7 @@
## Linear ticket
<!-- if you are an internal contributor (e.g., your username is postfixed with -berri or -berriai), add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
## Pre-Submission checklist

32
.github/workflows/create-release.yml vendored
View file

@ -122,10 +122,28 @@ jobs:
makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
}
try {
await github.rest.git.createRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: `refs/tags/${tag}`,
sha: commitHash,
});
} catch (error) {
if (error.status !== 422) throw error;
const existing = await github.rest.git.getRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: `tags/${tag}`,
});
if (existing.data.object.sha !== commitHash) {
throw new Error(`Tag ${tag} already exists at ${existing.data.object.sha}, expected ${commitHash}`);
}
}
const response = await github.rest.repos.createRelease({
draft: true,
generate_release_notes: true,
target_commitish: commitHash,
name: tag,
owner: context.repo.owner,
prerelease: isPrerelease,
@ -138,11 +156,21 @@ jobs:
owner: context.repo.owner,
repo: context.repo.repo,
release_id: response.data.id,
tag_name: tag,
body: updatedBody,
draft: false,
make_latest: makeLatest,
});
if (!isPrerelease) {
await github.rest.repos.updateRelease({
owner: context.repo.owner,
repo: context.repo.repo,
release_id: response.data.id,
tag_name: tag,
make_latest: makeLatest,
});
}
} catch (error) {
core.setFailed(error.message);
}

2
.github/workflows/test-linting.yml vendored
View file

@ -63,7 +63,7 @@ jobs:
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git diff --name-only "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
echo "No changed litellm Python files to check with ruff format."
exit 0

2
.gitignore vendored
View file

@ -130,3 +130,5 @@ crash.*.log
# pytest coverage data
.coverage
ui/litellm-dashboard/out/

10
CLAUDE.md
View file

@ -17,9 +17,13 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
Always use @.github/pull_request_template.md as a guide for your PR body
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
@ -43,9 +47,11 @@ If you're trying to create a new function that relies on untyped stuff, instead
If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
Commit and push your work when you're done without asking
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names

33
Makefile
View file

@ -4,7 +4,7 @@
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev format \
info lint lint-dev lint-checks format \
lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
install-dev install-proxy-dev install-test-deps install-hooks \
@ -53,6 +53,11 @@ help:
UV := uv
UV_RUN := $(UV) run --no-sync
LINT_DEP_INSTALL ?= install-dev
LINT_DEP_BASE ?= lint-fetch-base
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
# Show info
info:
@echo "UV: $(UV)"
@ -107,12 +112,12 @@ lint-fetch-base:
# running proxy need.
lint-install:
$(UV) sync --inexact --frozen --group proxy-dev
$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
$(UV_RUN) python scripts/prisma_generate_if_needed.py
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
# only the litellm Python files changed vs the base are checked, so a pre-existing
# format issue elsewhere doesn't block an unrelated commit.
lint-format-check-changed: install-dev lint-fetch-base
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \
@ -121,7 +126,7 @@ lint-format-check-changed: install-dev lint-fetch-base
fi
# Linting targets
lint-ruff: install-dev
lint-ruff: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) ruff check . && cd ..
# faster linter for developing ...
@ -156,12 +161,12 @@ lint-ruff-FULL-dev: install-dev
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
else echo "No changed .py files to check."; fi
lint-basedpyright: install-dev lint-fetch-base
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: install-dev lint-fetch-base
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
# --update lowers each limit by what this branch fixed since its branch point, so
@ -179,7 +184,7 @@ lint-ruff-budget: install-dev
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
# means the CI check will pass too.
lint-gate: install-dev lint-fetch-base
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging
lint-ruff-budget-update: install-dev lint-fetch-base
@ -191,10 +196,10 @@ lint-type-discipline-budget-update: install-dev lint-fetch-base
# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
check-circular-imports: install-dev
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
check-import-safety: $(LINT_DEP_INSTALL)
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
@ -202,9 +207,13 @@ check-import-safety: install-dev
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin/litellm_internal_staging). lint-install is first so the
# Prisma client exists before basedpyright runs.
lint: lint-install lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety

38
basedpyright-code-budget.json
View file

@ -3,16 +3,16 @@
"limit": 37484
},
"reportArgumentType": {
"limit": 2721
"limit": 2704
},
"reportAssignmentType": {
"limit": 330
},
"reportAttributeAccessIssue": {
"limit": 519
"limit": 516
},
"reportCallIssue": {
"limit": 131
"limit": 124
},
"reportConstantRedefinition": {
"limit": 59
@ -42,7 +42,7 @@
"limit": 18
},
"reportIndexIssue": {
"limit": 39
"limit": 37
},
"reportInvalidTypeForm": {
"limit": 35
@ -51,7 +51,7 @@
"limit": 5
},
"reportMatchNotExhaustive": {
"limit": 2
"limit": 0
},
"reportMissingParameterType": {
"limit": 5900
@ -63,25 +63,25 @@
"limit": 41
},
"reportOperatorIssue": {
"limit": 9
"limit": 0
},
"reportOptionalCall": {
"limit": 7
"limit": 0
},
"reportOptionalIterable": {
"limit": 6
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1086
"limit": 1085
},
"reportOptionalOperand": {
"limit": 6
"limit": 0
},
"reportOptionalSubscript": {
"limit": 17
"limit": 0
},
"reportPossiblyUnboundVariable": {
"limit": 78
"limit": 77
},
"reportPrivateUsage": {
"limit": 2438
@ -90,28 +90,28 @@
"limit": 12
},
"reportReturnType": {
"limit": 226
"limit": 225
},
"reportTypedDictNotRequiredAccess": {
"limit": 30
"limit": 27
},
"reportUndefinedVariable": {
"limit": 5
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45905
"limit": 45895
},
"reportUnknownLambdaType": {
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40556
"limit": 40541
},
"reportUnknownParameterType": {
"limit": 20418
},
"reportUnknownVariableType": {
"limit": 32168
"limit": 32151
},
"reportUnnecessaryCast": {
"limit": 177
@ -141,6 +141,6 @@
"limit": 1005
},
"reportUnusedVariable": {
"limit": 1298
"limit": 1297
}
}

2
docker/build_admin_ui.sh
View file

@ -57,8 +57,6 @@ source ~/.nvm/nvm.sh
nvm install v18.17.0
nvm use v18.17.0
# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
# cd in to /ui/litellm-dashboard
cd ui/litellm-dashboard

73
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -351,10 +351,8 @@ class BaseEmailLogger(CustomLogger):
)
# Calculate percentage and alert threshold
percentage = (
threshold_pct
if threshold_pct is not None
else int(EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100)
percentage = threshold_pct if threshold_pct is not None else int(
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100
)
threshold_fraction = percentage / 100.0
alert_threshold_str = (
@ -479,9 +477,12 @@ class BaseEmailLogger(CustomLogger):
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}"
# Check if we've already sent this alert
result = await _cache.async_get_cache(key=_cache_key)
if result is None:
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is None or send_count <= 1:
# Create WebhookEvent for soft budget alert
event_message = f"Soft Budget Crossed - Total Soft Budget: ${user_info.soft_budget}"
webhook_event = WebhookEvent(
@ -510,18 +511,12 @@ class BaseEmailLogger(CustomLogger):
await self.send_team_soft_budget_alert_email(webhook_event)
else:
await self.send_soft_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending soft budget alert email: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
return
# For max_budget_alert, check if we've already sent an alert
@ -547,9 +542,12 @@ class BaseEmailLogger(CustomLogger):
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:max_budget_alert:{_id}"
# Check if we've already sent this alert
result = await _cache.async_get_cache(key=_cache_key)
if result is None:
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is None or send_count <= 1:
# Calculate percentage
percentage = int(
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100
@ -578,18 +576,12 @@ class BaseEmailLogger(CustomLogger):
try:
await self.send_max_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending max budget alert email: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
return
async def _handle_multi_threshold_max_budget_alert(
@ -615,11 +607,9 @@ class BaseEmailLogger(CustomLogger):
continue
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
result = await _cache.async_get_cache(key=_cache_key)
if result is not None:
continue
_cache_key = (
f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
)
# Parse emails + auto-include owner
emails = _parse_email_list(raw_emails)
@ -634,9 +624,15 @@ class BaseEmailLogger(CustomLogger):
continue
recipient_emails = list(set(emails))
event_message = (
f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is not None and send_count > 1:
continue
event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
webhook_event = WebhookEvent(
event="max_budget_alert",
event_message=event_message,
@ -662,16 +658,21 @@ class BaseEmailLogger(CustomLogger):
threshold_pct=threshold_pct,
recipient_emails=recipient_emails,
)
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending multi-threshold max budget alert email for {threshold_pct}%: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None:
try:
await cache.async_delete_cache(key=cache_key)
except Exception:
verbose_proxy_logger.debug(
"Failed to release budget alert claim for %s; it expires with the TTL",
cache_key,
)
async def _get_email_params(
self,

233
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -13,6 +13,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -26,6 +28,7 @@ class CheckBatchCost:
proxy_logging_obj: "ProxyLogging",
prisma_client: "PrismaClient",
llm_router: "Router",
track_unmanaged_vertex_batch_cost: bool = False,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -33,6 +36,7 @@ class CheckBatchCost:
self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
self._track_unmanaged_vertex_batch_cost = track_unmanaged_vertex_batch_cost
# Cached after the first poll cycle. Once we know the column is absent we skip
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
@ -110,6 +114,182 @@ class CheckBatchCost:
order={"created_at": "asc"},
)
@staticmethod
def _record_error(
prom_logger: Optional["PrometheusLogger"], error_type: str
) -> None:
if prom_logger is not None:
prom_logger.record_check_batch_cost_error(error_type)
def _resolve_job_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
"""
Resolve (model_id, batch_id) for a managed-object row, where model_id is a router
deployment id and batch_id is the raw provider batch id.
Managed batches encode both in a base64 unified id. Unmanaged Vertex batches, created with
a raw gs:// input_file_id, store the raw provider job id as unified_object_id; when
track_unmanaged_vertex_batch_cost is enabled the model is derived from the gs:// path and
mapped to a configured vertex_ai deployment. Returns None (recording a metric) when the row
can't be routed.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
unified_object_id = job.unified_object_id
decoded = _is_base64_encoded_unified_file_id(unified_object_id)
if decoded:
model_id = get_model_id_from_unified_batch_id(decoded)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
self._record_error(prom_logger, "invalid_model_id")
return None
return model_id, get_batch_id_from_unified_batch_id(decoded)
if self._track_unmanaged_vertex_batch_cost:
return self._resolve_unmanaged_vertex_routing(job, prom_logger)
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
def _resolve_unmanaged_vertex_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
from litellm.llms.vertex_ai.batches.transformation import (
VertexAIBatchTransformation,
)
input_file_id = self._get_input_file_id(job)
if not VertexAIBatchTransformation.is_unmanaged_gcs_batch_input_file_id(
input_file_id
):
verbose_proxy_logger.info(
f"Skipping job {job.unified_object_id}: not an unmanaged vertex batch "
"(no gs:// input_file_id with a publishers/ model path)"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
assert input_file_id is not None # narrowed by is_unmanaged_gcs_batch_input_file_id
bare_model_name = VertexAIBatchTransformation.get_bare_model_name_from_gcs_file(
input_file_id
)
deployment_id = self._get_vertex_ai_deployment_id_for_bare_model(
bare_model_name
)
if deployment_id is None:
verbose_proxy_logger.info(
f"Skipping unmanaged vertex batch {job.unified_object_id}: no vertex_ai "
f"deployment configured for model {bare_model_name}"
)
self._record_error(prom_logger, "unmanaged_no_matching_deployment")
return None
return deployment_id, job.unified_object_id
def _get_vertex_ai_deployment_id_for_bare_model(
self, bare_model_name: str
) -> Optional[str]:
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
deployment_id = (
self._get_vertex_ai_deployment_id(model_group) if model_group else None
)
if deployment_id is not None:
return deployment_id
return self._get_vertex_ai_deployment_id_from_matching_deployments(
bare_model_name
)
def _get_vertex_ai_deployment_id_from_matching_deployments(
self, bare_model_name: str
) -> Optional[str]:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment in self.llm_router.get_model_list(model_name=None) or []:
litellm_params = deployment.get("litellm_params") or {}
actual_model = litellm_params.get("model")
if not isinstance(actual_model, str):
continue
if not self._is_bare_model_match(actual_model, bare_model_name):
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=actual_model,
custom_llm_provider=litellm_params.get("custom_llm_provider"),
)
except Exception:
continue
if llm_provider != "vertex_ai":
continue
model_info = deployment.get("model_info") or {}
deployment_id = model_info.get("id")
if isinstance(deployment_id, str):
return deployment_id
return None
@staticmethod
def _is_bare_model_match(actual_model: str, bare_model_name: str) -> bool:
return (
actual_model == bare_model_name
or actual_model.endswith(f"/{bare_model_name}")
or actual_model.endswith(f":{bare_model_name}")
)
def _get_vertex_ai_deployment_id(self, model_group: str) -> Optional[str]:
"""
Returns the first deployment id for `model_group` whose provider is vertex_ai,
skipping deployments from other providers that happen to share the model group name.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
if deployment_info is None:
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=deployment_info.litellm_params.model,
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
except Exception:
continue
if llm_provider == "vertex_ai":
return deployment_id
return None
@staticmethod
def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object = job.file_object
if isinstance(file_object, str):
try:
file_object = json.loads(file_object)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(file_object, dict):
return None
try:
return LiteLLMBatch.model_validate(file_object).input_file_id
except Exception:
return None
async def check_batch_cost(self):
"""
Check if the batch JOB has been tracked.
@ -127,8 +307,6 @@ class CheckBatchCost:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
try:
@ -192,31 +370,10 @@ class CheckBatchCost:
else:
jobs = await self._fallback_find_jobs()
for job in jobs:
# get the model from the job
unified_object_id = job.unified_object_id
decoded_unified_object_id = _is_base64_encoded_unified_file_id(
unified_object_id
)
if not decoded_unified_object_id:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_unified_id")
continue
else:
unified_object_id = decoded_unified_object_id
model_id = get_model_id_from_unified_batch_id(unified_object_id)
batch_id = get_batch_id_from_unified_batch_id(unified_object_id)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_model_id")
routing = self._resolve_job_routing(job, prom_logger)
if routing is None:
continue
model_id, batch_id = routing
verbose_proxy_logger.info(
f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}"
@ -233,7 +390,7 @@ class CheckBatchCost:
)
except Exception as e:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
f"Skipping job {job.unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
)
if prom_logger:
prom_logger.record_check_batch_cost_error(
@ -311,7 +468,7 @@ class CheckBatchCost:
deployment_info = self.llm_router.get_deployment(model_id=model_id)
if deployment_info is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid deployment info"
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
)
if prom_logger:
prom_logger.record_check_batch_cost_error(
@ -458,6 +615,26 @@ class CheckBatchCost:
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
elif response.status in ("failed", "expired", "cancelled"):
try:
update_data = {
"status": response.status,
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=update_data,
)
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
)
# Record polling run metrics (always, even if nothing was processed)
if prom_logger:
prom_logger.record_check_batch_cost_run(

16
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -125,23 +125,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
update_data = {
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"updated_by": user_api_key_dict.user_id,
}
if file_object is not None:
db_data["file_object"] = file_object.model_dump_json()
file_object_json = file_object.model_dump_json()
db_data["file_object"] = file_object_json
update_data["file_object"] = file_object_json
# Extract storage metadata from hidden params if present
hidden_params = getattr(file_object, "_hidden_params", {}) or {}
if "storage_backend" in hidden_params:
db_data["storage_backend"] = hidden_params["storage_backend"]
update_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
update_data["storage_url"] = hidden_params["storage_url"]
verbose_logger.debug(
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
f"storage_url={db_data.get('storage_url')}"
)
result = await self.prisma_client.db.litellm_managedfiletable.create(
data=db_data
result = await self.prisma_client.db.litellm_managedfiletable.upsert(
where={"unified_file_id": file_id},
data={"create": db_data, "update": update_data},
)
verbose_logger.debug(
f"LiteLLM Managed File object with id={file_id} stored in db: {result}"

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.45"
version = "0.1.47"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.45"
version = "0.1.47"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260629000000_add_max_concurrent_requests_to_mcp_server_table/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "max_concurrent_requests" INTEGER;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260630190000_add_budget_fallbacks_to_litellm_verification_token/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';

3
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -338,6 +338,7 @@ model LiteLLM_MCPServerTable {
byok_api_key_help_url String?
source_url String?
timeout Float?
max_concurrent_requests Int?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
@ -418,6 +419,7 @@ model LiteLLM_VerificationToken {
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
budget_fallbacks Json @default("{}")
budget_id String?
organization_id String?
object_permission_id String?
@ -511,6 +513,7 @@ model LiteLLM_DeletedVerificationToken {
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
budget_fallbacks Json @default("{}")
router_settings Json? @default("{}")
budget_id String?
organization_id String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.74"
version = "0.4.75"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.74"
version = "0.4.75"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

8
litellm/__init__.py
View file

@ -588,6 +588,7 @@ gemini_models: Set = set()
xai_models: Set = set()
zai_models: Set = set()
deepseek_models: Set = set()
tencent_models: Set = set()
runwayml_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
@ -801,6 +802,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.add(key)
elif value.get("litellm_provider") == "tencent":
tencent_models.add(key)
elif value.get("litellm_provider") == "runwayml":
runwayml_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
@ -1093,6 +1096,7 @@ models_by_provider: dict = {
"zai": zai_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"tencent": tencent_models,
"runwayml": runwayml_models,
"mistral": mistral_chat_models,
"azure_ai": azure_ai_models,
@ -1804,6 +1808,9 @@ if TYPE_CHECKING:
from .llms.deepseek.chat.transformation import (
DeepSeekChatConfig as _DeepSeekChatConfig,
)
from .llms.tencent.chat.transformation import (
TencentChatConfig as _TencentChatConfig,
)
from .llms.sap.chat.transformation import (
GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig,
)
@ -1846,6 +1853,7 @@ if TYPE_CHECKING:
# Type stubs for lazy-loaded config classes (to help mypy understand types)
VLLMConfig: Type[_VLLMConfig]
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
TencentChatConfig: Type[_TencentChatConfig]
GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig]
GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig]
AzureOpenAIO1Config: Type[_AzureOpenAIO1Config]

2
litellm/_lazy_imports_registry.py
View file

@ -284,6 +284,7 @@ LLM_CONFIG_NAMES = (
"LiteLLMProxyChatConfig",
"VLLMConfig",
"DeepSeekChatConfig",
"TencentChatConfig",
"LMStudioChatConfig",
"LmStudioEmbeddingConfig",
"NscaleConfig",
@ -1096,6 +1097,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
),
"VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"),
"DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
"TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"),
"LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
"LmStudioEmbeddingConfig": (
".llms.lm_studio.embed.transformation",

30
litellm/a2a_protocol/main.py
View file

@ -129,6 +129,33 @@ def _set_agent_id_on_logging_obj(
litellm_logging_obj.model_call_details["agent_id"] = agent_id
_A2A_COST_PARAM_KEYS = ("cost_per_query", "input_cost_per_token", "output_cost_per_token")
def _set_litellm_params_on_logging_obj(
kwargs: dict[str, Any],
litellm_params: dict[str, Any],
) -> None:
"""
Merge the agent's pricing params into model_call_details["litellm_params"]
so A2ACostCalculator can read them.
The non-streaming path reuses the proxy-built logging object, whose
litellm_params already carries metadata / proxy_server_request / user-key
context, so merge the pricing keys in rather than replacing the dict.
"""
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
return
cost_params = {key: litellm_params[key] for key in _A2A_COST_PARAM_KEYS if litellm_params.get(key) is not None}
if not cost_params:
return
existing = logging_obj.model_call_details.get("litellm_params") or {}
logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params}
def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
"""
Extract agent info and set model/custom_llm_provider for cost tracking.
@ -477,6 +504,9 @@ async def asend_message(
completion_tokens=completion_tokens,
)
# Merge agent pricing params into the logging obj so cost is calculated
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
# Set agent_id on logging obj for SpendLogs tracking
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)

7
litellm/a2a_protocol/utils.py
View file

@ -121,8 +121,13 @@ class A2ARequestUtils:
Returns:
Tuple of (prompt_tokens, completion_tokens, total_tokens)
"""
# Count input tokens
# Count input tokens. Dump the message to a dict first so extraction hits
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens = A2ARequestUtils.count_tokens(input_text)

10
litellm/caching/disk_cache.py
View file

@ -59,8 +59,9 @@ class DiskCache(BaseCache):
def increment_cache(self, key, value: int, **kwargs) -> int:
# get the value
init_value = self.get_cache(key=key) or 0
value = init_value + value # type: ignore
cached_value = self.get_cache(key=key)
init_value = cached_value if isinstance(cached_value, int) else 0
value = init_value + value
self.set_cache(key, value, **kwargs)
return value
@ -76,8 +77,9 @@ class DiskCache(BaseCache):
async def async_increment(self, key, value: int, **kwargs) -> int:
# get the value
init_value = await self.async_get_cache(key=key) or 0
value = init_value + value # type: ignore
cached_value = await self.async_get_cache(key=key)
init_value = cached_value if isinstance(cached_value, int) else 0
value = init_value + value
await self.async_set_cache(key, value, **kwargs)
return value

2
litellm/constants.py
View file

@ -508,6 +508,7 @@ LITELLM_CHAT_PROVIDERS = [
"text-completion-codestral",
"text-completion-inception",
"deepseek",
"tencent",
"sambanova",
"maritalk",
"cloudflare",
@ -729,6 +730,7 @@ openai_compatible_providers: List = [
"volcengine",
"codestral",
"deepseek",
"tencent",
"deepinfra",
"perplexity",
"xinference",

46
litellm/cost_calculator.py
View file

@ -52,6 +52,9 @@ from litellm.llms.databricks.cost_calculator import (
from litellm.llms.deepseek.cost_calculator import (
cost_per_token as deepseek_cost_per_token,
)
from litellm.llms.tencent.cost_calculator import (
cost_per_token as tencent_cost_per_token,
)
from litellm.llms.fireworks_ai.cost_calculator import (
cost_per_token as fireworks_ai_cost_per_token,
)
@ -219,7 +222,7 @@ def _cost_per_token_custom_pricing_helper(
output_cost = completion_tokens * output_cost_per_token
return input_cost, output_cost
elif custom_cost_per_second is not None:
output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore
output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
return 0, output_cost
return None
@ -625,6 +628,8 @@ def cost_per_token(
return gemini_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
elif custom_llm_provider == "deepseek":
return deepseek_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "tencent":
return tencent_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "perplexity":
return perplexity_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "xai":
@ -657,29 +662,27 @@ def cost_per_token(
data_residency=data_residency,
)
if model_info.get("input_cost_per_second", None) is not None and response_time_ms is not None:
input_cost_per_second = model_info.get("input_cost_per_second")
if input_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; response time: %s",
model,
model_info.get("input_cost_per_second", None),
input_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
prompt_tokens_cost_usd_dollar = (
model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
if model_info.get("output_cost_per_second", None) is not None and response_time_ms is not None:
output_cost_per_second = model_info.get("output_cost_per_second")
if output_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - output_cost_per_second: %s; response time: %s",
model,
model_info.get("output_cost_per_second", None),
output_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
completion_tokens_cost_usd_dollar = (
model_info["output_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
verbose_logger.debug(
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
@ -1495,6 +1498,7 @@ def completion_cost(
custom_llm_provider=custom_llm_provider,
litellm_model_name=model,
data_residency=data_residency,
litellm_logging_obj=litellm_logging_obj,
)
elif call_type == _MCP_CALL_TYPE:
from litellm.proxy._experimental.mcp_server.cost_calculator import (
@ -2297,6 +2301,7 @@ def handle_realtime_stream_cost_calculation(
custom_llm_provider: str,
litellm_model_name: str,
data_residency: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
) -> float:
"""
Handles the cost calculation for realtime stream responses.
@ -2332,14 +2337,25 @@ def handle_realtime_stream_cost_calculation(
input_cost_per_token += _input_cost_per_token
output_cost_per_token += _output_cost_per_token
break # exit if we find a valid model
total_cost = input_cost_per_token + output_cost_per_token
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results):
total_cost += handle_realtime_transcription_cost_calculation(
transcription_cost = (
handle_realtime_transcription_cost_calculation(
results=results,
custom_llm_provider=custom_llm_provider,
litellm_model_name=litellm_model_name,
)
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
else 0.0
)
total_cost = input_cost_per_token + output_cost_per_token + transcription_cost
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=input_cost_per_token,
completion_tokens_cost_usd_dollar=output_cost_per_token,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=total_cost,
additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None,
)
return total_cost

6
litellm/exceptions.py
View file

@ -1165,12 +1165,18 @@ class ModifyResponseException(Exception):
request_data: Dict[str, Any],
guardrail_name: Optional[str] = None,
detection_info: Optional[Dict[str, Any]] = None,
original_response: Optional[Any] = None,
):
self.message = message
self.model = model
self.request_data = request_data
self.guardrail_name = guardrail_name
self.detection_info = detection_info or {}
# The LLM response that was blocked (post-call). Carries the real token
# usage the upstream call consumed, so the synthetic block response can
# report it instead of discarding it. None for pre-call blocks (the LLM
# was never invoked).
self.original_response = original_response
super().__init__(message)

22
litellm/experimental_mcp_client/client.py
View file

@ -520,13 +520,28 @@ class MCPClient:
# Return empty list instead of raising to allow graceful degradation
return []
@staticmethod
def error_tool_result(exc: Exception) -> MCPCallToolResult:
"""The error result ``call_tool`` returns when it swallows a failure (no re-execution)."""
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{type(exc).__name__}: {str(exc)}")],
isError=True,
)
async def call_tool(
self,
call_tool_request_params: MCPCallToolRequestParams,
host_progress_callback: Optional[Callable] = None,
raise_on_error: bool = False,
) -> MCPCallToolResult:
"""
Call an MCP Tool.
Args:
raise_on_error: When True, re-raise the underlying exception instead of returning an
``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect
an upstream 401 so it can re-mint the exchanged token and retry once; every other
caller keeps the default and gets graceful ``isError`` degradation.
"""
verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'")
@ -579,11 +594,10 @@ class MCPClient:
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out."
)
if raise_on_error:
raise
# Return a default error result instead of raising
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{error_type}: {str(e)}")], # Empty content for error case
isError=True,
)
return self.error_tool_result(e)
async def list_prompts(self) -> List[Prompt]:
"""List available prompts from the server."""

10
litellm/fine_tuning/main.py
View file

@ -256,8 +256,6 @@ def create_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
# Prepare Azure-specific parameters for extra_body
extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
@ -442,7 +440,7 @@ def cancel_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
@ -457,8 +455,6 @@ def cancel_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job(
api_base=api_base,
@ -616,8 +612,6 @@ def list_fine_tuning_jobs(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs(
api_base=api_base,
@ -759,8 +753,6 @@ def retrieve_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.retrieve_fine_tuning_job(
api_base=api_base,

6
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -61,13 +61,15 @@ class AzureSentinelLogger(CustomBatchLogger):
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
audit_stream_name (str, optional): Stream name from DCR for audit logs.
If not provided, audit logs use the standard stream name.
If not provided, will use AZURE_SENTINEL_AUDIT_STREAM_NAME env var or the standard stream name.
"""
self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv("AZURE_SENTINEL_DCR_IMMUTABLE_ID")
resolved_stream_name = stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
resolved_audit_stream_name = (
audit_stream_name or os.getenv("AZURE_SENTINEL_AUDIT_STREAM_NAME") or resolved_stream_name
)
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
resolved_tenant_id = tenant_id or os.getenv("AZURE_SENTINEL_TENANT_ID") or os.getenv("AZURE_TENANT_ID")
resolved_client_id = client_id or os.getenv("AZURE_SENTINEL_CLIENT_ID") or os.getenv("AZURE_CLIENT_ID")

14
litellm/integrations/datadog/datadog.py
View file

@ -354,14 +354,14 @@ class DataDogLogger(
Raises:
Raises a NON Blocking verbose_logger.exception if an error occurs
"""
if not self.log_queue:
verbose_logger.exception("Datadog: log_queue does not exist")
return
batch_to_send = self.log_queue[:]
self.log_queue = []
try:
if not self.log_queue:
verbose_logger.exception("Datadog: log_queue does not exist")
return
batch_to_send = self.log_queue[:]
self.log_queue = []
verbose_logger.debug(
"Datadog - about to flush %s events on %s",
len(batch_to_send),

80
litellm/integrations/prometheus.py
View file

@ -4,6 +4,7 @@
from __future__ import annotations
import asyncio
import math
import os
import sys
from datetime import datetime, timedelta
@ -65,6 +66,26 @@ if TYPE_CHECKING:
else:
AsyncIOScheduler = Any
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
def _get_budget_metrics_per_request_timeout() -> float:
raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
if raw is None:
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
try:
parsed = float(raw)
except ValueError:
parsed = None
if parsed is None or not math.isfinite(parsed) or parsed <= 0:
verbose_logger.debug(
"[Non-Blocking] Prometheus: invalid PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT=%r; using default %ss.",
raw,
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT,
)
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
return parsed
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -1607,7 +1628,15 @@ class PrometheusLogger(CustomLogger):
_user_spend = _metadata.get("user_api_key_user_spend", None)
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
results = await asyncio.gather(
# Bound the per-request budget-metric emission so that slow Redis/DB
# lookups under load cannot consume the whole LoggingWorker watchdog
# (LOGGING_WORKER_MAX_TIME_PER_COROUTINE, default 20s) and get the entire
# success-logging event cancelled. Budget gauges are also refreshed by the
# periodic cron every PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES,
# so dropping one slow per-request emission only loses sub-cron real-time
# detail, not correctness.
budget_metrics_timeout = _get_budget_metrics_per_request_timeout()
gather_coro = asyncio.gather(
self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
@ -1634,6 +1663,16 @@ class PrometheusLogger(CustomLogger):
),
return_exceptions=True,
)
try:
results = await asyncio.wait_for(gather_coro, timeout=budget_metrics_timeout)
except asyncio.TimeoutError:
verbose_logger.debug(
"[Non-Blocking] Prometheus: per-request budget metric emission "
"exceeded %ss under load; skipping (values are refreshed by the "
"periodic budget-metrics cron job).",
budget_metrics_timeout,
)
return
for i, r in enumerate(results):
if isinstance(r, Exception):
verbose_logger.debug(
@ -2004,6 +2043,43 @@ class PrometheusLogger(CustomLogger):
return False
@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> Optional[str]:
"""
Best-effort provider for the client-side failure path.
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
"""
litellm_params = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
model = litellm_params.get("model") or request_data.get("model")
if not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
except litellm.exceptions.BadRequestError:
return None
except Exception as e: # noqa: BLE001 - metrics labeling must never break request/failure handling
verbose_logger.debug(
"prometheus: unexpected error inferring api_provider from model=%s: %s",
model,
e,
)
return None
async def async_post_call_failure_hook(
self,
request_data: dict,
@ -2039,6 +2115,7 @@ class PrometheusLogger(CustomLogger):
_metadata = request_data.get("metadata", {}) or {}
model_id = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider = self._extract_api_provider_from_request_data(request_data)
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
@ -2060,6 +2137,7 @@ class PrometheusLogger(CustomLogger):
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
api_provider=api_provider,
stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None),
)
_label_ctx = PrometheusLabelFactoryContext(enum_values)

25
litellm/integrations/s3_v2.py
View file

@ -54,6 +54,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: Optional[str] = None,
s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
@ -92,6 +93,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files=s3_strip_base64_files,
s3_use_key_prefix=s3_use_key_prefix,
s3_use_virtual_hosted_style=s3_use_virtual_hosted_style,
s3_server_side_encryption=s3_server_side_encryption,
)
verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}")
@ -145,6 +147,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: Optional[str] = None,
params_source: Optional[dict] = None,
):
"""
@ -194,6 +197,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style
)
self.s3_server_side_encryption = params.get("s3_server_side_encryption") or s3_server_side_encryption
return
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
@ -273,6 +278,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
try:
import base64
import hashlib
import requests
@ -317,14 +323,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# Calculate SHA256 hash of the content
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
content_md5 = base64.b64encode(
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
).decode()
# Prepare the request
headers = {
"Content-Type": "application/json",
"Content-MD5": content_md5,
"x-amz-content-sha256": content_hash,
"Content-Language": "en",
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**(
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
if self.s3_server_side_encryption
else {}
),
}
req = requests.Request("PUT", url, data=json_string, headers=headers)
prepped = req.prepare()
@ -447,6 +462,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
try:
import base64
import hashlib
import requests
@ -482,14 +498,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# Calculate SHA256 hash of the content
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
content_md5 = base64.b64encode(
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
).decode()
# Prepare the request
headers = {
"Content-Type": "application/json",
"Content-MD5": content_md5,
"x-amz-content-sha256": content_hash,
"Content-Language": "en",
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**(
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
if self.s3_server_side_encryption
else {}
),
}
req = requests.Request("PUT", url, data=json_string, headers=headers)
prepped = req.prepare()

157
litellm/integrations/websearch_interception/handler.py
View file

@ -91,6 +91,7 @@ class WebSearchInterceptionLogger(CustomLogger):
messages: List[Dict],
tools: Optional[List[Dict]],
custom_llm_provider: Optional[str],
kwargs: Optional[dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""
Short-circuit web-search-only requests by executing the search directly.
@ -176,7 +177,10 @@ class WebSearchInterceptionLogger(CustomLogger):
# Execute search — keep the structured SearchResponse so the native
# block can carry per-result url/title/page_age.
try:
search_result_text, structured = await self._execute_search(query)
if kwargs is None:
search_result_text, structured = await self._execute_search(query)
else:
search_result_text, structured = await self._execute_search(query, kwargs=kwargs)
except Exception as e:
verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}")
search_result_text, structured = f"Search failed: {e}", None
@ -936,7 +940,7 @@ class WebSearchInterceptionLogger(CustomLogger):
query = tool_call["input"].get("query")
if query:
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
search_tasks.append(self._execute_search(query))
search_tasks.append(self._execute_search(query, kwargs=kwargs))
else:
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query")
# Add empty result for tools without query
@ -1009,7 +1013,9 @@ class WebSearchInterceptionLogger(CustomLogger):
)
return patch, structured_results
async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
async def _execute_search(
self, query: str, kwargs: Optional[dict[str, Any]] = None
) -> Tuple[str, Optional[SearchResponse]]:
"""
Execute a single web search using router's search tools.
@ -1031,36 +1037,13 @@ class WebSearchInterceptionLogger(CustomLogger):
)
llm_router = None
# Determine search provider from router's search_tools
search_tool = self._select_search_tool_from_router(llm_router=llm_router)
search_provider: Optional[str] = None
if llm_router is not None and hasattr(llm_router, "search_tools"):
if self.search_tool_name:
# Find specific search tool by name
matching_tools = [
tool
for tool in llm_router.search_tools
if tool.get("search_tool_name") == self.search_tool_name
]
if matching_tools:
search_tool = matching_tools[0]
search_provider = search_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"with provider '{search_provider}'"
)
else:
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
"falling back to first available or perplexity"
)
# If no specific tool or not found, use first available
if not search_provider and llm_router.search_tools:
first_tool = llm_router.search_tools[0]
search_provider = first_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
)
search_litellm_params: dict[str, Any] = {}
if search_tool is not None:
await self._authorize_search_tool(search_tool=search_tool, kwargs=kwargs)
search_litellm_params = dict(search_tool.get("litellm_params", {}) or {})
search_provider = search_litellm_params.get("search_provider")
# Fallback to perplexity if no router or no search tools configured
if not search_provider:
@ -1073,7 +1056,12 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
)
result = await litellm.asearch(query=query, search_provider=search_provider)
search_kwargs = {
key: value
for key, value in search_litellm_params.items()
if key != "search_provider" and value is not None
}
result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs)
# Format using transformation function
search_result_text = WebSearchTransformation.format_search_response(result)
@ -1086,6 +1074,107 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}")
raise
async def _authorize_search_tool(
self,
search_tool: dict[str, Any],
kwargs: Optional[dict[str, Any]],
) -> None:
search_tool_name = search_tool.get("search_tool_name")
if not isinstance(search_tool_name, str) or not search_tool_name:
return
user_api_key_auth = self._get_user_api_key_auth_from_kwargs(kwargs)
if user_api_key_auth is None:
return
from litellm.proxy.auth.auth_checks import (
can_key_call_search_tool,
can_team_call_search_tool,
get_team_object,
)
await can_key_call_search_tool(
search_tool_name=search_tool_name,
valid_token=user_api_key_auth,
)
team_id = getattr(user_api_key_auth, "team_id", None)
if team_id:
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
team_object = await get_team_object(
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None),
proxy_logging_obj=proxy_logging_obj,
)
await can_team_call_search_tool(
search_tool_name=search_tool_name,
team_object=team_object,
)
@staticmethod
def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any:
if not kwargs:
return None
for metadata_key in ("metadata", "litellm_metadata"):
metadata = kwargs.get(metadata_key)
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
return metadata["user_api_key_auth"]
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
return None
for metadata_key in ("metadata", "litellm_metadata"):
metadata = litellm_params.get(metadata_key)
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
return metadata["user_api_key_auth"]
return None
def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]:
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools = list(getattr(llm_router, "search_tools") or [])
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
def _select_search_tool_from_list(
self,
search_tools: list[dict[str, Any]],
source: str,
) -> Optional[dict[str, Any]]:
if self.search_tool_name:
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
if matching_tools:
search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"from {source} with provider '{search_provider}'"
)
return matching_tools[0]
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, "
"falling back to first available or perplexity"
)
if search_tools:
first_tool = search_tools[0]
search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool from {source} "
f"with provider '{search_provider}'"
)
return first_tool
return None
async def _execute_chat_completion_agentic_loop(
self,
model: str,
@ -1145,7 +1234,7 @@ class WebSearchInterceptionLogger(CustomLogger):
if query:
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
search_tasks.append(self._execute_search(query))
search_tasks.append(self._execute_search(query, kwargs=kwargs))
else:
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query")
# Add empty result for tools without query

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -652,6 +652,10 @@ def _get_openai_compatible_provider_info(
api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" # type: ignore
dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY")
elif custom_llm_provider == "tencent":
api_base = api_base or get_secret("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1"
dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY")
elif custom_llm_provider == "fireworks_ai":
# fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
(

2
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -106,6 +106,8 @@ def get_supported_openai_params(
return litellm.VLLMConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "deepseek":
return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "tencent":
return litellm.TencentChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
return litellm.CohereChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "maritalk":

29
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,7 +1,23 @@
from typing import Dict, Optional
from typing import Any, Dict, Iterator, Optional
from litellm.types.utils import StandardCallbackDynamicParams
_CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata")
def iter_client_callback_metadata_dicts(
kwargs: dict[str, Any],
) -> Iterator[tuple[str, dict[str, Any]]]:
litellm_params = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):
nested = litellm_params.get("metadata")
if isinstance(nested, dict):
yield "litellm_params.metadata", nested
for key in _CLIENT_CALLBACK_METADATA_SLOTS:
candidate = kwargs.get(key)
if isinstance(candidate, dict):
yield key, candidate
def _is_env_reference(value: object) -> bool:
return isinstance(value, str) and "os.environ/" in value
@ -55,6 +71,7 @@ _supported_callback_params = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"turn_off_message_logging",
]
_request_blocked_callback_params = {
@ -87,19 +104,13 @@ def initialize_standard_callback_dynamic_params(
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
metadata = (kwargs.get("metadata") or {}).copy()
litellm_params = kwargs.get("litellm_params") or {}
if isinstance(litellm_params, dict):
metadata.update(litellm_params.get("metadata") or {})
if isinstance(metadata, dict):
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
if param in _request_blocked_callback_params:
continue
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source="metadata")
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

9
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4722,7 +4722,7 @@ class StandardLoggingPayloadSetup:
api_base: Optional[str] = None,
) -> StandardLoggingModelInformation:
model_cost_name = _select_model_name_for_cost_calc(
model=None,
model=base_model if custom_pricing else None,
completion_response=init_response_obj, # type: ignore
base_model=base_model,
custom_pricing=custom_pricing,
@ -5268,6 +5268,11 @@ def get_standard_logging_object_payload(
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
raw_response_cost = kwargs.get("response_cost")
response_cost: float = raw_response_cost or 0.0
@ -5389,7 +5394,7 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
print(json.dumps(payload, indent=4)) # noqa: T201
print(json.dumps(payload, indent=4), flush=True) # noqa: T201
def get_standard_logging_metadata(

2
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -656,7 +656,7 @@ def convert_to_model_response_object(
message: Optional[Message] = None
finish_reason: Optional[str] = None
if _should_convert_tool_call_to_json_mode(
if tool_calls is not None and _should_convert_tool_call_to_json_mode(
tool_calls=tool_calls,
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
):

6
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -72,6 +72,9 @@ def _process_image_response(response: Response, url: str) -> str:
async def async_convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
@ -95,6 +98,9 @@ async def async_convert_url_to_base64(url: str) -> str:
def convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(

18
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -131,8 +131,26 @@ class SensitiveDataMasker:
return masked_data
def mask(self, data: object) -> object:
if isinstance(data, Mapping):
return self.mask_dict(dict(data))
if isinstance(data, list):
return self._mask_sequence(
data,
0,
DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
None,
False,
)
return data
_default_masker = SensitiveDataMasker()
_error_masker = SensitiveDataMasker(visible_prefix=4, visible_suffix=0)
def mask_sensitive_structure(data: object) -> object:
return _error_masker.mask(data)
def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]:

48
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -7,6 +7,7 @@ from litellm.types.llms.openai import (
ChatCompletionAudioDelta,
)
from litellm.types.utils import (
CacheCreationTokenDetails,
ChatCompletionAudioResponse,
ChatCompletionMessageToolCall,
Choices,
@ -541,6 +542,12 @@ class ChunkProcessor:
web_search_requests: Optional[int] = None
completion_tokens_details: Optional[CompletionTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
# Anthropic emits the cache-creation TTL breakdown (5m/1h split) only on
# the `message_start` event; the later `message_delta` carries the flat
# cache-creation count but drops the nested breakdown. prompt_tokens_details
# is last-wins, so without preserving this separately the 1h breakdown is
# lost and 1h cache writes get billed at the 5m rate.
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
for chunk in chunks:
usage_chunk: Optional[Usage] = None
if "usage" in chunk:
@ -594,7 +601,18 @@ class ChunkProcessor:
"web_search_requests",
)
prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"]
prompt_tokens_details = cast(
Optional[PromptTokensDetailsWrapper],
usage_chunk_dict["prompt_tokens_details"],
)
cache_creation_token_details = self._capture_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
prompt_tokens_details = self._attach_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
completion_tokens = self._reset_anthropic_cursor_completion_tokens(
chunks=chunks,
@ -613,6 +631,34 @@ class ChunkProcessor:
prompt_tokens_details=prompt_tokens_details,
)
@staticmethod
def _capture_cache_creation_token_details(
prompt_tokens_details: Optional[PromptTokensDetailsWrapper],
current: Optional[CacheCreationTokenDetails],
) -> Optional[CacheCreationTokenDetails]:
incoming = cast(
Optional[CacheCreationTokenDetails],
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if incoming is not None:
return incoming
return current
@staticmethod
def _attach_cache_creation_token_details(
prompt_tokens_details: Optional[PromptTokensDetailsWrapper],
cache_creation_token_details: Optional[CacheCreationTokenDetails],
) -> Optional[PromptTokensDetailsWrapper]:
if prompt_tokens_details is None or cache_creation_token_details is None:
return prompt_tokens_details
existing = cast(
Optional[CacheCreationTokenDetails],
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if existing is not None:
return prompt_tokens_details
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
@staticmethod
def _reset_anthropic_cursor_completion_tokens(
chunks: list[dict[str, Any] | ModelResponse],

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1884,7 +1884,7 @@ class CustomStreamWrapper:
await self.fetch_stream()
if is_async_iterable(self.completion_stream):
async for chunk in self.completion_stream: # type: ignore[union-attr]
async for chunk in self.completion_stream: # pyright: ignore[reportOptionalIterable] # is_async_iterable guard proves __aiter__
if chunk == "None" or chunk is None:
continue # skip None chunks

21
litellm/llms/a2a/chat/transformation.py
View file

@ -10,7 +10,7 @@ import httpx
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from ..common_utils import (
A2AError,
@ -312,6 +312,25 @@ class A2AConfig(BaseConfig):
# Set ID from response
model_response.id = response_json.get("id", str(uuid.uuid4()))
# A2A agents don't return token usage; estimate it so per-token pricing
# produces real cost and callers don't receive usage of 0/0/0.
try:
from litellm.utils import token_counter
prompt_tokens = token_counter(model="gpt-3.5-turbo", messages=messages)
completion_tokens = token_counter(model="gpt-3.5-turbo", text=text, count_response_tokens=True)
setattr(
model_response,
"usage",
Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
),
)
except Exception: # noqa: BLE001 - best-effort estimate; a tokenizer hiccup must not break the response
pass
return model_response
def get_model_response_iterator(

199
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -48,7 +48,10 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -70,6 +73,170 @@ class AnthropicMessagesHandler(BaseTranslation):
super().__init__()
self.adapter = LiteLLMAnthropicMessagesAdapter()
@staticmethod
def _build_streaming_usage_response(
responses_so_far: list[Any],
request_data: Optional[dict],
) -> Optional[ModelResponse]:
chunks = tuple(response for response in responses_so_far if isinstance(response, (str, bytes)))
if not chunks:
return None
try:
return AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks(
all_chunks=chunks,
model=str((request_data or {}).get("model") or ""),
)
except (AttributeError, TypeError, ValueError):
return None
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Optional[list[Any]] = None,
) -> list[bytes]:
"""
Build an Anthropic SSE sequence delivering the guardrail block message
and terminating the stream cleanly.
- ``stream_started`` False (buffered / pre-stream): nothing has been
sent, so emit a complete standalone message (message_start ->
content_block_* -> message_delta -> message_stop) via
FakeAnthropicMessagesStreamIterator, the same converter the
/v1/messages pre-stream block handler uses.
- ``stream_started`` True (sampling / detect-only end-of-stream): real
chunks were already sent, so *continue* the in-progress message --
close the open content block, append the block message as a new text
block, then end the message. Emitting a second ``message_start`` here
would make Anthropic clients reject the stream.
"""
if stream_started:
return self._block_continuation_chunks(exc, responses_so_far or [])
return self._standalone_block_chunks(exc)
def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
import uuid
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage,
)
from litellm.types.utils import AnthropicMessagesResponse
block_response = AnthropicMessagesResponse(
id=f"msg_{uuid.uuid4()}",
type="message",
role="assistant",
content=[{"type": "text", "text": exc.message}],
model=exc.model,
stop_reason="end_turn",
usage=blocked_response_usage(getattr(exc, "original_response", None)),
)
return list(FakeAnthropicMessagesStreamIterator(response=block_response))
def _block_continuation_chunks(self, exc: "ModifyResponseException", responses_so_far: list[Any]) -> list[bytes]:
"""Continue an already-started message: close the open content block,
append the block message as a new text block, then end the message --
without a second message_start."""
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage,
)
def _sse(event_type: str, payload: dict) -> bytes:
return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
output_tokens = blocked_response_usage(getattr(exc, "original_response", None))["output_tokens"]
open_index, max_index = self._content_block_state(responses_so_far)
new_index = (max_index + 1) if max_index is not None else 0
chunks: list[bytes] = []
if open_index is not None:
chunks.append(_sse("content_block_stop", {"type": "content_block_stop", "index": open_index}))
chunks += [
_sse(
"content_block_start",
{
"type": "content_block_start",
"index": new_index,
"content_block": {"type": "text", "text": ""},
},
),
_sse(
"content_block_delta",
{
"type": "content_block_delta",
"index": new_index,
"delta": {"type": "text_delta", "text": exc.message},
},
),
_sse("content_block_stop", {"type": "content_block_stop", "index": new_index}),
_sse(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": output_tokens},
},
),
_sse("message_stop", {"type": "message_stop"}),
]
return chunks
@staticmethod
def _content_block_state(
responses_so_far: list[Any],
) -> tuple[Optional[int], Optional[int]]:
"""From the SSE chunks already sent to the client, return (open
content-block index or None, highest content-block index seen or None).
A single streamed item may bundle multiple SSE events (raw bytes) or be
an already-parsed event dict, so every event across every item is
considered -- matching how ``get_streaming_string_so_far`` reads the
same stream."""
open_indices: set[int] = set()
max_index: Optional[int] = None
for item in responses_so_far:
for data in AnthropicMessagesHandler._iter_sse_events(item):
event_type = data.get("type")
index = data.get("index")
if not isinstance(index, int):
continue
if event_type == "content_block_start":
open_indices.add(index)
max_index = index if max_index is None else max(max_index, index)
elif event_type == "content_block_stop":
open_indices.discard(index)
open_index = max(open_indices) if open_indices else None
return open_index, max_index
@staticmethod
def _iter_sse_events(item: Any) -> list[dict]:
"""Yield the event-data dicts in one stream chunk.
Handles both formats this stream can carry (see
``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle
several events separated by a blank line -- and an already-parsed event
``dict``."""
if isinstance(item, dict):
return [item]
if not isinstance(item, (bytes, bytearray)):
return []
events: list[dict] = []
for block in item.decode("utf-8", errors="replace").split("\n\n"):
for line in block.split("\n"):
line = line.strip()
if not line.startswith("data:"):
continue
try:
parsed = json.loads(line[len("data:") :].strip())
except json.JSONDecodeError:
continue
if isinstance(parsed, dict):
events.append(parsed)
return events
def _translate_to_openai(self, data: dict) -> ChatCompletionRequest:
"""Translate Anthropic request to OpenAI chat completion format."""
(
@ -406,6 +573,8 @@ class AnthropicMessagesHandler(BaseTranslation):
Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
"""
from litellm.integrations.custom_guardrail import ModifyResponseException
has_ended = self._check_streaming_has_ended(responses_so_far)
if has_ended:
# build the model response from the responses_so_far
@ -430,25 +599,35 @@ class AnthropicMessagesHandler(BaseTranslation):
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
_guardrailed_inputs = (
await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
try:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=guardrail_inputs,
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
)
except ModifyResponseException as e:
if e.original_response is None:
e.original_response = built_response or self._build_streaming_usage_response(
responses_so_far, request_data
)
raise
else:
verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices")
return responses_so_far
string_so_far = self.get_streaming_string_so_far(responses_so_far)
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
inputs={"texts": [string_so_far]},
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
try:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs={"texts": [string_so_far]},
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
except ModifyResponseException as e:
if e.original_response is None:
e.original_response = self._build_streaming_usage_response(responses_so_far, request_data)
raise
return responses_so_far
def _prepare_request_data(

65
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -78,7 +78,7 @@ async def _prepare_context_managed_request(
system: Optional[Any],
context_management_spec: Any,
litellm_metadata: Optional[Dict],
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
llm_router: Any,
user_api_key_auth: Any = None,
) -> Optional[PolyfillResult]:
@ -95,7 +95,7 @@ async def _prepare_context_managed_request(
# silently drop intermediate turns.
polyfill_will_run = _polyfill_will_run(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if polyfill_will_run:
@ -117,7 +117,7 @@ async def _prepare_context_managed_request(
system=working_system,
context_management_spec=context_management_spec,
litellm_metadata=litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=llm_router,
user_api_key_auth=user_api_key_auth,
)
@ -143,18 +143,19 @@ async def _prepare_context_managed_request(
def _polyfill_will_run(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> bool:
"""Return True when ``compact_20260112`` will run via the polyfill dispatcher.
Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or
effective ``drop_params`` short-circuits the polyfill. The pre-processing
skip only applies when the dispatcher will actually invoke
``apply_compact_20260112`` (which has its own compaction-block slicing).
Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or an
explicit ``context_management`` entry in ``additional_drop_params``
short-circuits the polyfill. The pre-processing skip only applies when the
dispatcher will actually invoke ``apply_compact_20260112`` (which has its
own compaction-block slicing).
"""
edits = _normalize_spec_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if edits is None:
return False
@ -169,7 +170,7 @@ def _polyfill_will_run(
def _spec_has_non_compact_edits(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> bool:
"""Return True when the spec includes edits other than ``compact_20260112``.
@ -180,7 +181,7 @@ def _spec_has_non_compact_edits(
"""
edits = _normalize_spec_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if edits is None:
return False
@ -195,10 +196,22 @@ def _spec_has_non_compact_edits(
)
def _context_management_explicitly_dropped(additional_drop_params: Optional[list[str]]) -> bool:
"""True when the caller opted out of context_management via ``additional_drop_params``.
``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
``drop_params`` only exists to drop genuinely unsupported params.
"""
if not isinstance(additional_drop_params, list):
return False
return "context_management" in additional_drop_params
def _normalize_spec_edits(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> Optional[List[Dict[str, Any]]]:
"""Return the normalized ``edits`` list, or ``None`` if the polyfill won't run.
@ -208,8 +221,7 @@ def _normalize_spec_edits(
if not context_management_spec:
return None
effective_drop_params = drop_params if drop_params is not None else litellm.drop_params
if effective_drop_params:
if _context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.experimental_pass_through.context_management.dispatcher import (
@ -230,22 +242,23 @@ async def _run_polyfill_if_enabled(
system: Optional[Any],
context_management_spec: Any,
litellm_metadata: Optional[Dict],
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
llm_router: Any,
user_api_key_auth: Any = None,
) -> Optional[PolyfillResult]:
"""Run the async context_management polyfill if a spec is present.
Returns ``None`` when the spec is empty or drop_params is on. Raises
``AnthropicContextManagementError`` so the /v1/messages endpoint can
emit an Anthropic-format 400. All other exceptions are best-effort
swallowed (matches v0 behavior).
Returns ``None`` when the spec is empty or ``context_management`` is
listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
does not disable the polyfill because context_management is a supported
param). Raises ``AnthropicContextManagementError`` so the /v1/messages
endpoint can emit an Anthropic-format 400. All other exceptions are
best-effort swallowed (matches v0 behavior).
"""
if not context_management_spec:
return None
effective_drop_params = drop_params if drop_params is not None else litellm.drop_params
if effective_drop_params:
if _context_management_explicitly_dropped(additional_drop_params):
return None
try:
@ -274,7 +287,7 @@ async def _run_polyfill_if_enabled(
# emits an Anthropic-format error.
if _spec_has_non_compact_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
):
raise AnthropicContextManagementError(
status_code=500,
@ -533,7 +546,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
) -> Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]:
"""Handle non-Anthropic models asynchronously using the adapter"""
context_management = kwargs.pop("context_management", None)
drop_params: Optional[bool] = kwargs.get("drop_params", None)
additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None)
litellm_router = kwargs.pop("litellm_router", None)
if litellm_router is None:
try:
@ -555,7 +568,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
system=system,
context_management_spec=context_management,
litellm_metadata=proxy_litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=litellm_router,
user_api_key_auth=user_api_key_auth,
)
@ -661,7 +674,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# ``compact_20260112`` editor can ``await`` the summarization model);
# bridge to it via ``run_async_function``.
context_management = kwargs.pop("context_management", None)
drop_params: Optional[bool] = kwargs.get("drop_params", None)
additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None)
# Deliberately do NOT auto-attach the proxy ``llm_router`` here:
# ``run_async_function`` spawns a new event loop in a worker thread
# to bridge to the async dispatcher, but the proxy router's httpx
@ -696,7 +709,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
system=system,
context_management_spec=context_management,
litellm_metadata=proxy_litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=litellm_router,
user_api_key_auth=user_api_key_auth,
)

8
litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
View file

@ -193,6 +193,14 @@ class AgenticAnthropicStreamingIterator:
raise StopAsyncIteration
async def aclose(self) -> None:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
aclose_if_supported,
)
await aclose_if_supported(self._inner)
await aclose_if_supported(self._follow_up_iterator)
async def _process_agentic_hooks(self) -> None:
"""Rebuild the Anthropic response from collected SSE bytes and call hooks."""
if self._hook_processing_done:

3
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -148,6 +148,7 @@ async def _try_websearch_short_circuit(
tools: Optional[List[Dict]],
custom_llm_provider: Optional[str],
stream: Optional[bool],
kwargs: Optional[dict] = None,
) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
"""
Attempt to short-circuit a web-search-only request.
@ -177,6 +178,7 @@ async def _try_websearch_short_circuit(
messages=messages,
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
if response is not None:
anthropic_response = cast(AnthropicMessagesResponse, response)
@ -292,6 +294,7 @@ async def anthropic_messages(
tools=tools,
custom_llm_provider=custom_llm_provider,
stream=original_stream,
kwargs={**kwargs, "metadata": metadata},
)
if short_circuit_response is not None:
return short_circuit_response

56
litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
View file

@ -17,7 +17,9 @@ How it works:
import uuid
from typing import Any, AsyncIterator, Dict, List, Optional, Union
import litellm
import litellm.constants as _c
from litellm.litellm_core_utils.url_utils import validate_url
from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -76,16 +78,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
raise ValueError("advisor tool definition must include a 'model' field specifying the advisor model")
_raw_max_uses = advisor_tool.get("max_uses")
max_uses: int = ADVISOR_MAX_USES if _raw_max_uses is None else int(_raw_max_uses)
# Optional routing overrides for the advisor sub-call (e.g. proxy routing).
# If not set in the tool definition, litellm resolves from env vars.
# The advisor tool is caller-controlled; only honor a client-supplied
# api_base/api_key when the proxy has enabled clientside credentials,
# otherwise let litellm resolve from server config.
advisor_api_key: Optional[str] = None
advisor_api_base: Optional[str] = None
if _allow_client_side_advisor_credentials():
advisor_api_key = advisor_tool.get("api_key")
advisor_api_base = advisor_tool.get("api_base")
advisor_api_key, advisor_api_base = _resolve_advisor_credentials(advisor_tool)
# Build the synthetic tool definition the provider will receive.
synthetic_advisor_tool = _make_synthetic_advisor_tool()
@ -186,6 +179,49 @@ def _allow_client_side_advisor_credentials() -> bool:
return general_settings.get("allow_client_side_credentials") is True
def _resolve_advisor_credentials(advisor_tool: dict) -> tuple[Optional[str], Optional[str]]:
"""Resolve the (api_key, api_base) override for the advisor sub-call.
A caller-supplied ``api_base`` is only honored alongside a caller-supplied
``api_key``: without one, ``AnthropicModelInfo.get_auth_header()`` falls
back to the proxy's own Anthropic credentials, which would then be sent to
the caller-chosen ``api_base``. A caller-supplied ``api_base`` is also
required to be https with TLS verification on, and SSRF-validated so it
can't target a private/internal/cloud-metadata address, mirroring
``proxy.auth.auth_utils.check_complete_credentials``. https with TLS
verification is required because ``validate_url`` only rewrites the
connection to a DNS-pinned IP for http, or for https with
``litellm.ssl_verify`` disabled; otherwise it returns the URL unchanged
and relies on certificate validation to block DNS rebinding, so this
closes the same gap without threading the pinned URL through the whole
``anthropic_messages()`` call chain.
"""
if not _allow_client_side_advisor_credentials():
return None, None
api_key: Optional[str] = advisor_tool.get("api_key")
api_base: Optional[str] = advisor_tool.get("api_base")
if api_base is None:
return api_key, None
if not api_key:
raise ValueError(
"advisor tool definition sets 'api_base' without 'api_key'. A "
"caller-supplied api_base is only honored alongside a "
"caller-supplied api_key, so the proxy's own credentials are "
"never sent to a caller-chosen destination."
)
if not api_base.startswith("https://"):
raise ValueError(f"advisor tool definition sets 'api_base'={api_base!r}, which must use the https scheme.")
if getattr(litellm, "ssl_verify", True) is False:
raise ValueError(
"advisor tool definition sets 'api_base' but the proxy has TLS verification "
"disabled (litellm.ssl_verify=False), so a caller-supplied api_base can't be "
"safely validated against DNS rebinding."
)
if getattr(litellm, "user_url_validation", True):
validate_url(api_base)
return api_key, api_base
def _make_synthetic_advisor_tool() -> Dict:
"""Build a regular tool definition the executor provider can understand."""
return {

99
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -1,8 +1,13 @@
import asyncio
import json
from datetime import datetime
from typing import Any, AsyncIterator, List, Union
from typing import Any, AsyncIterator, List, Protocol, Union, runtime_checkable
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
@ -12,6 +17,93 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ = PassThroughEndpointLogging()
INCOMPLETE_STREAM_ERROR_MESSAGE = (
"Provider stream ended before emitting a message_stop event; "
"the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."
)
def _is_message_stop_chunk(chunk: object) -> bool:
if isinstance(chunk, dict):
return chunk.get("type") == "message_stop"
if isinstance(chunk, (bytes, bytearray)):
return any(line == b"event: message_stop" for line in chunk.splitlines())
return False
def _is_provider_error_chunk(chunk: object) -> bool:
if isinstance(chunk, dict):
return chunk.get("type") == "error"
if isinstance(chunk, (bytes, bytearray)):
return any(line == b"event: error" for line in chunk.splitlines())
return False
def _is_terminal_stream_chunk(chunk: object) -> bool:
return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
def _incomplete_stream_error_sse_event() -> bytes:
payload = json.dumps(
{
"type": "error",
"error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
}
)
return f"event: error\ndata: {payload}\n\n".encode()
class AnthropicMessagesStreamHiddenParams(TypedDict):
additional_headers: dict[str, str]
@runtime_checkable
class SupportsAclose(Protocol):
async def aclose(self) -> None: ...
async def aclose_if_supported(stream: object) -> None:
if isinstance(stream, SupportsAclose):
await stream.aclose()
_RESPONSE_HEADERS_ADAPTER: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
def anthropic_messages_stream_hidden_params(
response_headers: httpx.Headers,
) -> AnthropicMessagesStreamHiddenParams:
return AnthropicMessagesStreamHiddenParams(
additional_headers=_RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers))
)
class AnthropicMessagesStreamingResponse:
"""
Wraps the /v1/messages SSE byte stream so upstream provider response
headers (e.g. Bedrock's x-amzn-requestid / x-amzn-trace-id) survive as
``_hidden_params["additional_headers"]``, which the proxy forwards to
clients as ``llm_provider-*`` response headers. Bare async generators
cannot carry attributes, so header context was previously dropped.
"""
def __init__(
self,
completion_stream: AsyncIterator[bytes],
hidden_params: AnthropicMessagesStreamHiddenParams,
) -> None:
self.completion_stream = completion_stream
self._hidden_params = hidden_params
def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
return self
async def __anext__(self) -> bytes:
return await self.completion_stream.__anext__()
async def aclose(self) -> None:
await aclose_if_supported(self.completion_stream)
class BaseAnthropicMessagesStreamingIterator:
"""
@ -102,13 +194,18 @@ class BaseAnthropicMessagesStreamingIterator:
This method provides the common logic for both Anthropic and Bedrock implementations.
"""
collected_chunks = []
saw_terminal_event = False
async for chunk in completion_stream:
if self.completion_start_time is None:
self.completion_start_time = datetime.now()
saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk)
encoded_chunk = self._convert_chunk_to_sse_format(chunk)
collected_chunks.append(encoded_chunk)
yield encoded_chunk
if not saw_terminal_event:
yield _incomplete_stream_error_sse_event()
# Handle logging after all chunks are processed
await self._handle_streaming_logging(collected_chunks)

356
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -15,6 +15,7 @@ from typing import Any, Dict
from urllib.parse import quote
import httpx
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
@ -38,6 +39,30 @@ from litellm.secret_managers.main import get_secret_str
AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"
class AzureDocumentIntelligenceLine(BaseModel):
content: str | None = None
class AzureDocumentIntelligencePage(BaseModel):
pageNumber: int | None = None
width: float | None = None
height: float | None = None
unit: str | None = None
lines: tuple[AzureDocumentIntelligenceLine, ...] = ()
class AzureDocumentIntelligenceAnalyzeResult(BaseModel):
content: str | None = None
pages: tuple[AzureDocumentIntelligencePage, ...] = ()
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
class AzureDocumentIntelligenceOperation(BaseModel):
status: str | None = None
analyzeResult: AzureDocumentIntelligenceAnalyzeResult | None = None
class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"""
Azure Document Intelligence OCR transformation configuration.
@ -67,11 +92,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
(1-based, e.g. "1-3,5,7-9"). To keep the public request shape
aligned with Mistral OCR, callers pass `pages` using Mistral
semantics — a list of 0-based integers — or a pre-formatted
Azure-style string. Other Mistral-specific params (e.g.
Azure-style string. Azure DI also exposes a `features` query
parameter enabling add-on capabilities (e.g. "keyValuePairs",
"languages"), passed as a list of feature names or a
comma-separated string. Other Mistral-specific params (e.g.
`include_image_base64`) are not supported by Azure DI and are
ignored during transformation.
"""
return ["pages"]
return ["pages", "features"]
def map_ocr_params(
self,
@ -85,16 +113,18 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Translates Mistral-style `pages` (list[int], 0-based) into Azure's
`pages` query string (1-based, e.g. "1,2,3" or "1-3,5"). A raw
string that already matches Azure's format is passed through
unchanged.
unchanged. `features` (list[str] or comma-separated string) is
normalized into Azure's comma-joined `features` query string.
"""
pages = non_default_params.get("pages")
if pages is None:
return optional_params
normalized = self._normalize_pages_param(pages)
if normalized:
optional_params["pages"] = normalized
return optional_params
features = non_default_params.get("features")
normalized_pages = self._normalize_pages_param(pages) if pages is not None else ""
normalized_features = self._normalize_features_param(features) if features is not None else ""
return {
**optional_params,
**({"pages": normalized_pages} if normalized_pages else {}),
**({"features": normalized_features} if normalized_features else {}),
}
@staticmethod
def _normalize_pages_param(pages: Any) -> str:
@ -140,6 +170,39 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
raise ValueError("`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'.")
@staticmethod
def _normalize_features_param(features: object) -> str:
"""
Convert a caller-provided `features` value to Azure DI's query-string
form (comma-joined feature names, e.g. "keyValuePairs,languages").
Accepted inputs:
- list[str]: feature names like ["keyValuePairs", "languages"].
- str: a single feature name or comma-separated names.
"""
invalid_features_error = ValueError(
f"Invalid `features` for Azure Document Intelligence: {features!r}. "
f"Expected a list of feature names or a comma-separated string like "
f"'keyValuePairs' or 'keyValuePairs,languages'."
)
if isinstance(features, str):
raw_tokens = features.split(",")
elif isinstance(features, list):
if len(features) == 0:
return ""
raw_tokens = [feature for feature in features if isinstance(feature, str)]
if len(raw_tokens) != len(features):
raise invalid_features_error
else:
raise invalid_features_error
tokens = tuple(token.strip() for token in raw_tokens)
feature_pattern = re.compile(r"^[A-Za-z][A-Za-z0-9]*$")
if not all(feature_pattern.match(token) for token in tokens):
raise invalid_features_error
return ",".join(tokens)
def validate_environment(
self,
headers: Dict,
@ -228,13 +291,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
)
# Azure DI accepts `pages` as a query param (1-based, e.g. "1-3,5").
# Azure DI accepts `pages` (1-based, e.g. "1-3,5") and `features`
# (comma-joined names, e.g. "keyValuePairs") as query params.
# `optional_params` has already been normalized in `map_ocr_params`.
pages = optional_params.get("pages") if optional_params else None
if pages:
url += f"&pages={quote(str(pages), safe=',-')}"
features = optional_params.get("features") if optional_params else None
pages_query = f"&pages={quote(str(pages), safe=',-')}" if pages else ""
features_query = f"&features={quote(str(features), safe=',')}" if features else ""
return url
return f"{url}{pages_query}{features_query}"
def _extract_base64_from_data_uri(self, data_uri: str) -> str:
"""
@ -328,27 +393,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
return OCRRequestData(data=data, files=None)
def _extract_page_markdown(self, page_data: Dict[str, Any]) -> str:
"""
Extract text from Azure DI page and format as markdown.
Azure DI provides text in 'lines' array. We concatenate them with newlines.
Args:
page_data: Azure DI page object
Returns:
Markdown-formatted text
"""
lines = page_data.get("lines", [])
if not lines:
return ""
# Extract text content from each line
text_lines = [line.get("content", "") for line in lines]
# Join with newlines to preserve structure
return "\n".join(text_lines)
def _transform_azure_page(self, azure_page: AzureDocumentIntelligencePage) -> OCRPage:
page_number = azure_page.pageNumber if azure_page.pageNumber is not None else 1
markdown = "\n".join(line.content or "" for line in azure_page.lines)
dimensions = self._convert_dimensions(
width=azure_page.width if azure_page.width is not None else 8.5,
height=azure_page.height if azure_page.height is not None else 11,
unit=azure_page.unit if azure_page.unit is not None else "inch",
)
return OCRPage(index=page_number - 1, markdown=markdown, dimensions=dimensions)
def _convert_dimensions(self, width: float, height: float, unit: str) -> OCRPageDimensions:
"""
@ -526,6 +579,52 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
retry_after = self._get_retry_after(response=response)
await asyncio.sleep(retry_after)
def _get_polling_target(self, raw_response: httpx.Response) -> tuple[str, Dict[str, str]]:
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
poll_headers = {"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")}
return operation_url, poll_headers
def _transform_completed_response(self, model: str, raw_response: httpx.Response) -> OCRResponse:
"""
Transform a completed Azure Document Intelligence analyze operation
into the Mistral OCR response shape, preserving Azure-native
`analyzeResult` fields (`content`, `tables`, `keyValuePairs`) as
top-level response fields.
"""
operation = AzureDocumentIntelligenceOperation.model_validate(raw_response.json())
verbose_logger.debug(f"Azure Document Intelligence response status: {operation.status}")
if operation.status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}")
analyze_result = (
operation.analyzeResult if operation.analyzeResult is not None else AzureDocumentIntelligenceAnalyzeResult()
)
mistral_pages = [self._transform_azure_page(azure_page) for azure_page in analyze_result.pages]
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
content=analyze_result.content,
tables=analyze_result.tables,
keyValuePairs=analyze_result.keyValuePairs,
)
def transform_ocr_response(
self,
model: str,
@ -552,11 +651,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"unit": "inch",
"lines": [{"content": "text", "boundingBox": [...]}]
}
]
],
"tables": [...],
"keyValuePairs": [...]
}
}
Mistral OCR format:
Mistral OCR format (with Azure-native fields preserved):
{
"pages": [
{
@ -567,7 +668,10 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
],
"model": "azure_ai/doc-intelligence/prebuilt-layout",
"usage_info": {"pages_processed": 1},
"object": "ocr"
"object": "ocr",
"content": "Full document text...",
"tables": [...],
"keyValuePairs": [...]
}
Args:
@ -578,86 +682,17 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Returns:
OCRResponse in Mistral format
"""
try:
# Check if we got 202 Accepted (async operation started)
if raw_response.status_code == 202:
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
if raw_response.status_code != 202:
return self._transform_completed_response(model=model, raw_response=raw_response)
# Get Operation-Location header
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")
}
# Get timeout from kwargs or use default
timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT
# Poll until operation completes
raw_response = self._poll_operation_sync(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=timeout_secs,
)
# Now parse the completed response
response_json = raw_response.json()
verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}")
# Check if request succeeded
status = response_json.get("status")
if status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}")
# Extract analyze result
analyze_result = response_json.get("analyzeResult", {})
azure_pages = analyze_result.get("pages", [])
# Transform pages to Mistral format
mistral_pages = []
for azure_page in azure_pages:
page_number = azure_page.get("pageNumber", 1)
index = page_number - 1 # Convert to 0-based index
# Extract markdown text
markdown = self._extract_page_markdown(azure_page)
# Convert dimensions
width = azure_page.get("width", 8.5)
height = azure_page.get("height", 11)
unit = azure_page.get("unit", "inch")
dimensions = self._convert_dimensions(width=width, height=height, unit=unit)
# Build OCR page
ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions)
mistral_pages.append(ocr_page)
# Build usage info
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
# Return Mistral OCR response
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error(f"Error parsing Azure Document Intelligence response: {e}")
raise e
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response = self._poll_operation_sync(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(model=model, raw_response=completed_response)
async def async_transform_ocr_response(
self,
@ -680,81 +715,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Returns:
OCRResponse in Mistral format
"""
try:
# Check if we got 202 Accepted (async operation started)
if raw_response.status_code == 202:
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
if raw_response.status_code != 202:
return self._transform_completed_response(model=model, raw_response=raw_response)
# Get Operation-Location header
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs (see sync path). VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")
}
# Get timeout from kwargs or use default
timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT
# Poll until operation completes (async)
raw_response = await self._poll_operation_async(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=timeout_secs,
)
# Now parse the completed response
response_json = raw_response.json()
verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}")
# Check if request succeeded
status = response_json.get("status")
if status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}")
# Extract analyze result
analyze_result = response_json.get("analyzeResult", {})
azure_pages = analyze_result.get("pages", [])
# Transform pages to Mistral format
mistral_pages = []
for azure_page in azure_pages:
page_number = azure_page.get("pageNumber", 1)
index = page_number - 1 # Convert to 0-based index
# Extract markdown text
markdown = self._extract_page_markdown(azure_page)
# Convert dimensions
width = azure_page.get("width", 8.5)
height = azure_page.get("height", 11)
unit = azure_page.get("unit", "inch")
dimensions = self._convert_dimensions(width=width, height=height, unit=unit)
# Build OCR page
ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions)
mistral_pages.append(ocr_page)
# Build usage info
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
# Return Mistral OCR response
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error(f"Error parsing Azure Document Intelligence response (async): {e}")
raise e
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response = await self._poll_operation_async(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(model=model, raw_response=completed_response)

29
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -2,7 +2,10 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, List, Optional
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import AllMessageValues
@ -98,6 +101,30 @@ class BaseTranslation(ABC):
"""
return responses_so_far
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Optional[list[Any]] = None,
) -> Optional[list[bytes]]:
"""
Build the streaming chunks that deliver a guardrail block message and
cleanly terminate the stream in this provider's wire format.
``stream_started`` is True when real chunks were already sent to the
client: the result must *continue* the in-progress message (e.g. close
the open content block and append the block message) rather than start
a new one, which clients reject. ``responses_so_far`` provides the prior
chunks needed to do so. When False, nothing has been sent and a
standalone block message is emitted.
Returns None when the format has no safe terminator; the caller then
re-raises ``exc`` so the proxy can surface a clean error instead.
Override in provider subclasses that support synthesizing a block
stream.
"""
return None
def get_structured_messages(self, data: dict) -> Optional[List["AllMessageValues"]]:
"""
Convert request data to OpenAI-spec structured messages.

92
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -1,10 +1,100 @@
from __future__ import annotations
from typing import Any, List
import json
from typing import Any, List, Optional
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues
def _anthropic_stream_chunk_events(item: Any) -> list[dict]:
if isinstance(item, dict):
return [item]
if isinstance(item, bytes):
chunk = item.decode("utf-8", errors="replace")
elif isinstance(item, str):
chunk = item
else:
return []
events: list[dict] = []
for block in chunk.split("\n\n"):
for line in block.splitlines():
stripped = line.strip()
if not stripped.startswith("data:"):
continue
payload = stripped[len("data:") :].strip()
if not payload or payload == "[DONE]":
continue
try:
parsed = json.loads(payload)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict):
events.append(parsed)
return events
def _usage_from_anthropic_stream_chunks(original_response: list[Any]) -> Optional[AnthropicUsage]:
input_tokens = 0
output_tokens = 0
found_usage = False
for item in original_response:
for event in _anthropic_stream_chunk_events(item):
event_type = event.get("type")
if event_type == "message_start":
message = event.get("message") or {}
usage_obj = message.get("usage") or {}
elif event_type == "message_delta":
usage_obj = event.get("usage") or {}
else:
usage_obj = {}
if not isinstance(usage_obj, dict):
continue
if usage_obj.get("input_tokens") is not None:
input_tokens = int(usage_obj.get("input_tokens") or 0)
found_usage = True
if usage_obj.get("output_tokens") is not None:
output_tokens = int(usage_obj.get("output_tokens") or 0)
found_usage = True
if not found_usage:
return None
return AnthropicUsage(input_tokens=input_tokens, output_tokens=output_tokens)
def blocked_response_usage(original_response: Optional[Any]) -> AnthropicUsage:
"""
Token usage for a synthetic guardrail-blocked response.
A post-call block replaces the LLM's response with the violation message,
but the upstream call already consumed tokens -- report that real usage
(carried on ``ModifyResponseException.original_response``) rather than
discarding it. Pre-call blocks never invoked the LLM (no original_response),
so usage is zero.
"""
usage_obj: Any = None
if isinstance(original_response, list):
stream_usage = _usage_from_anthropic_stream_chunks(original_response)
if stream_usage is not None:
return stream_usage
elif isinstance(original_response, dict):
usage_obj = original_response.get("usage")
elif original_response is not None:
usage_obj = getattr(original_response, "usage", None)
def _tokens(key: str, fallback_key: str) -> int:
if isinstance(usage_obj, dict):
return int(usage_obj.get(key, usage_obj.get(fallback_key, 0)) or 0)
return int(getattr(usage_obj, key, getattr(usage_obj, fallback_key, 0)) or 0)
return AnthropicUsage(
input_tokens=_tokens("input_tokens", "prompt_tokens"),
output_tokens=_tokens("output_tokens", "completion_tokens"),
)
def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None)
if per is not None:

3
litellm/llms/base_llm/ocr/transformation.py
View file

@ -70,6 +70,9 @@ class OCRResponse(LiteLLMPydanticObjectBase):
model: str
document_annotation: Any | None = None
usage_info: OCRUsageInfo | None = None
content: str | None = None
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
object: str = "ocr"
model_config = {"extra": "allow"}

28
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -76,6 +76,7 @@ from litellm.utils import (
from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_converse_supports_parallel_tool_use_config,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_claude_4_5_on_bedrock,
@ -1106,18 +1107,28 @@ class AmazonConverseConfig(BaseConfig):
if cache_control is None:
return None
cache_point = CachePointBlock(type="default")
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None:
if is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
cache_point = self._build_cache_point_block(cache_control, model)
if block_type == "system":
return SystemContentBlock(cachePoint=cache_point)
else:
return ContentBlock(cachePoint=cache_point)
@staticmethod
def _build_cache_point_block(control: Optional[dict], model: Optional[str] = None) -> CachePointBlock:
"""Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict.
``type`` is always ``"default"`` (the only value Bedrock's Converse API
accepts). ``ttl`` is only honored for models that support extended TTL
caching (Claude 4.5 family on Bedrock).
"""
cache_point = CachePointBlock(type="default")
if isinstance(control, dict) and "ttl" in control:
ttl = control["ttl"]
if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
return cache_point
def _transform_system_message(
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
@ -1241,7 +1252,7 @@ class AmazonConverseConfig(BaseConfig):
# Handle parallel_tool_calls configuration
parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None)
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
if parallel_tool_use_config is not None and bedrock_converse_supports_parallel_tool_use_config(model):
for key, value in parallel_tool_use_config.items():
if (
key in additional_request_params
@ -1526,7 +1537,8 @@ class AmazonConverseConfig(BaseConfig):
if cache_injection_points and len(bedrock_tools) > 0:
for point in cache_injection_points:
if point.get("location") == "tool_config":
bedrock_tools.append({"cachePoint": {"type": "default"}})
cache_point = self._build_cache_point_block(point.get("control"), model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
break
bedrock_tool_config: Optional[ToolConfigBlock] = None

2
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1558,7 +1558,7 @@ class AWSEventStreamDecoder:
text = chunk_data["outputText"]
# ai21 mapping
elif "ai21" in self.model: # fake ai21 streaming
text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore
text = chunk_data["completions"][0]["data"]["text"]
is_finished = True
finish_reason = "stop"
######## /bedrock/converse mappings ###############

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py
View file

@ -51,10 +51,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
Qwen2 uses "text" field, but we also support "generation" field for compatibility.
"""
try:
if hasattr(raw_response, "json"):
response_data = raw_response.json()
else:
response_data = raw_response
response_data = raw_response.json()
# Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
generated_text = response_data.get("generation", "") or response_data.get("text", "")

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py
View file

@ -175,10 +175,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
Transform Qwen3 Bedrock response to OpenAI format
"""
try:
if hasattr(raw_response, "json"):
response_data = raw_response.json()
else:
response_data = raw_response
response_data = raw_response.json()
# Extract the generated text - Qwen3 uses "generation" field
generated_text = response_data.get("generation", "")

45
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -5,6 +5,7 @@ from functools import partial
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args
import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -24,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
_get_httpx_client,
)
from litellm.types.llms.bedrock import GuardrailConfigBlock
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
@ -37,6 +39,38 @@ else:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
_GUARDRAIL_CONFIG_VALIDATOR: "TypeAdapter[GuardrailConfigBlock]" = TypeAdapter(GuardrailConfigBlock)
_GUARDRAIL_CONFIG_EXPECTED_FORMAT = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
try:
guardrail_config = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config)
except ValidationError as e:
raise BedrockError(
status_code=400,
message="Invalid guardrailConfig={}. Expected format: {}. Error: {}".format(
raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT, e
),
)
if "guardrailIdentifier" not in guardrail_config:
raise BedrockError(
status_code=400,
message="guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {}".format(
raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT
),
)
trace = guardrail_config.get("trace")
candidate_headers = {
"X-Amzn-Bedrock-GuardrailIdentifier": guardrail_config.get("guardrailIdentifier"),
"X-Amzn-Bedrock-GuardrailVersion": guardrail_config.get("guardrailVersion"),
"X-Amzn-Bedrock-Trace": trace.upper() if trace is not None else None,
}
return {name: value for name, value in candidate_headers.items() if value is not None}
class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
def __init__(self, **kwargs):
@ -390,7 +424,16 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
return headers
raw_guardrail_config = optional_params.pop("guardrailConfig", None)
if raw_guardrail_config is None:
return headers
existing_header_names = frozenset(name.lower() for name in headers)
guardrail_headers = {
name: value
for name, value in _bedrock_invoke_guardrail_headers(raw_guardrail_config).items()
if name.lower() not in existing_header_names
}
return {**headers, **guardrail_headers}
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]

48
litellm/llms/bedrock/common_utils.py
View file

@ -685,39 +685,27 @@ def get_bedrock_base_model(model: str) -> str:
return model
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
return any(
(litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True
for candidate in (model, get_bedrock_base_model(model))
)
def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model is a Claude 4.5 model on Bedrock.
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
Check if the model supports Bedrock prompt caching with an extended '1h' TTL
(in addition to the default 5m TTL).
Backed by the ``cache_creation_input_token_cost_above_1hr`` field in
``model_prices_and_context_window.json`` instead of a hardcoded list of
model-name patterns, so newly released models pick up support as soon as
their pricing entry ships, with no code change required here.
"""
model_lower = model.lower()
claude_4_5_patterns = [
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
"haiku-4.5",
"haiku_4.5",
"haiku-4-5",
"haiku_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
"sonnet-4.6",
"sonnet_4.6",
"sonnet-4-6",
"sonnet_4_6",
"opus-4.6",
"opus_4.6",
"opus-4-6",
"opus_4_6",
"opus-4.7",
"opus_4.7",
"opus-4-7",
"opus_4_7",
]
return any(pattern in model_lower for pattern in claude_4_5_patterns)
return any(
(litellm.model_cost.get(candidate) or {}).get("cache_creation_input_token_cost_above_1hr") is not None
for candidate in (model, get_bedrock_base_model(model))
)
_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$")

16
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2084,12 +2084,18 @@ class BaseLLMHTTPHandler:
initial_response: Union[AsyncIterator, AnthropicMessagesResponse]
if stream:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
anthropic_messages_stream_hidden_params,
)
completion_stream = anthropic_messages_provider_config.get_async_streaming_response_iterator(
model=model,
httpx_response=response,
request_body=request_body,
litellm_logging_obj=logging_obj,
)
stream_hidden_params = anthropic_messages_stream_hidden_params(response.headers)
if not self._has_agentic_completion_hook(logging_obj):
# No callback overrides async_should_run_agentic_loop, so the
@ -2097,7 +2103,10 @@ class BaseLLMHTTPHandler:
# and rebuilding the response from SSE at end-of-stream to call
# hooks that all return (False, {}). Stream through directly and
# skip that per-chunk + end-of-stream overhead.
return completion_stream
return AnthropicMessagesStreamingResponse(
completion_stream=completion_stream,
hidden_params=stream_hidden_params,
)
from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
AgenticAnthropicStreamingIterator,
@ -2114,7 +2123,10 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
)
return initial_response
return AnthropicMessagesStreamingResponse(
completion_stream=initial_response,
hidden_params=stream_hidden_params,
)
else:
initial_response = anthropic_messages_provider_config.transform_anthropic_messages_response(
model=model,

2
litellm/llms/oci/chat/cohere.py
View file

@ -110,7 +110,7 @@ def adapt_messages_to_cohere_standard(
tool_calls: Optional[List[CohereToolCall]] = None
if role == "assistant" and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item]
tool_calls = []
for tc in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item]
for tc in msg["tool_calls"]: # pyright: ignore[reportOptionalIterable] # truthiness check above rules out None
raw_arguments: Any = tc.get("function", {}).get("arguments", {})
if isinstance(raw_arguments, str):
try:

6
litellm/llms/ollama/chat/transformation.py
View file

@ -363,7 +363,11 @@ class OllamaChatConfig(BaseConfig):
response_json_message["reasoning_content"] = reasoning_content
response_json_message["content"] = content
if request_data.get("format", "") == "json" and litellm_params.get("function_name") is not None:
if (
request_data.get("format", "") == "json"
and litellm_params.get("function_name") is not None
and response_json_message is not None
):
function_call = json.loads(response_json_message["content"])
message = litellm.Message(
content=None,

2
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -716,7 +716,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
elif isinstance(content, list) and content_idx_optional is not None:
# Replace specific text item in list content
choice.message.content[content_idx_optional]["text"] = guardrail_response # type: ignore
content[content_idx_optional]["text"] = guardrail_response
async def _apply_guardrail_responses_to_output_tool_calls(
self,

10
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -45,6 +45,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
ResponsesAPIStreamEvents,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
@ -586,7 +587,14 @@ class OpenAIResponsesHandler(BaseTranslation):
"""
Check if the streaming has ended.
"""
return all(response.choices[0].finish_reason is not None for response in responses_so_far)
if not responses_so_far:
return False
terminal_types = {
ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
}
return responses_so_far[-1].get("type") in terminal_types
def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str:
"""

2
litellm/llms/sap/chat/transformation.py
View file

@ -155,7 +155,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
def headers(self) -> Dict[str, str]:
if self.token_creator is None:
self.run_env_setup()
access_token = self.token_creator() # type: ignore
access_token = self.token_creator() # pyright: ignore[reportOptionalCall] # run_env_setup set it or raised
return {
"Authorization": access_token,
"AI-Resource-Group": self.resource_group,

0
litellm/llms/tencent/__init__.py Normal file
View file

0
litellm/llms/tencent/chat/__init__.py Normal file
View file

68
litellm/llms/tencent/chat/transformation.py Normal file
View file

@ -0,0 +1,68 @@
"""
Translates from OpenAI's `/v1/chat/completions` to Tencent TokenHub's
OpenAI-compatible endpoint.
"""
from typing import Optional
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class TencentChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
params = super().get_supported_openai_params(model)
if supports_reasoning(model, custom_llm_provider="tencent"):
params.extend(["thinking", "reasoning_effort"])
return params
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
thinking_value = optional_params.pop("thinking", None)
reasoning_effort = optional_params.pop("reasoning_effort", None)
if thinking_value is not None:
if isinstance(thinking_value, dict):
optional_params["thinking"] = thinking_value
elif reasoning_effort is not None and reasoning_effort != "none":
optional_params["thinking"] = {"type": "enabled"}
return optional_params
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> tuple[Optional[str], Optional[str]]:
api_base = api_base or get_secret_str("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1"
dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY")
return api_base, dynamic_api_key
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if not api_base:
api_base = "https://tokenhub-intl.tencentcloudmaas.com/v1"
api_base = api_base.rstrip("/")
if api_base.endswith("/chat/completions"):
return api_base
if not api_base.endswith("/v1"):
api_base = f"{api_base}/v1"
return f"{api_base}/chat/completions"

6
litellm/llms/tencent/cost_calculator.py Normal file
View file

@ -0,0 +1,6 @@
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")

0
litellm/llms/tencent/messages/__init__.py Normal file
View file

85
litellm/llms/tencent/messages/transformation.py Normal file
View file

@ -0,0 +1,85 @@
"""
Tencent Anthropic-compatible messages transformation config.
Tencent TokenHub exposes an Anthropic-compatible Messages API endpoint
alongside its standard OpenAI-compatible chat completions endpoint.
"""
from typing import Any, Optional
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
class TencentAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
Tencent TokenHub exposes an Anthropic-compatible Messages API.
Unlike the chat completions endpoint (which uses /v1), the Anthropic
endpoint may use a different base URL. Configure via
TENCENT_ANTHROPIC_API_BASE or TENCENT_API_BASE.
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "tencent"
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("TENCENT_API_KEY") or litellm.api_key
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> str:
return (
api_base
or get_secret_str("TENCENT_ANTHROPIC_API_BASE")
or get_secret_str("TENCENT_API_BASE")
or "https://tokenhub-intl.tencentcloudmaas.com"
)
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> tuple[dict, Optional[str]]:
return super().validate_anthropic_messages_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=self.get_api_key(api_key=api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base_url = self.get_api_base(api_base=api_base).rstrip("/")
if base_url.endswith("/v1/messages"):
return base_url
if base_url.endswith("/v1/chat/completions"):
base_url = base_url[: -len("/v1/chat/completions")]
elif base_url.endswith("/v1"):
base_url = base_url[: -len("/v1")]
return f"{base_url}/v1/messages"

270
litellm/llms/tinyfish/search/transformation.py
View file

@ -6,53 +6,42 @@ Docs: https://docs.tinyfish.ai/search-api
from __future__ import annotations
from typing import Literal, TypedDict
import json
from typing import Literal
from urllib.parse import urlencode
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
class _TinyfishSearchRequestRequired(TypedDict):
query: str
class TinyfishSearchRequest(_TinyfishSearchRequestRequired, total=False):
location: str
language: str
page: int
include_thumbnail: bool
max_results: int
class _TinyfishResultItem(BaseModel, frozen=True):
title: str = ""
url: str = ""
snippet: str = ""
class _TinyfishApiResponse(BaseModel, frozen=True):
results: tuple[_TinyfishResultItem, ...] = ()
_UrlEncodableParams = TypeAdapter(dict[str, str | int | bool])
_StrList = TypeAdapter(list[str])
_StrFrozenSet = TypeAdapter(frozenset[str])
_TINYFISH_PARAMS_KEY = "_tinyfish_params"
_TINYFISH_DOCS_URL = "https://docs.tinyfish.ai/search-api"
_TINYFISH_RESULT_CAP = 10 # TinyFish's natural per-page SERP ceiling
class TinyfishSearchConfig(BaseSearchConfig):
TINYFISH_API_BASE = "https://api.search.tinyfish.ai"
def __init__(self) -> None:
super().__init__()
# Threaded from transform_search_request → transform_search_response so the
# response slice honors the caller's max_results without re-sending it on
# the wire (TinyFish doesn't honor it server-side). Safe because the
# config is instantiated per-call via ProviderConfigManager.
self._caller_max_results: int | None = None
@staticmethod
def ui_friendly_name() -> str:
return "TinyFish"
@ -97,36 +86,77 @@ class TinyfishSearchConfig(BaseSearchConfig):
optional_params: dict[str, object],
**kwargs: object,
) -> dict[str, object]:
"""
Transform a LiteLLM search request to TinyFish's querystring format.
Maps LiteLLM's unified-spec params (see
``BaseSearchConfig.get_supported_perplexity_optional_params``) to
TinyFish equivalents:
- ``query`` (str or list[str]) → ``query`` (list joined by spaces)
- ``country`` → ``location``
- ``search_domain_filter`` (list[str]) → folded into the query as
``(<query>) (site:a OR site:b ...)`` (TinyFish has no first-class
field today; see ML-2084 for the planned ``include_domains``)
- ``max_results`` → not sent on the wire; stashed on
``self._caller_max_results`` for client-side response truncation
(TinyFish doesn't honor it server-side)
- ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent)
Any other ``optional_params`` keys are forwarded to TinyFish as-is.
dict/list values are JSON-encoded so they survive ``urlencode``.
Returns:
``{_TINYFISH_PARAMS_KEY: <dict of querystring entries>}``.
``get_complete_url`` reads this back to build the final URL.
"""
resolved_query = " ".join(query) if isinstance(query, list) else query
request_data: TinyfishSearchRequest = {"query": resolved_query}
country = optional_params.get("country")
if isinstance(country, str):
request_data["location"] = country
raw_max = optional_params.get("max_results")
if isinstance(raw_max, (int, float, str)):
request_data["max_results"] = max(1, min(int(raw_max), 20))
try:
domains = _StrList.validate_python(optional_params.get("search_domain_filter"))
except (ValidationError, TypeError):
domains = []
if domains:
request_data["query"] = _append_domain_filters(request_data["query"], domains)
resolved_query = _append_domain_filters(resolved_query, domains)
result_data: dict[str, object] = dict(request_data)
request_data: dict[str, object] = {"query": resolved_query}
country = optional_params.get("country")
if isinstance(country, str):
request_data["location"] = country
# max_results is enforced client-side on the response (TinyFish ignores
# the param and always returns ~10). Clamp to [1, 10] and stash on self
# so transform_search_response can slice without re-reading the URL.
raw_max = optional_params.get("max_results")
if isinstance(raw_max, (int, float, str)):
try:
self._caller_max_results = max(1, min(int(raw_max), _TINYFISH_RESULT_CAP))
except (ValueError, TypeError, OverflowError):
# OverflowError covers int(float('inf')) and similar non-finite floats.
verbose_logger.warning(
"TinyFish Search: max_results=%r is not a valid integer; ignoring.",
raw_max,
)
raw_supported: object = (
self.get_supported_perplexity_optional_params() # any-ok: base class returns bare set
)
supported_perplexity = _StrFrozenSet.validate_python(raw_supported)
for param, value in optional_params.items():
if param not in supported_perplexity and param not in result_data:
result_data[param] = value
if param not in supported_perplexity and param not in request_data:
# `fetch` expects a JSON-encoded object on the wire; accept the
# natural Python dict form and serialize here so callers don't
# have to pre-stringify.
if isinstance(value, dict):
value = json.dumps(value, separators=(",", ":"))
# `urlencode` would render Python bool as "True"/"False"
# (capitalized). ux-labs validators require lowercase
# "true"/"false" (e.g. `include_thumbnail`); normalize here.
elif isinstance(value, bool):
value = "true" if value else "false"
request_data[param] = value
return {_TINYFISH_PARAMS_KEY: result_data}
return {_TINYFISH_PARAMS_KEY: request_data}
def transform_search_response(
self,
@ -134,24 +164,158 @@ class TinyfishSearchConfig(BaseSearchConfig):
logging_obj: LiteLLMLoggingObj | None,
**kwargs: object,
) -> SearchResponse:
raw_json: object = raw_response.json() # any-ok: httpx Response.json() -> Any
parsed = _TinyfishApiResponse.model_validate(raw_json)
"""
Transform a TinyFish response to LiteLLM's unified ``SearchResponse``.
max_results_str: str = "20"
if raw_response.request:
raw_param: object = raw_response.request.url.params.get( # any-ok: httpx QueryParams.get() -> Any
"max_results", "20"
Mappings (per-result):
- ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null)
- ``url`` → ``SearchResult.url`` (defaults to ``""``)
- ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``)
- all other per-result fields (``position``, ``site_name``,
``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as
extras on ``SearchResult`` via its ``extra="allow"`` config.
Top-level ``parameter_warnings`` (see ML-2085) is read when present and
each entry is re-fired via ``verbose_logger.warning``. Absent or
malformed entries are silently skipped — never throws.
Error paths routed through ``self._wrap_error`` for uniform
``"TinyFish Search: <msg>. See <docs> for details."`` wrapping:
- non-2xx HTTP status (caught here because ``AsyncHTTPHandler.get``
does not call ``raise_for_status``)
- 200 with non-JSON body
- 200 with valid JSON whose shape doesn't satisfy ``SearchResponse``
Returns:
``SearchResponse`` truncated to ``self._caller_max_results`` (or
``_TINYFISH_RESULT_CAP`` when the caller didn't set ``max_results``).
"""
# AsyncHTTPHandler.get does not call raise_for_status, so non-2xx
# responses arrive here looking successful. Dispatch through
# get_error_class so callers see a uniform attributed error.
if not (200 <= raw_response.status_code < 300):
raise self._wrap_error(
error_message=raw_response.text,
status_code=raw_response.status_code,
headers=dict(raw_response.headers),
)
max_results_str = str(raw_param)
max_results: int = min(int(max_results_str), 20)
results = [
SearchResult(title=item.title, url=item.url, snippet=item.snippet) for item in parsed.results[:max_results]
]
try:
raw_json: object = raw_response.json() # any-ok: httpx Response.json() -> Any
except json.JSONDecodeError:
raise self._wrap_error(
error_message=f"Expected JSON response, got: {raw_response.text[:200]}",
status_code=raw_response.status_code,
headers=dict(raw_response.headers),
)
return SearchResponse(results=results, object="search")
_default_missing_result_fields(raw_json)
try:
parsed = SearchResponse.model_validate(raw_json)
except ValidationError as e:
raise self._wrap_error(
error_message=(f"Response shape does not match LiteLLM's SearchResponse schema: {e}"),
status_code=raw_response.status_code,
headers=dict(raw_response.headers),
)
_emit_parameter_warnings(parsed)
max_results = self._caller_max_results or _TINYFISH_RESULT_CAP
return SearchResponse(results=list(parsed.results[:max_results]))
def _wrap_error(
self,
error_message: str,
status_code: int,
headers: dict[str, str],
) -> Exception:
"""
Build an attributed ``BaseLLMException`` from a TinyFish error body.
Used only at the call sites we control inside
``transform_search_response`` (non-2xx, JSONDecodeError, ValidationError).
Not an override of ``BaseSearchConfig.get_error_class``: that path is
left to inherit from the base so it auto-picks-up any future LiteLLM
improvements. Trade-off: network failures (routed through LiteLLM
core's ``_handle_error`` → ``BaseSearchConfig.get_error_class``) won't
carry the ``TinyFish Search:`` prefix — the bare error already names
the host in the URL, so attribution is implicit there.
"""
# ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}.
# Best-effort unwrap to surface the inner message; fall back to the raw body
# for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text).
inner_message = error_message
try:
body: object = json.loads(error_message) # any-ok: json.loads -> Any
if isinstance(body, dict):
error_obj: object = body.get("error") # any-ok: untyped dict
if isinstance(error_obj, dict):
candidate: object = error_obj.get("message") # any-ok: untyped dict
if isinstance(candidate, str) and candidate:
inner_message = candidate
except (json.JSONDecodeError, TypeError):
pass
return BaseLLMException(
status_code=status_code,
message=f"TinyFish Search: {inner_message}. See {_TINYFISH_DOCS_URL} for details.",
headers=headers,
)
def _append_domain_filters(query: str, domains: list[str]) -> str:
domain_clauses = " OR ".join(f"site:{d}" for d in domains)
return f"({query}) ({domain_clauses})"
def _default_missing_result_fields(raw_json: object) -> None:
"""Default missing/null title/url/snippet to "" on each result item in place.
SearchResult requires these three fields; a degraded TinyFish result flows
through with empty strings instead of failing the whole call.
"""
if not isinstance(raw_json, dict):
return
results_in = raw_json.get("results")
if not isinstance(results_in, list):
return
for item in results_in:
if not isinstance(item, dict):
continue
for field in ("title", "url", "snippet"):
if not isinstance(item.get(field), str):
item[field] = ""
def _emit_parameter_warnings(parsed: SearchResponse) -> None:
"""Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings.
Defensive: skip silently on any shape we don't recognize so a malformed
entry (or an early/partial rollout of the field) never throws.
Schema per entry: ``{type, parameter, message, docs_url?}``.
"""
warnings_field: object = (
getattr(parsed, "parameter_warnings", None) # any-ok: extras=allow field
)
if not isinstance(warnings_field, list):
return
for entry in warnings_field:
if not isinstance(entry, dict):
continue
warning_type: object = entry.get("type") # any-ok: untyped dict
parameter: object = entry.get("parameter") # any-ok: untyped dict
message: object = entry.get("message") # any-ok: untyped dict
if not isinstance(warning_type, str) or not warning_type:
continue
if not isinstance(parameter, str) or not parameter:
continue
if not isinstance(message, str) or not message:
continue
verbose_logger.warning(
"TinyFish Search parameter_warning (%s) `%s`: %s",
warning_type,
parameter,
message,
)

20
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any, Dict, Optional
from litellm._uuid import uuid
from litellm.llms.vertex_ai.common_utils import (
@ -47,7 +47,7 @@ class VertexAIBatchTransformation:
) -> LiteLLMBatch:
return LiteLLMBatch(
id=cls._get_batch_id_from_vertex_ai_batch_response(response),
completion_window="24hrs",
completion_window="24h",
created_at=_convert_vertex_datetime_to_openai_datetime(vertex_datetime=response.get("createTime", "")),
endpoint="",
input_file_id=cls._get_input_file_id_from_vertex_ai_batch_response(response),
@ -207,3 +207,19 @@ class VertexAIBatchTransformation:
parts = model_path.split("/")
model = f"publishers/{'/'.join(parts[:3])}"
return model
@classmethod
def is_unmanaged_gcs_batch_input_file_id(cls, input_file_id: Optional[str]) -> bool:
"""
Returns True if `input_file_id` is a raw gs:// Vertex batch input file (i.e. not a
LiteLLM-managed unified file id) with a `publishers/` model path that
`_get_model_from_gcs_file` can parse.
"""
return input_file_id is not None and input_file_id.startswith("gs://") and "publishers/" in input_file_id
@classmethod
def get_bare_model_name_from_gcs_file(cls, gcs_file_uri: str) -> str:
"""
Extracts the bare model name (e.g. "gemini-1.5-flash-001") from a gcs file uri.
"""
return cls._get_model_from_gcs_file(gcs_file_uri).rsplit("/", 1)[-1]

14
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -2315,17 +2315,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# Store thoughtSignatures in provider_specific_fields
if thought_signatures is not None:
if "provider_specific_fields" not in chat_completion_message:
chat_completion_message["provider_specific_fields"] = {}
chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore
thought_signature_fields = chat_completion_message.get("provider_specific_fields") or {}
thought_signature_fields["thought_signatures"] = thought_signatures
chat_completion_message["provider_specific_fields"] = thought_signature_fields
# Store server-side tool invocations in provider_specific_fields
if server_side_tool_invocations is not None:
if "provider_specific_fields" not in chat_completion_message:
chat_completion_message["provider_specific_fields"] = {}
chat_completion_message["provider_specific_fields"]["server_side_tool_invocations"] = (
server_side_tool_invocations # type: ignore
)
tool_invocation_fields = chat_completion_message.get("provider_specific_fields") or {}
tool_invocation_fields["server_side_tool_invocations"] = server_side_tool_invocations
chat_completion_message["provider_specific_fields"] = tool_invocation_fields
if isinstance(model_response, ModelResponseStream):
choice = VertexGeminiConfig._create_streaming_choice(

10
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -1,5 +1,5 @@
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any, Optional
from litellm._logging import verbose_logger
@ -175,7 +175,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
@ -217,10 +217,10 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
contents = [{"role": "user", "parts": [{"text": prompt}]}]
# Prepare generation config
generation_config: Dict[str, Any] = {"responseModalities": ["IMAGE"]}
generation_config: dict[str, Any] = {"responseModalities": ["IMAGE"]}
# Seed from user-supplied imageConfig dict; flat params are overlaid for backward compat.
image_config: Dict[str, Any] = dict(optional_params.get("imageConfig") or {})
image_config: dict[str, Any] = dict(optional_params.get("imageConfig") or {})
if "aspectRatio" in optional_params:
image_config["aspectRatio"] = optional_params["aspectRatio"]
@ -241,7 +241,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
elif "n" in optional_params:
generation_config["candidateCount"] = optional_params["n"]
request_body: Dict[str, Any] = {
request_body: dict[str, Any] = {
"contents": contents,
"generationConfig": generation_config,
}

56
litellm/main.py
View file

@ -8304,26 +8304,6 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List]
finish_reason = chunks[-1]["choices"][0]["finish_reason"]
logprobs = chunks[-1]["choices"][0]["logprobs"]
response = {
"id": id,
"object": object,
"created": created,
"model": model,
"system_fingerprint": system_fingerprint,
"choices": [
{
"text": None,
"index": 0,
"logprobs": logprobs,
"finish_reason": finish_reason,
}
],
"usage": {
"prompt_tokens": None,
"completion_tokens": None,
"total_tokens": None,
},
}
content_list = []
for chunk in chunks:
choices = chunk["choices"]
@ -8335,25 +8315,37 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List]
# Combine the "content" strings into a single string || combine the 'function' strings into a single string
combined_content = "".join(content_list)
# Update the "content" field within the response dictionary
response["choices"][0]["text"] = combined_content
if len(combined_content) > 0:
pass
else:
pass
# # Update usage information if needed
try:
response["usage"]["prompt_tokens"] = token_counter(model=model, messages=messages)
prompt_tokens = token_counter(model=model, messages=messages)
except Exception: # don't allow this failing to block a complete streaming response from being returned
print_verbose("token_counter failed, assuming prompt tokens is 0")
response["usage"]["prompt_tokens"] = 0
response["usage"]["completion_tokens"] = token_counter(
prompt_tokens = 0
completion_tokens = token_counter(
model=model,
text=combined_content,
count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages
)
response["usage"]["total_tokens"] = response["usage"]["prompt_tokens"] + response["usage"]["completion_tokens"]
response = {
"id": id,
"object": object,
"created": created,
"model": model,
"system_fingerprint": system_fingerprint,
"choices": [
{
"text": combined_content,
"index": 0,
"logprobs": logprobs,
"finish_reason": finish_reason,
}
],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
}
return TextCompletionResponse(**response)

1322
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

1
litellm/models/mcp_server.py
View file

@ -93,6 +93,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
has_user_credential: Optional[bool] = None
source_url: Optional[str] = None
timeout: Optional[float] = None
max_concurrent_requests: Optional[int] = None
approval_status: Optional[str] = Field(
default="active",
description="Approval status: 'pending_review', 'active', 'rejected'",

1
litellm/models/verification_token.py
View file

@ -39,6 +39,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
permissions: Dict = {}
model_spend: Dict = {}
model_max_budget: Dict = {}
budget_fallbacks: dict[str, list[str]] = {}
soft_budget_cooldown: bool = False
blocked: Optional[bool] = None
litellm_budget_table: Optional[dict] = None

4
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -11,6 +11,7 @@ from litellm.proxy._types import (
LiteLLM_TeamTable,
ProxyException,
SpecialHeaders,
SpecialMCPServerName,
SpecialMCPServerNames,
UserAPIKeyAuth,
)
@ -1041,6 +1042,9 @@ class MCPRequestHandler:
if object_permissions is None:
return list(set(team_access_group_servers))
if SpecialMCPServerName.all_proxy_servers.value in (object_permissions.mcp_servers or []):
return list(global_mcp_server_manager.get_registry().keys())
direct_mcp_servers = global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or [])
legacy_access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(

289
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -3,12 +3,13 @@ import html as _html
import json
import time
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
import httpx
from fastapi import APIRouter, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from pydantic import BaseModel, ValidationError
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
@ -31,11 +32,11 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
)
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.utils import get_server_root_path
from litellm.types.mcp import MCPAuth
from litellm.types.mcp import MCPAuth, MCPCredentials
from litellm.types.mcp_server.mcp_server_manager import MCPServer
if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import LiteLLM_MCPServerTable, UserAPIKeyAuth
# TTL cache for upstream OAuth metadata fetched from pass-through MCP servers.
# Keeps us from hammering the upstream IdP on each discovery request.
@ -390,6 +391,46 @@ async def _store_per_user_token_server_side(
)
def _raise_if_not_oauth2(mcp_server: MCPServer) -> None:
"""Reject a non-oauth2 server from the gateway's OAuth authorize/token/register flow."""
if mcp_server.auth_type == MCPAuth.oauth2:
return
raise HTTPException(
status_code=400,
detail={
"error": "server_not_oauth2",
"message": (
f"MCP server '{mcp_server.server_name or mcp_server.name}' does not use OAuth "
f"(auth_type={mcp_server.auth_type}). This server does not support the authorization-code "
"flow; it has no client_id, authorize, token, or registration endpoint. "
"Access is controlled by the server's configured auth_type and access groups"
),
},
)
def _raise_unless_oauth2_discovery_server(
mcp_server: Optional[MCPServer],
mcp_server_name: Optional[str],
description: str,
) -> None:
"""404 a NAMED discovery request unless it resolves to an oauth2 server.
A named server that is unknown (or hidden from the caller) and one that exists
but is non-oauth2 both return the same 404, so the well-known discovery paths
cannot be used to enumerate non-OAuth server names. Root discovery (no name) is
unaffected, and pass-through servers are resolved by the caller before this runs.
"""
if mcp_server_name is None:
return
if mcp_server is not None and mcp_server.auth_type == MCPAuth.oauth2:
return
raise HTTPException(
status_code=404,
detail=f"MCP server '{mcp_server_name}' is {description}",
)
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -457,6 +498,7 @@ async def exchange_token_with_server(
refresh_token: Optional[str] = None,
scope: Optional[str] = None,
):
_raise_if_not_oauth2(mcp_server)
if grant_type not in ("authorization_code", "refresh_token"):
raise HTTPException(status_code=400, detail="Unsupported grant_type")
@ -573,6 +615,178 @@ async def exchange_token_with_server(
return JSONResponse(result, headers=TOKEN_NO_CACHE_HEADERS)
class _DcrClientRegistration(BaseModel):
"""RFC 7591 dynamic client registration response, narrowed to the fields the gateway
must persist to authenticate later token-endpoint calls. Extra members are ignored."""
client_id: str
client_secret: Optional[str] = None
token_endpoint_auth_method: Optional[str] = None
class _PersistedDcrCredentials(BaseModel):
client_id: Optional[str] = None
client_secret: Optional[str] = None
token_endpoint_auth_method: Optional[str] = None
def _get_persisted_dcr_credentials(credentials: object) -> Optional[_PersistedDcrCredentials]:
if not credentials:
return None
try:
return (
_PersistedDcrCredentials.model_validate_json(credentials)
if isinstance(credentials, str)
else _PersistedDcrCredentials.model_validate(credentials)
)
except ValidationError:
return None
def _decrypt_persisted_dcr_credential(value: Optional[str], key: str) -> Optional[str]:
if value is None:
return None
return decrypt_value_helper(
value=value,
key=key,
exception_type="debug",
return_original_value=True,
)
def _apply_persisted_dcr_credentials(mcp_server: MCPServer, credentials: _PersistedDcrCredentials) -> bool:
client_id = _decrypt_persisted_dcr_credential(credentials.client_id, "client_id")
if not client_id:
return False
mcp_server.client_id = client_id
mcp_server.client_secret = _decrypt_persisted_dcr_credential(credentials.client_secret, "client_secret")
mcp_server.token_endpoint_auth_method = credentials.token_endpoint_auth_method
return True
async def _get_persisted_mcp_server_with_dcr_client_id(
mcp_server: MCPServer,
) -> Optional[tuple["LiteLLM_MCPServerTable", _PersistedDcrCredentials]]:
from litellm.proxy._experimental.mcp_server.db import get_mcp_server # noqa: PLC0415
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415
try:
prisma_client = get_prisma_client_or_throw("Database not connected. Cannot read MCP OAuth client registration.")
persisted_mcp_server = await get_mcp_server(
prisma_client=prisma_client,
server_id=mcp_server.server_id,
)
except Exception as exc: # noqa: BLE001
verbose_logger.debug(
"register_client_with_server: failed to read persisted DCR client registration for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return None
if persisted_mcp_server is None:
return None
credentials = _get_persisted_dcr_credentials(persisted_mcp_server.credentials)
if credentials is None or not credentials.client_id:
return None
return persisted_mcp_server, credentials
async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> bool:
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
return False
persisted_mcp_server, credentials = persisted
if not _apply_persisted_dcr_credentials(mcp_server, credentials):
return False
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415
global_mcp_server_manager,
)
try:
await global_mcp_server_manager.update_server(persisted_mcp_server)
except Exception as exc: # noqa: BLE001
verbose_logger.warning(
"register_client_with_server: failed to refresh persisted DCR client registration for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return bool(mcp_server.client_id)
DcrRegistrationPersistenceResult = Literal["persisted", "reused", "failed"]
async def _persist_dcr_client_registration(
mcp_server: MCPServer, registration_response: object
) -> DcrRegistrationPersistenceResult:
"""Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row.
The interactive authorization_code flow mints a ``client_id`` via Dynamic Client
Registration that discovery cannot re-derive; without persisting it the autonomous
``refresh_token`` grant has no client identity, so an expired access token forces a
full re-authorization instead of a silent refresh. Mirrors the ``encrypt_credentials``
write that ``client_credentials`` and token exchange already use. Failures are logged,
never raised: registration still returns to the caller even when persistence fails.
"""
try:
registration = _DcrClientRegistration.model_validate(registration_response)
except ValidationError as exc:
verbose_logger.warning(
"register_client_with_server: DCR response has no usable client_id for server_id=%s; "
"client registration not persisted (%s)",
mcp_server.server_id,
exc,
)
return "failed"
if await _reuse_persisted_dcr_client_if_available(mcp_server):
return "reused"
credentials: MCPCredentials = {
"client_id": registration.client_id,
**({"client_secret": registration.client_secret} if registration.client_secret is not None else {}),
**(
{"token_endpoint_auth_method": "client_secret_basic"}
if registration.token_endpoint_auth_method == "client_secret_basic"
else {}
),
}
from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415
global_mcp_server_manager,
)
from litellm.proxy._types import UpdateMCPServerRequest # noqa: PLC0415
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415
try:
prisma_client = get_prisma_client_or_throw(
"Database not connected. Cannot persist MCP OAuth client registration."
)
updated_row = await update_mcp_server(
prisma_client=prisma_client,
data=UpdateMCPServerRequest(
server_id=mcp_server.server_id,
credentials=credentials,
**({"token_url": mcp_server.token_url} if mcp_server.token_url else {}),
),
touched_by="mcp_oauth_dcr",
)
await global_mcp_server_manager.update_server(updated_row)
return "persisted"
except Exception as exc: # noqa: BLE001
verbose_logger.warning(
"register_client_with_server: failed to persist DCR client registration for server_id=%s: %s",
mcp_server.server_id,
exc,
)
return "failed"
async def register_client_with_server(
request: Request,
mcp_server: MCPServer,
@ -581,7 +795,9 @@ async def register_client_with_server(
response_types: Optional[list],
token_endpoint_auth_method: Optional[str],
fallback_client_id: Optional[str] = None,
persist_credentials: bool = False,
):
_raise_if_not_oauth2(mcp_server)
request_base_url = get_request_base_url(request)
dummy_return = {
"client_id": fallback_client_id or mcp_server.server_name,
@ -589,7 +805,10 @@ async def register_client_with_server(
"redirect_uris": [f"{request_base_url}/callback"],
}
if mcp_server.client_id and mcp_server.client_secret:
if mcp_server.client_id:
return dummy_return
if await _reuse_persisted_dcr_client_if_available(mcp_server):
return dummy_return
if mcp_server.authorization_url is None:
@ -625,6 +844,11 @@ async def register_client_with_server(
token_response = response.json()
if persist_credentials:
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response)
if persistence_result == "reused":
return dummy_return
return JSONResponse(token_response)
@ -655,6 +879,7 @@ async def authorize(
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
_raise_if_not_oauth2(mcp_server)
# Use server's stored client_id when caller doesn't supply one.
# Raise a clear error instead of passing an empty string — an empty
# client_id would silently produce a broken authorization URL.
@ -1063,6 +1288,15 @@ async def _build_oauth_protected_resource_response(
detail=(f"Upstream oauth-protected-resource metadata unavailable for MCP server {mcp_server.name!r}"),
)
obo_response = _obo_protected_resource_response(mcp_server, resource_url)
if obo_response is not None:
return obo_response
# An OBO server with no configured issuer falls through to the gateway default so discovery still
# returns metadata; every other non-oauth2 named server 404s to avoid enumeration.
if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange:
_raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource")
return {
"authorization_servers": [
(f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}")
@ -1072,6 +1306,51 @@ async def _build_oauth_protected_resource_response(
}
def _obo_protected_resource_response(mcp_server: Optional[MCPServer], resource_url: str) -> Optional[dict]:
"""The OBO (token_exchange) PRM, or None when this server is not OBO / no issuer is configured.
The client SSOs with the IdP to obtain a subject token, which LiteLLM then exchanges, so discovery
points at the JWT-auth issuer(s) LiteLLM trusts (the same IdP that issues and validates the
subject), not the gateway. None falls the caller back to the gateway default so discovery still
returns metadata; it just can't name the IdP.
"""
if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange:
return None
issuers = _jwt_auth_issuers()
if not issuers:
return None
return {
"authorization_servers": issuers,
"resource": resource_url,
"scopes_supported": (mcp_server.scopes if mcp_server.scopes else []),
}
def _jwt_auth_issuers() -> list:
"""The OAuth issuer identifier(s) LiteLLM's JWT auth trusts, for the OBO PRM authorization_servers.
In token_exchange the IdP that issues the subject JWT is the same one LiteLLM validates it
against, so OBO discovery points clients at the JWT-auth issuer to obtain a subject token.
Sourced from ``JWT_ISSUER`` and any configured ``litellm_jwtauth.issuers``.
"""
import os # noqa: PLC0415
from litellm.proxy.proxy_server import general_settings # noqa: PLC0415
issuers: list = []
env_issuer = os.getenv("JWT_ISSUER")
if env_issuer:
issuers.append(env_issuer)
jwtauth = general_settings.get("litellm_jwtauth") if isinstance(general_settings, dict) else None
raw_issuers = jwtauth.get("issuers") if isinstance(jwtauth, dict) else getattr(jwtauth, "issuers", None)
for cfg in raw_issuers or []:
issuer = cfg.get("issuer") if isinstance(cfg, dict) else getattr(cfg, "issuer", None)
if issuer and issuer not in issuers:
issuers.append(issuer)
return issuers
# Standard MCP pattern: /.well-known/oauth-protected-resource/mcp/{server_name}
# This is the pattern expected by standard MCP clients (mcp-inspector, VSCode Copilot)
@router.get(
@ -1149,6 +1428,8 @@ def _build_oauth_authorization_server_response(
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
_raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server")
return {
"issuer": request_base_url, # point to your proxy
"authorization_endpoint": authorization_endpoint,

749
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

File diff suppressed because it is too large Load diff

124
litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py
View file

@ -12,7 +12,7 @@ every other mode so the caller defers to v1 (parity-safe); it grows one branch p
from __future__ import annotations
import base64
from typing import TYPE_CHECKING, NoReturn, Optional
from typing import TYPE_CHECKING, Literal, NoReturn, Optional
from fastapi import HTTPException
from pydantic import SecretStr
@ -26,6 +26,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ServerSpec,
SharedKey,
Subject,
TokenExchangeConfig,
)
from litellm.types.mcp import MCPAuth
@ -61,8 +62,9 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
an ``assert_never`` tail, so a newly added auth mode fails the type gate here until it is
explicitly mapped or explicitly deferred, rather than silently falling through to v1. Live
modes: ``none``, the static-header family (``api_key`` plus the Authorization schemes,
all shared-key), and ``oauth2`` per-user tokens (``authorization_code``); client_credentials
(M2M), delegated/passthrough oauth2, token exchange, and SigV4 return None and stay on v1.
all shared-key), ``oauth2`` per-user tokens (``authorization_code``), and
``oauth2_token_exchange`` (OBO); client_credentials (M2M), delegated/passthrough
oauth2, and SigV4 return None and stay on v1.
"""
if server.is_byok:
return None # per-user BYOK source not migrated yet -> defer to v1 (any auth_type)
@ -92,11 +94,47 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
)
# client_credentials (M2M) and delegate/passthrough oauth2 stay on v1
return None
case MCPAuth.oauth2_token_exchange | MCPAuth.aws_sigv4:
return None # token exchange and SigV4 are not migrated yet -> defer to v1
case MCPAuth.oauth2_token_exchange:
return _token_exchange_spec(server, resource)
case MCPAuth.aws_sigv4:
return None # SigV4 is not migrated yet -> defer to v1
assert_never(auth_type)
def _token_exchange_spec(server: MCPServer, resource: str) -> Optional[ServerSpec]:
"""Build a token_exchange (OBO) spec, or defer (None) when it is not OBO-configured.
An OBO server with ``client_id``/``client_secret`` is owned by the v2 arm even if the
``token_exchange_endpoint``/``token_url`` is absent: a missing endpoint then fails closed (412) at
the exchanger rather than silently deferring to v1 and connecting unauthenticated, since the
gateway must not guess the IdP or fall back to a weaker source. Without client credentials there is
nothing to own, so the server stays on v1 (parity-safe). ``profile`` selects the wire dialect
(``rfc8693`` default, ``entra_obo`` for Microsoft Entra On-Behalf-Of); an unrecognized value
normalizes to ``rfc8693`` so a bad config value cannot crash spec-building. ``audience`` is
forwarded only when the operator set it; a missing one is omitted, not derived.
"""
endpoint = server.token_exchange_endpoint or server.token_url
if not server.client_id or not server.client_secret:
return None
profile: Literal["rfc8693", "entra_obo"] = (
"entra_obo" if server.token_exchange_profile == "entra_obo" else "rfc8693"
)
return ServerSpec(
server_id=server.server_id,
resource=resource,
config=TokenExchangeConfig(
profile=profile,
subject_token_type=server.subject_token_type or "urn:ietf:params:oauth:token-type:access_token",
token_exchange_endpoint=endpoint,
audience=server.audience,
client_id=server.client_id,
client_secret=SecretStr(server.client_secret),
token_endpoint_auth_method=server.token_endpoint_auth_method,
scopes=tuple(server.scopes or ()),
),
)
def _shared_key_spec(
server: MCPServer,
resource: str,
@ -148,23 +186,75 @@ def raise_public(error: CredError) -> NoReturn:
assert_never(error.tag)
def raise_user_oauth_challenge(server: MCPServer) -> NoReturn:
def oauth_protected_resource_path(root_path: str, server: MCPServer) -> str:
"""The server's RFC 9728 Protected Resource Metadata path, the shared anchor of both challenges.
``root_path`` is the proxy's ``SERVER_ROOT_PATH``, resolved by the caller (the imperative shell)
so this stays a pure function of its inputs; ``"/"`` and ``""`` both mean no prefix. The path is
relative, so it resolves against the caller's own host (correct even behind a reverse proxy).
"""
prefix = "" if root_path == "/" else root_path
name = server.alias or server.server_name or server.name or server.server_id
return f"/.well-known/oauth-protected-resource{prefix}/mcp/{name}"
def raise_user_oauth_challenge(server: MCPServer, *, root_path: str) -> NoReturn:
"""Raise the 401 an ``authorization_code`` server returns at egress when the user has no token.
Points at the server's RFC 9728 Protected Resource Metadata (``resource_metadata``), which names
the upstream authorization server the client must complete OAuth with. The URL is per-server and
relative, so it resolves against the caller's own host (correct even behind a reverse proxy)
without needing request context. The listing-phase 401 still emits the RFC 8414 ``authorization_uri``
form pending the format unification; both target the same server, so the difference is cosmetic.
Points at the server's RFC 9728 Protected Resource Metadata, which names the upstream
authorization server the client must complete OAuth with. The listing-phase 401 still emits the
RFC 8414 ``authorization_uri`` form pending the format unification; both target the same server,
so the difference is cosmetic.
"""
from litellm.proxy.utils import get_server_root_path # noqa: PLC0415
root = get_server_root_path()
prefix = "" if root == "/" else root
name = server.alias or server.server_name or server.name or server.server_id
resource_metadata = f"/.well-known/oauth-protected-resource{prefix}/mcp/{name}"
resource_metadata = oauth_protected_resource_path(root_path, server)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"WWW-Authenticate": f'Bearer resource_metadata="{resource_metadata}"'},
)
def raise_token_exchange_challenge(
server: MCPServer,
*,
root_path: str,
claims: str | None = None,
) -> NoReturn:
"""Raise the RFC 9728 / RFC 6750 challenge an OBO (``token_exchange``) server returns when the
caller's subject token is missing or the IdP rejected it.
Points at the server's Protected Resource Metadata, whose ``authorization_servers`` names the IdP
the client must SSO with to obtain a subject token; ``error="invalid_token"`` tells a
spec-compliant MCP client to discover that AS and retry with a fresh bearer. Mirrors
``raise_user_oauth_challenge`` but for the exchange flow: there is no gateway-side browser OAuth —
the client re-authenticates directly with the IdP, and LiteLLM then exchanges the resulting token.
An IdP step-up rejection (Entra Conditional Access / CAE) passes its ``claims`` blob. Per the
Microsoft claims-challenge format the challenge then uses ``error="insufficient_claims"`` (the
value MSAL-family clients key on) and carries the claims base64-encoded in a ``claims`` parameter
the client replays to the IdP to satisfy the step-up. Without a claims blob the challenge keeps
``error="invalid_token"`` and is byte-identical to the static one. Both the error value (one of
two literals) and the base64 claims draw from a fixed alphabet, so nothing from the IdP body
reaches the header unescaped.
"""
resource_metadata = oauth_protected_resource_path(root_path, server)
encoded_claims = base64.b64encode(claims.encode()).decode() if claims else None
error = "insufficient_claims" if encoded_claims else "invalid_token"
error_description = (
"Step-up authentication required; satisfy the returned claims challenge with the IdP and retry"
if encoded_claims
else "Missing or invalid subject token; authenticate with the IdP and retry"
)
www_authenticate = ", ".join(
(
f'Bearer resource_metadata="{resource_metadata}"',
f'error="{error}"',
f'error_description="{error_description}"',
*((f'claims="{encoded_claims}"',) if encoded_claims else ()),
)
)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"WWW-Authenticate": www_authenticate},
)

71
litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
View file

@ -8,7 +8,8 @@ an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed
at runtime instead of returning `None`.
`none` and `api_key` (shared-key source) are live, as is `authorization_code`, which reads the
user's token from the injected `OAuthTokenStore`. The remaining arms are `not_implemented` stubs
user's token from the injected `OAuthTokenStore`, and `token_exchange`, which swaps the caller's
inbound token through the injected `TokenExchanger`. The remaining arms are `not_implemented` stubs
that each land in a follow-up PR with their seam. Pure v2: no imports from v1.
"""
@ -31,6 +32,9 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Ok,
Result,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import (
TokenExchanger,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ApiKeyConfig,
AuthorizationCodeConfig,
@ -55,16 +59,36 @@ class _NullOAuthTokenStore:
return None
class _NullTokenExchanger:
"""Fail-closed default: with no exchanger wired, token_exchange cannot produce a credential."""
async def exchange(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> Result[OAuthToken, CredError]:
return Error(CredError.of_misconfigured("token exchange collaborator not wired"))
async def invalidate(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> None:
return None
class UpstreamCredentialProvider:
"""Produces the one `httpx.Auth` for a `(subject, upstream)` pair, per declared mode.
Collaborators (the per-mode credential stores and token fetchers) are injected as each arm is
built; the live `none` and `api_key`-shared arms read from the config and need none, while
`authorization_code` reads the user's token from the injected `OAuthTokenStore`.
`authorization_code` reads the user's token from the injected `OAuthTokenStore` and
`token_exchange` swaps the caller's token through the injected `TokenExchanger`.
"""
def __init__(self, oauth_token_store: OAuthTokenStore | None = None) -> None:
def __init__(
self,
oauth_token_store: OAuthTokenStore | None = None,
token_exchanger: TokenExchanger | None = None,
) -> None:
self._oauth_token_store: OAuthTokenStore = oauth_token_store or _NullOAuthTokenStore()
self._token_exchanger: TokenExchanger = token_exchanger or _NullTokenExchanger()
async def resolve_credentials(self, subject: Subject, server: ServerSpec) -> Result[httpx.Auth, CredError]:
match server.config:
@ -76,8 +100,8 @@ class UpstreamCredentialProvider:
return _not_implemented(AuthSpecKind.passthrough)
case ClientCredentialsConfig():
return _not_implemented(AuthSpecKind.client_credentials)
case TokenExchangeConfig():
return _not_implemented(AuthSpecKind.token_exchange)
case TokenExchangeConfig() as config:
return await self._token_exchange(subject, server, config)
case AuthorizationCodeConfig():
return await self._authorization_code(subject, server)
case AwsSigV4Config():
@ -110,6 +134,43 @@ class UpstreamCredentialProvider:
return Error(CredError.of_unauthorized("Authorization required: complete the OAuth flow for this server."))
return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
async def _token_exchange(
self, subject: Subject, server: ServerSpec, config: TokenExchangeConfig
) -> Result[StaticHeaderAuth, CredError]:
"""RFC 8693 OBO: exchange the caller's inbound token for an upstream-bound bearer.
No inbound token means there is nothing to exchange, so the arm fails closed with a 401 rather
than falling through to a weaker source (§1.5); the exchanger handles the IdP round-trip and
caching and returns the upstream token or a typed error.
"""
inbound = subject.inbound_token
if inbound is None:
return Error(
CredError.of_unauthorized(
"Token exchange requires a caller token to exchange (OBO).",
www_authenticate='Bearer error="invalid_request"',
)
)
match await self._token_exchanger.exchange(
inbound.get_secret_value(), server, config, tenant_id=subject.tenant_id
):
case Ok(token):
return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
case Error(err):
return Error(err)
async def invalidate_credentials(self, subject: Subject, server: ServerSpec) -> None:
"""Drop any cached credential the resolver owns for this `(subject, server)`.
Used after an upstream rejects the injected credential, so the next resolve re-mints rather
than serving the same rejected token until TTL. Only `token_exchange` holds a re-mintable
cached credential here; other modes are a no-op.
"""
if isinstance(server.config, TokenExchangeConfig) and subject.inbound_token is not None:
await self._token_exchanger.invalidate(
subject.inbound_token.get_secret_value(), server, server.config, tenant_id=subject.tenant_id
)
async def _authz_token(self, subject: Subject, server: ServerSpec) -> OAuthToken | None:
"""The user's authorization_code token, or None when absent or the store is unreachable.

114
litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py Normal file
View file

@ -0,0 +1,114 @@
"""Composition root for the v2-native token_exchange (OBO) exchanger.
Wires the pure ``OboTokenExchanger`` to its runtime edges: the real httpx POST against the IdP and
the configured cache sizing/TTL constants. ``build_token_exchanger`` is built once at egress
construction and reused, so the in-process exchanged-token cache survives across requests. Unlike the
per-user store, nothing here reads a runtime global at build time (the httpx client is acquired per
call), so it needs no lazy wrapper.
"""
from __future__ import annotations
import httpx
from litellm._logging import verbose_logger
from litellm.constants import (
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
InMemoryTokenCacheBackend,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import (
OboTokenExchanger,
SubjectTokenRejected,
TokenExchangeClientError,
)
# RFC 6749 5.2 error codes that mean the gateway's own request/credentials are wrong (not the
# caller's subject token), so they surface as a 500 the caller can't fix by re-authenticating.
_GATEWAY_FAULT_OAUTH_ERRORS = frozenset(
{"invalid_client", "unauthorized_client", "unsupported_grant_type", "invalid_target", "invalid_scope"}
)
def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | None]:
"""Read the RFC 6749 5.2 ``error`` code and the IdP's step-up ``claims`` blob from a
token-endpoint error body, as ``(error, claims)`` with None for whatever is absent.
``claims`` is the Entra Conditional Access / CAE challenge (a JSON string the client must
replay to the IdP to satisfy the step-up); it is the caller's own requirement, not an IdP
internal, so it may travel to the caller. The ``error_description`` is deliberately not read:
it can carry IdP internals and must never reach the caller.
"""
try:
body: object = response.json()
except Exception: # noqa: BLE001
return None, None
if not isinstance(body, dict):
return None, None
code = body.get("error")
claims = body.get("claims")
return (
code if isinstance(code, str) else None,
claims if isinstance(claims, str) and claims else None,
)
async def _post_exchange_endpoint(
url: str, form: dict[str, str], client_auth_headers: dict[str, str]
) -> dict[str, object] | None:
from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415
get_async_httpx_client, # pyright: ignore
)
from litellm.types.llms.custom_http import httpxSpecialProvider # noqa: PLC0415
# litellm's httpx handler and httpx.Response are only partially typed; the IdP returns a JSON
# object and the exchanger validates each field, so the untyped boundary is contained here.
# A 4xx is the IdP rejecting the subject (non-retryable -> 401 via SubjectTokenRejected); any
# other failure is a miss (-> None -> upstream_unavailable -> 503), matching v1's fail-closed.
headers = {"Accept": "application/json", **client_auth_headers}
try:
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) # pyright: ignore
response = await client.post(url, headers=headers, data=form) # pyright: ignore
response.raise_for_status() # pyright: ignore
parsed: object = response.json() # pyright: ignore
except httpx.HTTPStatusError as status_err:
status_code = status_err.response.status_code
if 400 <= status_code < 500:
oauth_error, claims = _oauth_error_fields(status_err.response)
if oauth_error in _GATEWAY_FAULT_OAUTH_ERRORS:
verbose_logger.warning(
"MCP token exchange rejected as %s (HTTP %d); check the gateway client credentials, "
"audience, and scope for this server",
oauth_error,
status_code,
)
raise TokenExchangeClientError(oauth_error) from status_err
raise SubjectTokenRejected(
f"IdP rejected the subject token (HTTP {status_code})",
claims=claims,
) from status_err
verbose_logger.warning("MCP token exchange request failed: %s", status_err)
return None
except Exception as exc: # noqa: BLE001
verbose_logger.warning("MCP token exchange request failed: %s", exc)
return None
if not isinstance(parsed, dict):
# A valid-but-non-object JSON body (list/string/number) would crash the field parsing; map it
# to a miss so it surfaces as a typed upstream_unavailable, not a 500.
verbose_logger.warning("MCP token exchange returned non-object JSON (%s)", type(parsed).__name__)
return None
return parsed # pyright: ignore
def build_token_exchanger() -> OboTokenExchanger:
return OboTokenExchanger(
_post_exchange_endpoint,
cache=InMemoryTokenCacheBackend(max_size=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE),
default_ttl_seconds=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
min_ttl_seconds=MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
expiry_buffer_seconds=MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
)

376
litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py Normal file
View file

@ -0,0 +1,376 @@
"""v2-native OBO token exchange: swap the caller's token for an upstream-bound one.
The pure core of the ``token_exchange`` mode. Given the caller's inbound token and the server's
``TokenExchangeConfig``, ``OboTokenExchanger.exchange`` POSTs the grant selected by ``config.profile``
to the configured endpoint and returns the upstream-bound ``access_token`` as a typed ``OAuthToken``,
or a typed ``CredError`` - never a raise (the HTTP edge is the injected ``ExchangeHttpPost``, whose
adapter contains the I/O). Two profiles share this one engine: ``rfc8693`` (the RFC 8693 token-exchange
grant) and ``entra_obo`` (Microsoft Entra On-Behalf-Of, which is the RFC 7523 ``jwt-bearer`` grant);
only the request form differs, so the cache, single-flight, and TTL machinery are dialect-agnostic. The
exchanged token is cached and single-flighted per ``(subject_token, tenant, config, server)`` so a
repeated caller token skips the IdP round-trip and concurrent calls collapse to one exchange, reusing
the shared in-process cache + coordinator foundation. A rotated caller token hashes to a new key and
re-exchanges. Pure v2 apart from the shared RFC 6749 client-auth helper, which carries no v1 state.
A missing/expired exchange is an error, never a fall-through to a weaker source (§1.5): the caller
presenting no token is the resolver arm's 401, and an IdP that does not return a usable token is an
``upstream_unavailable`` here.
"""
from __future__ import annotations
import hashlib
import time
from collections.abc import Awaitable, Callable
from typing import Literal, Protocol
from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
InMemoryTokenCacheBackend,
InProcessRefreshCoordinator,
OAuthToken,
RefreshCoordinator,
TokenCacheBackend,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Error,
Ok,
Result,
)
from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import (
build_token_endpoint_client_auth,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
CredError,
ServerSpec,
TokenExchangeConfig,
)
# A token with no declared expiry is cached for this long; one with an expiry is cached until then
# minus the skew buffer, floored at the minimum. Values mirror v1's MCP_OAUTH2_* constants; the
# composition root injects the configured ones.
_DEFAULT_TTL_SECONDS = 3600.0
_MIN_TTL_SECONDS = 10.0
_EXPIRY_BUFFER_SECONDS = 60.0
_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
# Microsoft Entra On-Behalf-Of speaks the RFC 7523 jwt-bearer grant, not RFC 8693, and gates delegation
# behind ``requested_token_use=on_behalf_of`` (a Microsoft extension present in neither RFC).
_JWT_BEARER_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer"
_REQUESTED_TOKEN_USE_OBO = "on_behalf_of"
# RFC 8693 3 token-type URNs that are not usable as an upstream Bearer access token. token_type
# already rejects the common non-access case (N_A); this catches a malformed STS that mints one of
# these but still labels it Bearer. An access_token / jwt / absent / unknown type is accepted (lenient).
_NON_ACCESS_ISSUED_TOKEN_TYPES = frozenset(
{
"urn:ietf:params:oauth:token-type:refresh_token",
"urn:ietf:params:oauth:token-type:id_token",
"urn:ietf:params:oauth:token-type:saml1",
"urn:ietf:params:oauth:token-type:saml2",
}
)
# The IdP returns an opaque JSON object; the post adapter hands it over untyped and the exchanger
# validates each field, so no Any leaks past this seam (None == any transport/HTTP failure). The
# second dict is the form body; the third is the client-auth headers (HTTP Basic for
# client_secret_basic, empty for client_secret_post).
ExchangeHttpPost = Callable[[str, "dict[str, str]", "dict[str, str]"], Awaitable["dict[str, object] | None"]]
class SubjectTokenRejected(Exception):
"""The IdP refused to exchange the subject token (an RFC 8693 4xx, e.g. ``invalid_grant``).
Distinct from a transport / IdP-availability failure, which the post adapter maps to ``None`` ->
``upstream_unavailable`` -> 503 (retryable). A rejected subject is the caller's problem, not the
gateway's, so the arm surfaces it as a non-retryable 401 (the OBO challenge) instead.
``claims`` is the IdP's step-up challenge blob (Entra Conditional Access / CAE) from the
rejection body; it threads into the 401 challenge so the client can satisfy the step-up and
retry. The ``error_description`` is never carried (it can leak IdP internals).
"""
def __init__(self, detail: str, *, claims: str | None = None) -> None:
super().__init__(detail)
self.claims = claims
class TokenExchangeClientError(Exception):
"""The IdP rejected the exchange for a reason that is the gateway's fault, not the caller's.
RFC 6749 5.2 codes such as ``invalid_client`` (the gateway's own STS credentials are wrong),
``unauthorized_client`` / ``unsupported_grant_type`` (the gateway is not permitted to exchange),
``invalid_target`` / ``invalid_scope`` (the gateway's audience/scope config for this server is
wrong). The caller cannot fix these by re-authenticating, so the arm surfaces them as a 500
(``misconfigured``), not the 401 OBO challenge. The IdP ``error_description`` is never carried.
"""
class TokenExchanger(Protocol):
"""Exchanges a caller token for an upstream-bound one, per the server's token_exchange config."""
async def exchange(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> Result[OAuthToken, CredError]: ...
async def invalidate(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> None: ...
def _cache_key(subject_token: str, tenant_id: str, config: TokenExchangeConfig) -> str:
"""Bind the cache entry to the caller token, the tenant, AND the exchange config that minted it.
A rotated caller token, a different tenant, profile, endpoint, audience, scope, client_id, secret,
auth method, or subject_token_type all change the key, so two tenants behind the same opaque token
never share an entry and a config change (including a profile flip that alters the wire form)
forces a fresh exchange instead of serving a token minted for the old config until TTL. Everything
is hashed, so no secret is held in the key.
"""
secret = config.client_secret.get_secret_value() if config.client_secret else ""
material = "\x00".join(
(
subject_token,
tenant_id,
config.profile,
config.token_exchange_endpoint or "",
config.audience or "",
config.subject_token_type,
config.client_id or "",
secret,
config.token_endpoint_auth_method or "",
" ".join(config.scopes),
)
)
return hashlib.sha256(material.encode()).hexdigest()
def _parse_expires_in(raw: object) -> int | None:
if isinstance(raw, bool):
return None
if isinstance(raw, (int, float)):
return int(raw)
if isinstance(raw, str):
try:
return int(float(raw))
except ValueError:
return None
return None
def _rfc8693_form(
*,
subject_token: str,
subject_token_type: str,
audience: str | None,
scopes: tuple[str, ...],
) -> dict[str, str]:
return {
"grant_type": _GRANT_TYPE,
"subject_token": subject_token,
"subject_token_type": subject_token_type,
**({"audience": audience} if audience else {}),
**({"scope": " ".join(scopes)} if scopes else {}),
}
def _entra_obo_form(
*,
subject_token: str,
scopes: tuple[str, ...],
) -> dict[str, str]:
# Microsoft Entra On-Behalf-Of (RFC 7523 jwt-bearer, not RFC 8693): the caller's inbound access
# token rides as ``assertion`` (its ``aud`` must be this gateway's ``client_id``); the target
# resource is carried in ``scope`` (e.g. api://<app-id>/.default), since Entra has no audience
# parameter and ignores subject_token_type; ``requested_token_use=on_behalf_of`` is the Microsoft
# extension that turns the jwt-bearer grant into a delegation. ``scope`` is required, and the
# exchange precondition rejects an empty one, so it is always present here. Client authentication
# (client_id/client_secret via post, or Basic) is layered on by the caller through
# build_token_endpoint_client_auth, so it is not built into the form here.
return {
"grant_type": _JWT_BEARER_GRANT_TYPE,
"assertion": subject_token,
"scope": " ".join(scopes),
"requested_token_use": _REQUESTED_TOKEN_USE_OBO,
}
def _build_exchange_form(
*,
profile: Literal["rfc8693", "entra_obo"],
subject_token: str,
subject_token_type: str,
audience: str | None,
scopes: tuple[str, ...],
) -> dict[str, str]:
match profile:
case "rfc8693":
return _rfc8693_form(
subject_token=subject_token,
subject_token_type=subject_token_type,
audience=audience,
scopes=scopes,
)
case "entra_obo":
return _entra_obo_form(
subject_token=subject_token,
scopes=scopes,
)
assert_never(profile)
class OboTokenExchanger:
"""``TokenExchanger`` that runs the profile's OBO grant once per caller token, then caches the result.
The HTTP post is injected (``None`` on any IdP failure, mirroring v1: a failed exchange is a miss,
not a 500). The cache and single-flight coordinator default to the in-process foundation; a
deployment with no shared state needs nothing more (v1's exchanged-token cache is per-process too).
The clock is injected so TTL/expiry is deterministic in tests.
"""
def __init__(
self,
http_post: ExchangeHttpPost,
*,
cache: TokenCacheBackend | None = None,
coordinator: RefreshCoordinator | None = None,
clock: Callable[[], float] = time.time,
default_ttl_seconds: float = _DEFAULT_TTL_SECONDS,
min_ttl_seconds: float = _MIN_TTL_SECONDS,
expiry_buffer_seconds: float = _EXPIRY_BUFFER_SECONDS,
) -> None:
self._http_post = http_post
self._cache: TokenCacheBackend = cache or InMemoryTokenCacheBackend(clock=clock)
self._coordinator: RefreshCoordinator = coordinator or InProcessRefreshCoordinator()
self._clock = clock
self._default_ttl_seconds = default_ttl_seconds
self._min_ttl_seconds = min_ttl_seconds
self._expiry_buffer_seconds = expiry_buffer_seconds
async def exchange(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> Result[OAuthToken, CredError]:
endpoint = config.token_exchange_endpoint
client_id = config.client_id
client_secret = config.client_secret
if not endpoint:
# No endpoint configured and none discoverable: fail closed (412) rather than guess an IdP
# or fall back to a weaker source. The caller's token is never sent anywhere.
return Error(
CredError.of_precondition_required("token exchange endpoint is not configured for this server")
)
if not client_id or client_secret is None:
return Error(CredError.of_misconfigured("token_exchange requires client_id and client_secret"))
if config.profile == "entra_obo" and not config.scopes:
# Entra carries the target resource in ``scope`` (api://<app-id>/.default); with no scope the
# IdP cannot resolve an audience, so fail closed as misconfigured rather than POST a form the
# IdP will reject.
return Error(
CredError.of_misconfigured("entra_obo token exchange requires a scope (e.g. api://<app-id>/.default)")
)
cache_key = _cache_key(subject_token, tenant_id, config)
server_id = server.server_id
cached = await self._cache.get(cache_key, server_id)
if cached is not None:
verbose_logger.debug("MCP token exchange cache hit for server %s", server_id)
return Ok(cached)
client_auth = build_token_endpoint_client_auth(
auth_method=config.token_endpoint_auth_method,
client_id=client_id,
client_secret=client_secret.get_secret_value(),
)
form = {
**_build_exchange_form(
profile=config.profile,
subject_token=subject_token,
subject_token_type=config.subject_token_type,
audience=config.audience,
scopes=config.scopes,
),
**client_auth.body,
}
async def run_exchange() -> OAuthToken | None:
fresh = await self._cache.get(cache_key, server_id)
if fresh is not None:
return fresh
verbose_logger.debug(
"Exchanging token for MCP server %s at %s (audience=%s)", server_id, endpoint, config.audience
)
body = await self._http_post(endpoint, form, client_auth.headers)
if body is None:
return None
token = self._token_from_body(body)
if token is None:
return None
await self._cache.set(cache_key, server_id, token, self._ttl_seconds(token))
verbose_logger.info("Token exchange succeeded for MCP server %s", server_id)
return token
async def reread() -> OAuthToken | None:
return await self._cache.get(cache_key, server_id)
try:
token = await self._coordinator.run(cache_key, server_id, refresh=run_exchange, reread=reread)
except SubjectTokenRejected as rejected:
# The IdP rejected the subject token (4xx). This is non-retryable: the caller must
# re-authenticate with the IdP, so it surfaces as a 401 (the OBO challenge), not a 503.
# A step-up rejection (Entra Conditional Access) carries the claims blob through so the
# edge's challenge tells the client how to satisfy it.
return Error(
CredError.of_unauthorized(
str(rejected) or "subject token rejected by the IdP",
claims=rejected.claims,
)
)
except TokenExchangeClientError:
# RFC 6749 5.2 gateway-fault code (invalid_client / invalid_target / ...): the caller can't
# fix it by re-authenticating, so surface a 500 rather than the OBO 401 challenge. The
# specific code is logged at the edge; the user-facing summary stays generic.
return Error(
CredError.of_misconfigured(
"token exchange configuration error: the gateway's credentials, audience, or scope "
"for this server were not accepted by the IdP"
)
)
if token is None:
return Error(CredError.of_upstream_unavailable("token exchange did not return a usable access token"))
return Ok(token)
async def invalidate(
self, subject_token: str, server: ServerSpec, config: TokenExchangeConfig, *, tenant_id: str = ""
) -> None:
"""Drop the cached exchanged token so the next call re-exchanges (e.g. after an upstream 401)."""
await self._cache.delete(_cache_key(subject_token, tenant_id, config), server.server_id)
def _token_from_body(self, body: dict[str, object]) -> OAuthToken | None:
access_token = body.get("access_token")
if not isinstance(access_token, str) or not access_token:
return None
# token_type is forwarded downstream as Bearer, so a present-but-non-Bearer type (e.g. N_A)
# must fail closed rather than be minted as a bogus Bearer; an absent type defaults to Bearer.
token_type = body.get("token_type")
if isinstance(token_type, str) and token_type.strip().lower() != "bearer":
verbose_logger.warning(
"MCP token exchange returned unusable token_type %r; refusing to forward it as Bearer", token_type
)
return None
# issued_token_type says what representation was minted; reject a clearly-non-access type
# (refresh/id/saml) even if token_type claimed Bearer. access_token / jwt / absent / unknown pass.
issued_token_type = body.get("issued_token_type")
if isinstance(issued_token_type, str) and issued_token_type in _NON_ACCESS_ISSUED_TOKEN_TYPES:
return None
expires_in = _parse_expires_in(body.get("expires_in"))
expires_at = self._clock() + expires_in if expires_in is not None else None
return OAuthToken(access_token=access_token, expires_at=expires_at)
def _ttl_seconds(self, token: OAuthToken) -> float:
if token.expires_at is None:
return self._default_ttl_seconds
lifetime = max(0.0, token.expires_at - self._clock())
# Floor at min_ttl, but never cache past the token's own expiry: a token whose remaining
# lifetime is below the buffer (or even below min_ttl) must not be served stale upstream.
return min(max(lifetime - self._expiry_buffer_seconds, self._min_ttl_seconds), lifetime)

37
litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py
View file

@ -67,11 +67,15 @@ class Unauthorized:
``detail`` is the human message; ``www_authenticate`` and ``body`` carry a scheme-specific
challenge (e.g. BYOK's provisioning prompt) so the edge can reproduce it verbatim.
``claims`` carries an IdP step-up challenge (e.g. Entra Conditional Access) so the edge can
fold it into the ``WWW-Authenticate`` it builds; the client replays the claims to the IdP to
satisfy the step-up, then retries with the fresh token.
"""
detail: str
www_authenticate: str | None = None
body: Mapping[str, str] | None = None
claims: str | None = None
@tagged_union(frozen=True)
@ -104,8 +108,16 @@ class CredError:
*,
www_authenticate: str | None = None,
body: Mapping[str, str] | None = None,
claims: str | None = None,
) -> CredError:
return CredError(unauthorized=Unauthorized(detail=detail, www_authenticate=www_authenticate, body=body))
return CredError(
unauthorized=Unauthorized(
detail=detail,
www_authenticate=www_authenticate,
body=body,
claims=claims,
)
)
@staticmethod
def of_misconfigured(detail: str) -> CredError:
@ -182,18 +194,33 @@ class ClientCredentialsConfig(BaseModel):
class TokenExchangeConfig(BaseModel):
"""RFC 8693 OBO; swap the caller's live subject_token for a token bound to the upstream's
audience (`server.resource`, RFC 8707). The gateway authenticates to the exchange endpoint
as an OAuth client (`client_id`/`client_secret`); the inbound token is sent only to that
endpoint, never to the upstream.
"""OBO: swap the caller's live inbound token for a token bound to the upstream's audience. The
gateway authenticates to the exchange endpoint as an OAuth client (`client_id`/`client_secret`);
the inbound token is sent only to that endpoint, never to the upstream.
`profile` selects the wire dialect, since not every IdP speaks RFC 8693:
- `rfc8693` (default) is the standard token-exchange grant: the inbound token is the
`subject_token` (typed by `subject_token_type`), the target is the optional `audience`.
- `entra_obo` is Microsoft Entra On-Behalf-Of, which is the RFC 7523 `jwt-bearer` grant rather
than 8693: the inbound token rides as `assertion`, the target resource is carried in `scopes`
(`api://<app-id>/.default`, since Entra has no audience parameter), and the Microsoft-only
`requested_token_use=on_behalf_of` extension makes the jwt-bearer grant a delegation.
`subject_token_type` and `audience` are unused in this profile.
`audience` (rfc8693 only) is optional and sent only when the operator configured one, since both
`audience` and `resource` are optional in RFC 8693 and the authorization server applies its own
default when neither is sent (fabricating one risks `invalid_target`).
"""
model_config = ConfigDict(frozen=True)
kind: Literal[AuthSpecKind.token_exchange] = AuthSpecKind.token_exchange
profile: Literal["rfc8693", "entra_obo"] = "rfc8693"
subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token"
token_exchange_endpoint: str | None = None
audience: str | None = None
client_id: str | None = None
client_secret: SecretStr | None = None
token_endpoint_auth_method: Literal["client_secret_basic", "client_secret_post"] | None = None
scopes: tuple[str, ...] = ()

39
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -77,6 +77,7 @@ if MCP_AVAILABLE:
ListMCPToolsRestAPIResponseObject,
MCPInfo,
MCPServer,
_fire_mcp_success_logging,
_tool_name_matches,
execute_mcp_tool,
filter_tools_by_allowed_tools,
@ -84,6 +85,24 @@ if MCP_AVAILABLE:
########################################################
############ MCP Server REST API Routes #################
async def _safe_fire_mcp_success_logging(
logging_obj: Optional[Any],
result: Any,
start_time: datetime,
end_time: datetime,
) -> None:
if logging_obj is None:
return
logging_results = await asyncio.gather(
_fire_mcp_success_logging(logging_obj, result, start_time, end_time),
return_exceptions=True,
)
logging_error = logging_results[0]
if isinstance(logging_error, asyncio.CancelledError):
raise logging_error
if isinstance(logging_error, BaseException):
verbose_logger.warning("MCP tool success logging failed (continuing): %s", logging_error)
def _get_server_auth_header(
server,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
@ -798,7 +817,8 @@ if MCP_AVAILABLE:
proxy_logging_obj=proxy_logging_obj,
general_settings=general_settings,
)
return await handle_mcp_tool_call(
_tool_start_time = datetime.now()
result = await handle_mcp_tool_call(
tool_name=tool_arguments.get("tool_name", ""),
arguments=tool_arguments.get("arguments") or {},
user_api_key_dict=user_api_key_dict,
@ -809,6 +829,8 @@ if MCP_AVAILABLE:
raw_headers=virtual_raw_headers,
litellm_logging_obj=virtual_logging_obj,
)
await _safe_fire_mcp_success_logging(virtual_logging_obj, result, _tool_start_time, datetime.now())
return result
# Validate required parameters early
server_id = data.get("server_id")
@ -876,11 +898,12 @@ if MCP_AVAILABLE:
user_oauth_extra_headers = await _get_user_oauth_extra_headers(target_server, user_api_key_dict)
# Call execute_mcp_tool directly (permission checks already done)
_tool_start_time = datetime.now()
result = await execute_mcp_tool(
name=tool_name,
arguments=tool_arguments,
allowed_mcp_servers=allowed_mcp_servers,
start_time=datetime.now(),
start_time=_tool_start_time,
user_api_key_auth=data.get("user_api_key_auth"),
mcp_auth_header=data.get("mcp_auth_header"),
mcp_server_auth_headers=data.get("mcp_server_auth_headers"),
@ -889,6 +912,7 @@ if MCP_AVAILABLE:
litellm_logging_obj=data.get("litellm_logging_obj"),
requested_server_id=canonical_server_id,
)
await _safe_fire_mcp_success_logging(logging_obj, result, _tool_start_time, datetime.now())
return result
except MCPMissingUserEnvVarsError as e:
verbose_logger.info(
@ -1055,11 +1079,6 @@ if MCP_AVAILABLE:
forwarded_authorization = (
effective_oauth2_headers.get("Authorization") if effective_oauth2_headers else None
)
is_interactive_authz_code = (
server_model.auth_type == MCPAuth.oauth2
and forwarded_authorization is not None
and to_server_spec(server_model) is not None
)
preview_cred_provider = (
UpstreamCredentialProvider(
oauth_token_store=PresentedOAuthTokenStore(
@ -1070,7 +1089,11 @@ if MCP_AVAILABLE:
)
)
)
if is_interactive_authz_code
if (
server_model.auth_type == MCPAuth.oauth2
and forwarded_authorization is not None
and to_server_spec(server_model) is not None
)
else None
)

79
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -1681,6 +1681,7 @@ if MCP_AVAILABLE:
log_list_tools_to_spendlogs: bool = False,
list_tools_log_source: Optional[str] = None,
litellm_trace_id: Optional[str] = None,
request_tags: Optional[list[str]] = None,
client_ip: Optional[str] = None,
) -> List[MCPTool]:
"""
@ -1724,6 +1725,7 @@ if MCP_AVAILABLE:
"litellm_trace_id": effective_litellm_trace_id,
"metadata": {
"spend_logs_metadata": spend_logs_metadata,
**({"tags": request_tags} if request_tags else {}),
},
# Provide a small input payload for standard logging
"input": [
@ -1836,6 +1838,7 @@ if MCP_AVAILABLE:
add_prefix=True, # Always add server prefix
raw_headers=raw_headers,
user_api_key_auth=user_api_key_auth,
oauth2_headers=oauth2_headers,
)
filtered_tools = filter_tools_by_allowed_tools(tools, server)
@ -1858,7 +1861,8 @@ if MCP_AVAILABLE:
# tools. Surfacing the upstream 401 to the client as a re-auth challenge is
# intentionally not done here: raising from this list handler cannot produce a
# 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC
# error), so that belongs in a request-scope preemptive check, tracked separately.
# error). Single-server routes surface it via the request-scope preemptive
# check in _raise_preemptive_401_for_unauthenticated_servers instead.
verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth")
return []
except Exception as e:
@ -1899,7 +1903,9 @@ if MCP_AVAILABLE:
end_time = datetime.now()
try:
await litellm_logging_obj.async_success_handler(
result=all_tools,
result=[
tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools
],
start_time=list_tools_start_time,
end_time=end_time,
)
@ -2688,12 +2694,18 @@ if MCP_AVAILABLE:
# Forward named client headers to OpenAPI tool upstream requests.
# MCPServer.extra_headers lists header names to copy from raw_headers.
# OAuth2 M2M: never take Authorization from the caller (matches
# _prepare_mcp_server_headers for managed MCP).
# The strip decision is centralized in _should_strip_caller_authorization so this
# OpenAPI/local path agrees with the managed paths: M2M and the resolver-owned modes
# (token_exchange's raw subject token, authorization_code's stored token) must never
# have the caller's Authorization forwarded verbatim upstream.
forwarded_headers: Optional[Dict[str, str]] = None
if mcp_server and mcp_server.extra_headers and raw_headers:
normalized_raw = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
skip_caller_authorization = bool(mcp_server.has_client_credentials)
skip_caller_authorization = _should_strip_caller_authorization(
mcp_server=mcp_server,
raw_headers=raw_headers,
user_api_key_auth=user_api_key_auth,
)
for header_name in mcp_server.extra_headers:
if not isinstance(header_name, str):
continue
@ -2741,6 +2753,22 @@ if MCP_AVAILABLE:
return response
async def _fire_mcp_success_logging(
logging_obj: LiteLLMLoggingObj,
result: Any,
start_time: datetime,
end_time: datetime,
) -> None:
logging_obj.post_call(original_response=result)
await logging_obj.async_post_mcp_tool_call_hook(
kwargs=logging_obj.model_call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
logging_obj.call_type = CallTypes.call_mcp_tool.value
await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
@client
async def call_mcp_tool(
name: str,
@ -2812,16 +2840,7 @@ if MCP_AVAILABLE:
raise
if litellm_logging_obj:
litellm_logging_obj.post_call(original_response=response)
end_time = datetime.now()
await litellm_logging_obj.async_post_mcp_tool_call_hook(
kwargs=litellm_logging_obj.model_call_details,
response_obj=response,
start_time=start_time,
end_time=end_time,
)
litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value
await litellm_logging_obj.async_success_handler(result=response, start_time=start_time, end_time=end_time)
await _fire_mcp_success_logging(litellm_logging_obj, response, start_time, datetime.now())
return response
async def mcp_get_prompt(
@ -3455,6 +3474,36 @@ if MCP_AVAILABLE:
headers={"www-authenticate": authorization_uri},
)
# token_exchange (OBO): the caller supplied no subject token. Challenge at connect
# (transport level, where WWW-Authenticate survives) with the RFC 9728 resource_metadata
# so the client discovers the IdP, SSOs, and retries with a subject token, which LiteLLM
# then exchanges. A tool-call-time 401 would be wrapped into a JSON-RPC error and the
# header lost, so the discovery flow needs this pre-emptive challenge.
if server and server.auth_type == MCPAuth.oauth2_token_exchange and not oauth2_headers:
from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415
raise_token_exchange_challenge,
)
from litellm.proxy.utils import get_server_root_path # noqa: PLC0415
raise_token_exchange_challenge(server, root_path=get_server_root_path())
# token_exchange (OBO) with a subject present: run the exchange here at the transport
# edge, so a rejected subject raises the RFC 9728 challenge (and a gateway fault its
# public status) instead of the session opening and list_tools masking the failure as
# an empty tool list. Gated to single-server routes; the multi-server aggregate keeps
# absorbing per-server auth failures so one bad server cannot 401 the whole connect.
if (
server
and server.auth_type == MCPAuth.oauth2_token_exchange
and oauth2_headers
and len(mcp_servers or []) == 1
):
await global_mcp_server_manager.preflight_token_exchange(
server=server,
oauth2_headers=oauth2_headers,
user_api_key_auth=user_api_key_auth,
)
# Pass-through OAuth: when the admin has opted a server into
# forwarding the client's bearer token (is_oauth_passthrough) and
# the client hasn't supplied one, fail fast with 401 and point

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0ae3np_qb52e-.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0zam.8alu6_vj.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0uu6lckpr0s15.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0.bx44y-6~tug.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0-s2am3eulbyd.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0.w8~sa9q0n_s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0efmbzvj03niy.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/055egae-ggkjh.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0hwip5a7qsmis.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0mh1wnrvmv_y7.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0uuigwiz-in3~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-0c4mv4-mc9n.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0p3v32gvsxp6h.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0x09ws363q4_0.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/08o64zaid_juv.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0g9k1~ppf2hw3.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/151r-5htw45m~.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0v85l0arelm41.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0zbgu4ogb6mba.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

36
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}

14
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"]
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"}
:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/08yy42xvwaak6.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0e9hs7onyj28m.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/075sund.-mh4~.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"5rDiFx0t_mOGYmV_8kSkw"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"KYqiq5stbD-H4YcZ-6OuP"}

0
litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/KYqiq5stbD-H4YcZ-6OuP/_clientMiddlewareManifest.js
View file

Some files were not shown because too many files have changed in this diff Show more