scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'litellm_internal_staging' into litellm_local-rate-limit-fallbacks

Browse source
This commit is contained in:
Yuneng Jiang 2026-07-06 18:04:32 -07:00
commit e5103e0290
No known key found for this signature in database
1507 changed files with 61067 additions and 9531 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
.circleci/config.yml
View file

@ -5,6 +5,16 @@ orbs:
win: circleci/windows@5.0 # Add Windows orb
commands:
skip_if_unrelated_changes:
parameters:
category:
type: enum
enum: ["backend", "client"]
default: "backend"
steps:
- run:
name: "Skip job when no << parameters.category >>-relevant files changed"
command: bash .circleci/scripts/path_filter.sh << parameters.category >>
setup_google_dns:
steps:
- run:
@ -282,6 +292,7 @@ jobs:
parallelism: 4
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -354,6 +365,7 @@ jobs:
parallelism: 4
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -427,6 +439,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -480,6 +493,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -545,6 +559,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -584,6 +599,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -624,6 +640,7 @@ jobs:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -656,6 +673,7 @@ jobs:
FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -705,6 +723,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -755,6 +774,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -787,6 +807,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -832,6 +853,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -877,6 +899,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -918,6 +941,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -963,6 +987,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1007,6 +1032,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- restore_cache:
@ -1045,6 +1071,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1089,6 +1116,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1132,6 +1160,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1163,6 +1192,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1205,6 +1235,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1248,6 +1279,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1291,6 +1323,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1321,6 +1354,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1366,6 +1400,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1407,6 +1442,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- restore_cache:
keys:
@ -1459,6 +1495,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1482,6 +1519,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1507,6 +1545,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1531,6 +1570,7 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns
@ -1606,6 +1646,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1698,6 +1739,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns
@ -1746,13 +1788,13 @@ jobs:
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
-e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1787,6 +1829,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1832,13 +1875,13 @@ jobs:
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1869,6 +1912,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -1911,14 +1955,14 @@ jobs:
-e COHERE_API_KEY=$COHERE_API_KEY \
-e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \
-e GCS_FLUSH_INTERVAL="1" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -1960,13 +2004,13 @@ jobs:
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE="bad-license" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
--port 4000
- run:
name: Start outputting logs for second container
@ -2000,6 +2044,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2041,13 +2086,13 @@ jobs:
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e PROXY_BATCH_WRITE_AT=2 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2085,6 +2130,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2117,13 +2163,13 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Run Docker container 2
command: |
@ -2139,13 +2185,13 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4001 \
--detailed_debug
--port 4001
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2180,6 +2226,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2201,19 +2248,20 @@ jobs:
# the OTEL test - should get this as a trace
command: |
docker run -d \
--restart on-failure \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2252,6 +2300,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
# Remove Docker CLI installation since it's already available in machine executor
- install_uv
@ -2289,13 +2338,13 @@ jobs:
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e GCS_FLUSH_INTERVAL="1" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2333,6 +2382,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2365,14 +2415,14 @@ jobs:
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2471,6 +2521,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- run:
@ -2499,13 +2550,13 @@ jobs:
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
@ -2537,6 +2588,7 @@ jobs:
- *python312_image
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: .
# Check file locations
@ -2567,6 +2619,8 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
@ -2609,6 +2663,8 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
@ -2629,7 +2685,7 @@ jobs:
cd ui/litellm-dashboard
CI=true npm run test -- --run \
--pool forks --poolOptions.forks.maxForks=8
--pool forks --poolOptions.forks.maxForks=6
e2e_ui_testing:
docker:
@ -2654,6 +2710,8 @@ jobs:
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_uv
- restore_cache:
@ -2791,6 +2849,8 @@ jobs:
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_uv
- restore_cache:
@ -2892,6 +2952,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- run:
name: Build Docker image
@ -2917,6 +2978,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
- setup_google_dns

27
.circleci/scripts/classify_changes.sh Executable file
View file

@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: classify_changes.sh <backend|client>}"
has_client=false
has_backend=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
ui/*) has_client=true ;;
docs/* | *.md | *.mdx) : ;;
*) has_backend=true ;;
esac
done
case "$category" in
backend)
[ "$has_backend" = true ] && echo run || echo skip
;;
client)
{ [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
;;
*)
echo run
;;
esac

40
.circleci/scripts/path_filter.sh Executable file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: path_filter.sh <backend|client>}"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
run_full() {
echo "path-filter[$category]: running job ($1)"
exit 0
}
[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
candidate_bases="main litellm_internal_staging litellm_oss_staging"
merge_base=""
for base in $candidate_bases; do
git fetch --quiet origin "$base" 2>/dev/null || continue
candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue
[ -n "$candidate" ] || continue
if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then
merge_base="$candidate"
fi
done
[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases"
changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed"
[ -n "$changed" ] || run_full "no files changed vs $merge_base"
echo "path-filter[$category]: changed files vs ${merge_base}:"
printf '%s\n' "$changed" | sed 's/^/ /' || true
decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed"
if [ "$decision" = run ]; then
run_full "$category-relevant changes detected"
fi
echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful"
circleci-agent step halt

4
.git-blame-ignore-revs
View file

@ -13,7 +13,7 @@
7edf3a9cb55548b143df1692f4ed7c4681d7fcf7
# style: reformat litellm/ with ruff format (#31317)
430b5b8f1b12dc261a49fda99ac5d1b22381a428
17bfd415aeb5a57fb646b5cc67da1c730aa7c50b
# style: unify ruff format width on 120 (#31518)
3dfbeabe626d203ac9de86024519d9a96c484ce4
48b5a5a0cc5a694a11219416ee0b6eb6e620e74e

2
.github/pull_request_template.md vendored
View file

@ -4,7 +4,7 @@
## Linear ticket
<!-- if you are an internal contributor (e.g., your username is postfixed with -berri or -berriai), add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
## Pre-Submission checklist

32
.github/workflows/create-release.yml vendored
View file

@ -122,10 +122,28 @@ jobs:
makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
}
try {
await github.rest.git.createRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: `refs/tags/${tag}`,
sha: commitHash,
});
} catch (error) {
if (error.status !== 422) throw error;
const existing = await github.rest.git.getRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: `tags/${tag}`,
});
if (existing.data.object.sha !== commitHash) {
throw new Error(`Tag ${tag} already exists at ${existing.data.object.sha}, expected ${commitHash}`);
}
}
const response = await github.rest.repos.createRelease({
draft: true,
generate_release_notes: true,
target_commitish: commitHash,
name: tag,
owner: context.repo.owner,
prerelease: isPrerelease,
@ -138,11 +156,21 @@ jobs:
owner: context.repo.owner,
repo: context.repo.repo,
release_id: response.data.id,
tag_name: tag,
body: updatedBody,
draft: false,
make_latest: makeLatest,
});
if (!isPrerelease) {
await github.rest.repos.updateRelease({
owner: context.repo.owner,
repo: context.repo.repo,
release_id: response.data.id,
tag_name: tag,
make_latest: makeLatest,
});
}
} catch (error) {
core.setFailed(error.message);
}

4
.github/workflows/helm_unit_test.yml vendored
View file

@ -38,4 +38,6 @@ jobs:
echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
- name: Run unit tests
run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
run: |
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
helm unittest -f 'tests/*.yaml' helm/litellm

2
.github/workflows/test-linting.yml vendored
View file

@ -63,7 +63,7 @@ jobs:
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git diff --name-only "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
echo "No changed litellm Python files to check with ruff format."
exit 0

11
.gitignore vendored
View file

@ -50,6 +50,8 @@ litellm/proxy/tests/package-lock.json
ui/litellm-dashboard/.next
ui/litellm-dashboard/node_modules
ui/litellm-dashboard/next-env.d.ts
ui/litellm-dashboard/package.json
ui/litellm-dashboard/package-lock.json
deploy/charts/litellm/*.tgz
deploy/charts/litellm/charts/*
deploy/charts/*.tgz
@ -85,12 +87,17 @@ litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
test.py
litellm_config.yaml
!.github/observatory/litellm_config.yaml
.cursor
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
STABILIZATION_TODO.md
@ -124,6 +131,4 @@ crash.*.log
# pytest coverage data
.coverage
# _experimental/out UI build output
# (both componentized and non-componentized build the UI on project release)
litellm/proxy/_experimental/out/
ui/litellm-dashboard/out/

16
CLAUDE.md
View file

@ -17,9 +17,13 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
Always use @.github/pull_request_template.md as a guide for your PR body
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
@ -33,17 +37,21 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed
Python max line length is 120, not 88
When you fix violations gated by `ruff-strict-budget.json` or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
Commit and push your work when you're done without asking
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6

58
Makefile
View file

@ -4,8 +4,8 @@
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev format \
lint-basedpyright lint-basedpyright-budget-update \
info lint lint-dev lint-checks format \
lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety pre-commit \
@ -27,12 +27,12 @@ help:
@echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
@echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + basedpyright)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -53,6 +53,11 @@ help:
UV := uv
UV_RUN := $(UV) run --no-sync
LINT_DEP_INSTALL ?= install-dev
LINT_DEP_BASE ?= lint-fetch-base
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
# Show info
info:
@echo "UV: $(UV)"
@ -88,7 +93,7 @@ install-hooks:
# Formatting
# Wrap width is ruff.toml's single source of truth (line-length = 120), shared by the
# formatter, E501, and the import sorter so there's no 88-vs-120 split to reconcile.
# formatter and the import sorter so there's no 88-vs-120 split to reconcile.
format: install-dev
cd litellm && $(UV_RUN) ruff format --exclude '/enterprise/' . && cd ..
@ -107,12 +112,12 @@ lint-fetch-base:
# running proxy need.
lint-install:
$(UV) sync --inexact --frozen --group proxy-dev
$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
$(UV_RUN) python scripts/prisma_generate_if_needed.py
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
# only the litellm Python files changed vs the base are checked, so a pre-existing
# format issue elsewhere doesn't block an unrelated commit.
lint-format-check-changed: install-dev lint-fetch-base
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \
@ -121,7 +126,7 @@ lint-format-check-changed: install-dev lint-fetch-base
fi
# Linting targets
lint-ruff: install-dev
lint-ruff: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) ruff check . && cd ..
# faster linter for developing ...
@ -156,15 +161,17 @@ lint-ruff-FULL-dev: install-dev
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
else echo "No changed .py files to check."; fi
lint-basedpyright: install-dev lint-fetch-base
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: install-dev lint-fetch-base
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
lint-basedpyright-budget-update: install-dev
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev lint-fetch-base
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
lint-format: format-check
@ -174,19 +181,22 @@ lint-ruff-budget: install-dev
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
# means the CI check will pass too.
lint-gate: install-dev lint-fetch-base
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging
lint-ruff-budget-update: install-dev
lint-ruff-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/ruff_strict_gate.py --update
# Ratchet all budgets in one shot (ruff strict + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-basedpyright-budget-update
lint-type-discipline-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/type_discipline_gate.py --update
check-circular-imports: install-dev
# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
check-import-safety: $(LINT_DEP_INSTALL)
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
@ -194,9 +204,13 @@ check-import-safety: install-dev
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin/litellm_internal_staging). lint-install is first so the
# Prisma client exists before basedpyright runs.
lint: lint-install lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety

4
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

144
basedpyright-code-budget.json
View file

@ -1,194 +1,146 @@
{
"reportAny": {
"baseline": 24989,
"slack": 2500
"limit": 37484
},
"reportArgumentType": {
"baseline": 1814,
"slack": 180
"limit": 2704
},
"reportAssignmentType": {
"baseline": 220,
"slack": 22
"limit": 330
},
"reportAttributeAccessIssue": {
"baseline": 346,
"slack": 35
"limit": 516
},
"reportCallIssue": {
"baseline": 87,
"slack": 10
"limit": 124
},
"reportConstantRedefinition": {
"baseline": 39,
"slack": 4
"limit": 59
},
"reportDeprecated": {
"baseline": 217,
"slack": 22
"limit": 326
},
"reportDuplicateImport": {
"baseline": 28,
"slack": 3
"limit": 42
},
"reportExplicitAny": {
"baseline": 6931,
"slack": 700
"limit": 10397
},
"reportFunctionMemberAccess": {
"baseline": 7,
"slack": 3
"limit": 11
},
"reportGeneralTypeIssues": {
"baseline": 151,
"slack": 15
"limit": 227
},
"reportIncompatibleMethodOverride": {
"baseline": 52,
"slack": 5
"limit": 78
},
"reportIncompatibleVariableOverride": {
"baseline": 8,
"slack": 3
"limit": 12
},
"reportInconsistentOverload": {
"baseline": 12,
"slack": 3
"limit": 18
},
"reportIndexIssue": {
"baseline": 26,
"slack": 3
"limit": 37
},
"reportInvalidTypeForm": {
"baseline": 23,
"slack": 3
"limit": 35
},
"reportInvalidTypeVarUse": {
"baseline": 2,
"slack": 3
"limit": 5
},
"reportMatchNotExhaustive": {
"baseline": 1,
"slack": 0
"limit": 0
},
"reportMissingParameterType": {
"baseline": 3933,
"slack": 390
"limit": 5900
},
"reportMissingTypeArgument": {
"baseline": 10612,
"slack": 1000
"limit": 15918
},
"reportMissingTypeStubs": {
"baseline": 27,
"slack": 10
"limit": 41
},
"reportOperatorIssue": {
"baseline": 6,
"slack": 3
"limit": 0
},
"reportOptionalCall": {
"baseline": 4,
"slack": 3
"limit": 0
},
"reportOptionalIterable": {
"baseline": 3,
"slack": 3
"limit": 0
},
"reportOptionalMemberAccess": {
"baseline": 724,
"slack": 72
"limit": 1085
},
"reportOptionalOperand": {
"baseline": 3,
"slack": 3
"limit": 0
},
"reportOptionalSubscript": {
"baseline": 11,
"slack": 3
"limit": 0
},
"reportPossiblyUnboundVariable": {
"baseline": 52,
"slack": 10
"limit": 77
},
"reportPrivateUsage": {
"baseline": 1625,
"slack": 160
"limit": 2438
},
"reportRedeclaration": {
"baseline": 8,
"slack": 3
"limit": 12
},
"reportReturnType": {
"baseline": 126,
"slack": 100
"limit": 225
},
"reportTypedDictNotRequiredAccess": {
"baseline": 20,
"slack": 3
"limit": 27
},
"reportUndefinedVariable": {
"baseline": 2,
"slack": 3
"limit": 0
},
"reportUnknownArgumentType": {
"baseline": 30603,
"slack": 3000
"limit": 45895
},
"reportUnknownLambdaType": {
"baseline": 75,
"slack": 10
"limit": 113
},
"reportUnknownMemberType": {
"baseline": 27037,
"slack": 2500
"limit": 40541
},
"reportUnknownParameterType": {
"baseline": 13612,
"slack": 1000
"limit": 20418
},
"reportUnknownVariableType": {
"baseline": 21445,
"slack": 2000
"limit": 32151
},
"reportUnnecessaryCast": {
"baseline": 118,
"slack": 10
"limit": 177
},
"reportUnnecessaryComparison": {
"baseline": 683,
"slack": 100
"limit": 1025
},
"reportUnnecessaryContains": {
"baseline": 4,
"slack": 3
"limit": 7
},
"reportUnnecessaryIsInstance": {
"baseline": 808,
"slack": 80
"limit": 1212
},
"reportUntypedBaseClass": {
"baseline": 110,
"slack": 11
"limit": 165
},
"reportUntypedFunctionDecorator": {
"baseline": 22,
"slack": 3
"limit": 33
},
"reportUnusedClass": {
"baseline": 22,
"slack": 3
"limit": 33
},
"reportUnusedFunction": {
"baseline": 137,
"slack": 10
"limit": 206
},
"reportUnusedImport": {
"baseline": 670,
"slack": 50
"limit": 1005
},
"reportUnusedVariable": {
"baseline": 865,
"slack": 50
"limit": 1297
}
}

10
codecov.yaml
View file

@ -15,6 +15,16 @@ ignore:
flag_management:
default_rules:
carryforward: true
# Dead flags no CI job uploads anymore: their carried-forward sessions were
# measured against old revisions, and the stale line maps mark comment lines
# of since-edited files as missed, sinking patch coverage on unrelated PRs.
individual_flags:
- name: proxy-mgmt-behavior
carryforward: false
- name: security
carryforward: false
- name: proxy-db-schema-migration
carryforward: false
component_management:
individual_components:

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

2
docker/build_admin_ui.sh
View file

@ -57,8 +57,6 @@ source ~/.nvm/nvm.sh
nvm install v18.17.0
nvm use v18.17.0
# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
# cd in to /ui/litellm-dashboard
cd ui/litellm-dashboard

61
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -477,9 +477,12 @@ class BaseEmailLogger(CustomLogger):
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}"
# Check if we've already sent this alert
result = await _cache.async_get_cache(key=_cache_key)
if result is None:
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is None or send_count <= 1:
# Create WebhookEvent for soft budget alert
event_message = f"Soft Budget Crossed - Total Soft Budget: ${user_info.soft_budget}"
webhook_event = WebhookEvent(
@ -508,18 +511,12 @@ class BaseEmailLogger(CustomLogger):
await self.send_team_soft_budget_alert_email(webhook_event)
else:
await self.send_soft_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending soft budget alert email: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
return
# For max_budget_alert, check if we've already sent an alert
@ -545,9 +542,12 @@ class BaseEmailLogger(CustomLogger):
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:max_budget_alert:{_id}"
# Check if we've already sent this alert
result = await _cache.async_get_cache(key=_cache_key)
if result is None:
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is None or send_count <= 1:
# Calculate percentage
percentage = int(
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100
@ -576,18 +576,12 @@ class BaseEmailLogger(CustomLogger):
try:
await self.send_max_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending max budget alert email: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
return
async def _handle_multi_threshold_max_budget_alert(
@ -617,10 +611,6 @@ class BaseEmailLogger(CustomLogger):
f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
)
result = await _cache.async_get_cache(key=_cache_key)
if result is not None:
continue
# Parse emails + auto-include owner
emails = _parse_email_list(raw_emails)
if user_info.user_email:
@ -634,6 +624,14 @@ class BaseEmailLogger(CustomLogger):
continue
recipient_emails = list(set(emails))
send_count = await _cache.async_increment_cache(
key=_cache_key,
value=1,
ttl=EMAIL_BUDGET_ALERT_TTL,
)
if send_count is not None and send_count > 1:
continue
event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
webhook_event = WebhookEvent(
event="max_budget_alert",
@ -660,16 +658,21 @@ class BaseEmailLogger(CustomLogger):
threshold_pct=threshold_pct,
recipient_emails=recipient_emails,
)
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending multi-threshold max budget alert email for {threshold_pct}%: {e}",
exc_info=True,
)
await self._release_budget_alert_claim(_cache, _cache_key)
async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None:
try:
await cache.async_delete_cache(key=cache_key)
except Exception:
verbose_proxy_logger.debug(
"Failed to release budget alert claim for %s; it expires with the TTL",
cache_key,
)
async def _get_email_params(
self,

233
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -13,6 +13,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -26,6 +28,7 @@ class CheckBatchCost:
proxy_logging_obj: "ProxyLogging",
prisma_client: "PrismaClient",
llm_router: "Router",
track_unmanaged_vertex_batch_cost: bool = False,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -33,6 +36,7 @@ class CheckBatchCost:
self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
self._track_unmanaged_vertex_batch_cost = track_unmanaged_vertex_batch_cost
# Cached after the first poll cycle. Once we know the column is absent we skip
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
@ -97,6 +101,182 @@ class CheckBatchCost:
order={"created_at": "asc"},
)
@staticmethod
def _record_error(
prom_logger: Optional["PrometheusLogger"], error_type: str
) -> None:
if prom_logger is not None:
prom_logger.record_check_batch_cost_error(error_type)
def _resolve_job_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
"""
Resolve (model_id, batch_id) for a managed-object row, where model_id is a router
deployment id and batch_id is the raw provider batch id.
Managed batches encode both in a base64 unified id. Unmanaged Vertex batches, created with
a raw gs:// input_file_id, store the raw provider job id as unified_object_id; when
track_unmanaged_vertex_batch_cost is enabled the model is derived from the gs:// path and
mapped to a configured vertex_ai deployment. Returns None (recording a metric) when the row
can't be routed.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
unified_object_id = job.unified_object_id
decoded = _is_base64_encoded_unified_file_id(unified_object_id)
if decoded:
model_id = get_model_id_from_unified_batch_id(decoded)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
self._record_error(prom_logger, "invalid_model_id")
return None
return model_id, get_batch_id_from_unified_batch_id(decoded)
if self._track_unmanaged_vertex_batch_cost:
return self._resolve_unmanaged_vertex_routing(job, prom_logger)
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
def _resolve_unmanaged_vertex_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
from litellm.llms.vertex_ai.batches.transformation import (
VertexAIBatchTransformation,
)
input_file_id = self._get_input_file_id(job)
if not VertexAIBatchTransformation.is_unmanaged_gcs_batch_input_file_id(
input_file_id
):
verbose_proxy_logger.info(
f"Skipping job {job.unified_object_id}: not an unmanaged vertex batch "
"(no gs:// input_file_id with a publishers/ model path)"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
assert input_file_id is not None # narrowed by is_unmanaged_gcs_batch_input_file_id
bare_model_name = VertexAIBatchTransformation.get_bare_model_name_from_gcs_file(
input_file_id
)
deployment_id = self._get_vertex_ai_deployment_id_for_bare_model(
bare_model_name
)
if deployment_id is None:
verbose_proxy_logger.info(
f"Skipping unmanaged vertex batch {job.unified_object_id}: no vertex_ai "
f"deployment configured for model {bare_model_name}"
)
self._record_error(prom_logger, "unmanaged_no_matching_deployment")
return None
return deployment_id, job.unified_object_id
def _get_vertex_ai_deployment_id_for_bare_model(
self, bare_model_name: str
) -> Optional[str]:
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
deployment_id = (
self._get_vertex_ai_deployment_id(model_group) if model_group else None
)
if deployment_id is not None:
return deployment_id
return self._get_vertex_ai_deployment_id_from_matching_deployments(
bare_model_name
)
def _get_vertex_ai_deployment_id_from_matching_deployments(
self, bare_model_name: str
) -> Optional[str]:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment in self.llm_router.get_model_list(model_name=None) or []:
litellm_params = deployment.get("litellm_params") or {}
actual_model = litellm_params.get("model")
if not isinstance(actual_model, str):
continue
if not self._is_bare_model_match(actual_model, bare_model_name):
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=actual_model,
custom_llm_provider=litellm_params.get("custom_llm_provider"),
)
except Exception:
continue
if llm_provider != "vertex_ai":
continue
model_info = deployment.get("model_info") or {}
deployment_id = model_info.get("id")
if isinstance(deployment_id, str):
return deployment_id
return None
@staticmethod
def _is_bare_model_match(actual_model: str, bare_model_name: str) -> bool:
return (
actual_model == bare_model_name
or actual_model.endswith(f"/{bare_model_name}")
or actual_model.endswith(f":{bare_model_name}")
)
def _get_vertex_ai_deployment_id(self, model_group: str) -> Optional[str]:
"""
Returns the first deployment id for `model_group` whose provider is vertex_ai,
skipping deployments from other providers that happen to share the model group name.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
if deployment_info is None:
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=deployment_info.litellm_params.model,
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
except Exception:
continue
if llm_provider == "vertex_ai":
return deployment_id
return None
@staticmethod
def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object = job.file_object
if isinstance(file_object, str):
try:
file_object = json.loads(file_object)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(file_object, dict):
return None
try:
return LiteLLMBatch.model_validate(file_object).input_file_id
except Exception:
return None
async def check_batch_cost(self):
"""
Check if the batch JOB has been tracked.
@ -114,8 +294,6 @@ class CheckBatchCost:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
try:
@ -172,31 +350,10 @@ class CheckBatchCost:
else:
jobs = await self._fallback_find_jobs()
for job in jobs:
# get the model from the job
unified_object_id = job.unified_object_id
decoded_unified_object_id = _is_base64_encoded_unified_file_id(
unified_object_id
)
if not decoded_unified_object_id:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_unified_id")
continue
else:
unified_object_id = decoded_unified_object_id
model_id = get_model_id_from_unified_batch_id(unified_object_id)
batch_id = get_batch_id_from_unified_batch_id(unified_object_id)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_model_id")
routing = self._resolve_job_routing(job, prom_logger)
if routing is None:
continue
model_id, batch_id = routing
verbose_proxy_logger.info(
f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}"
@ -213,7 +370,7 @@ class CheckBatchCost:
)
except Exception as e:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
f"Skipping job {job.unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
@ -287,7 +444,7 @@ class CheckBatchCost:
deployment_info = self.llm_router.get_deployment(model_id=model_id)
if deployment_info is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid deployment info"
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("deployment_not_found")
@ -413,6 +570,26 @@ class CheckBatchCost:
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
elif response.status in ("failed", "expired", "cancelled"):
try:
update_data = {
"status": response.status,
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=update_data,
)
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
)
# Record polling run metrics (always, even if nothing was processed)
if prom_logger:
prom_logger.record_check_batch_cost_run(

16
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -125,23 +125,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
update_data = {
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"updated_by": user_api_key_dict.user_id,
}
if file_object is not None:
db_data["file_object"] = file_object.model_dump_json()
file_object_json = file_object.model_dump_json()
db_data["file_object"] = file_object_json
update_data["file_object"] = file_object_json
# Extract storage metadata from hidden params if present
hidden_params = getattr(file_object, "_hidden_params", {}) or {}
if "storage_backend" in hidden_params:
db_data["storage_backend"] = hidden_params["storage_backend"]
update_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
update_data["storage_url"] = hidden_params["storage_url"]
verbose_logger.debug(
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
f"storage_url={db_data.get('storage_url')}"
)
result = await self.prisma_client.db.litellm_managedfiletable.create(
data=db_data
result = await self.prisma_client.db.litellm_managedfiletable.upsert(
where={"unified_file_id": file_id},
data={"create": db_data, "update": update_data},
)
verbose_logger.debug(
f"LiteLLM Managed File object with id={file_id} stored in db: {result}"

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.45"
version = "0.1.47"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.45"
version = "0.1.47"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

14
helm/litellm/templates/backend/deployment.yaml
View file

@ -45,11 +45,16 @@ spec:
value: /app/config/config.yaml
{{- end }}
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
{{- if .Values.gateway.config.create }}
{{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.backend.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.backend.livenessProbe }}
livenessProbe:
@ -61,11 +66,16 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
{{- if or .Values.gateway.config.create .Values.backend.volumes }}
volumes:
{{- if .Values.gateway.config.create }}
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.backend.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- with .Values.backend.nodeSelector }}
nodeSelector:

14
helm/litellm/templates/gateway/deployment.yaml
View file

@ -47,11 +47,16 @@ spec:
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if .Values.gateway.config.create }}
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.gateway.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
@ -63,11 +68,16 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
{{- if or .Values.gateway.config.create .Values.gateway.volumes }}
volumes:
{{- if .Values.gateway.config.create }}
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.gateway.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:

8
helm/litellm/templates/ui/deployment.yaml
View file

@ -46,6 +46,10 @@ spec:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.ui | nindent 10 }}
{{- with .Values.ui.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
@ -56,6 +60,10 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.ui.resources | nindent 12 }}
{{- with .Values.ui.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

172
helm/litellm/tests/deployment_volumes_tests.yaml Normal file
View file

@ -0,0 +1,172 @@
suite: test deployment volumes and volumeMounts
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
- backend/deployment.yaml
- ui/deployment.yaml
values:
- ./values/required.yaml
tests:
- it: gateway renders only the config volume by default
template: gateway/deployment.yaml
asserts:
- equal:
path: spec.template.spec.volumes
value:
- name: gateway-config
configMap:
name: RELEASE-NAME-litellm-gateway-config
- equal:
path: spec.template.spec.containers[0].volumeMounts
value:
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
- it: gateway merges user volumes and volumeMounts with the config volume
template: gateway/deployment.yaml
set:
gateway.volumes:
- name: custom-callbacks
configMap:
name: custom-callbacks
gateway.volumeMounts:
- name: custom-callbacks
mountPath: /app/custom_callbacks.py
subPath: custom_callbacks.py
asserts:
- equal:
path: spec.template.spec.volumes[0].name
value: gateway-config
- equal:
path: spec.template.spec.volumes[1]
value:
name: custom-callbacks
configMap:
name: custom-callbacks
- equal:
path: spec.template.spec.containers[0].volumeMounts[0].name
value: gateway-config
- equal:
path: spec.template.spec.containers[0].volumeMounts[1]
value:
name: custom-callbacks
mountPath: /app/custom_callbacks.py
subPath: custom_callbacks.py
- it: gateway renders user volumes even when config creation is disabled
template: gateway/deployment.yaml
set:
gateway.config.create: false
gateway.volumes:
- name: certs
secret:
secretName: tls-certs
gateway.volumeMounts:
- name: certs
mountPath: /etc/certs
readOnly: true
asserts:
- equal:
path: spec.template.spec.volumes
value:
- name: certs
secret:
secretName: tls-certs
- equal:
path: spec.template.spec.containers[0].volumeMounts
value:
- name: certs
mountPath: /etc/certs
readOnly: true
- it: gateway omits volumes when config creation is disabled and no user volumes are set
template: gateway/deployment.yaml
set:
gateway.config.create: false
asserts:
- isNull:
path: spec.template.spec.volumes
- isNull:
path: spec.template.spec.containers[0].volumeMounts
- it: backend merges user volumes and volumeMounts with the shared config volume
template: backend/deployment.yaml
set:
backend.volumes:
- name: sso-handler
configMap:
name: sso-handler
backend.volumeMounts:
- name: sso-handler
mountPath: /app/custom_sso.py
subPath: custom_sso.py
asserts:
- equal:
path: spec.template.spec.volumes[0].name
value: gateway-config
- equal:
path: spec.template.spec.volumes[1]
value:
name: sso-handler
configMap:
name: sso-handler
- equal:
path: spec.template.spec.containers[0].volumeMounts[1]
value:
name: sso-handler
mountPath: /app/custom_sso.py
subPath: custom_sso.py
- it: backend renders user volumes even when config creation is disabled
template: backend/deployment.yaml
set:
gateway.config.create: false
backend.volumes:
- name: data
emptyDir: {}
backend.volumeMounts:
- name: data
mountPath: /data
asserts:
- equal:
path: spec.template.spec.volumes
value:
- name: data
emptyDir: {}
- equal:
path: spec.template.spec.containers[0].volumeMounts
value:
- name: data
mountPath: /data
- it: ui renders no volumes by default
template: ui/deployment.yaml
asserts:
- isNull:
path: spec.template.spec.volumes
- isNull:
path: spec.template.spec.containers[0].volumeMounts
- it: ui renders user volumes and volumeMounts
template: ui/deployment.yaml
set:
ui.volumes:
- name: nginx-config
configMap:
name: custom-nginx
ui.volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/conf.d
asserts:
- equal:
path: spec.template.spec.volumes
value:
- name: nginx-config
configMap:
name: custom-nginx
- equal:
path: spec.template.spec.containers[0].volumeMounts
value:
- name: nginx-config
mountPath: /etc/nginx/conf.d

4
helm/litellm/tests/values/required.yaml Normal file
View file

@ -0,0 +1,4 @@
database:
writer:
host: postgres.example.com
dbname: litellm

13
helm/litellm/values.yaml
View file

@ -124,6 +124,11 @@ gateway:
extraEnv: [] # Add extra environment variables to the gateway
envConfigMaps: [] # Add extra environment variables to the gateway from config maps
envSecrets: [] # Add extra environment variables to the gateway from secrets
# Additional volumes on the gateway Deployment (e.g. a ConfigMap holding
# custom callback / SSO handler code, mounted next to the proxy config).
volumes: []
# Additional volumeMounts on the gateway container.
volumeMounts: []
config:
create: true
proxy_config: {}
@ -167,6 +172,10 @@ backend:
extraEnv: []
envConfigMaps: []
envSecrets: []
# Additional volumes on the backend Deployment.
volumes: []
# Additional volumeMounts on the backend container.
volumeMounts: []
image:
repository: ghcr.io/berriai/litellm-backend
tag: ""
@ -206,6 +215,10 @@ ui:
extraEnv: []
envConfigMaps: []
envSecrets: []
# Additional volumes on the ui Deployment.
volumes: []
# Additional volumeMounts on the ui container.
volumeMounts: []
image:
repository: ghcr.io/berriai/litellm-ui
tag: ""

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260629000000_add_max_concurrent_requests_to_mcp_server_table/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "max_concurrent_requests" INTEGER;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260630190000_add_budget_fallbacks_to_litellm_verification_token/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';

3
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -338,6 +338,7 @@ model LiteLLM_MCPServerTable {
byok_api_key_help_url String?
source_url String?
timeout Float?
max_concurrent_requests Int?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
@ -418,6 +419,7 @@ model LiteLLM_VerificationToken {
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
budget_fallbacks Json @default("{}")
budget_id String?
organization_id String?
object_permission_id String?
@ -511,6 +513,7 @@ model LiteLLM_DeletedVerificationToken {
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
budget_fallbacks Json @default("{}")
router_settings Json? @default("{}")
budget_id String?
organization_id String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.74"
version = "0.4.75"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.74"
version = "0.4.75"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

11
litellm/__init__.py
View file

@ -263,6 +263,8 @@ azure_key: Optional[str] = None
anthropic_key: Optional[str] = None
replicate_key: Optional[str] = None
bytez_key: Optional[str] = None
gdc_key: Optional[str] = None
gdc_api_base: Optional[str] = None
cohere_key: Optional[str] = None
infinity_key: Optional[str] = None
clarifai_key: Optional[str] = None
@ -586,6 +588,7 @@ gemini_models: Set = set()
xai_models: Set = set()
zai_models: Set = set()
deepseek_models: Set = set()
tencent_models: Set = set()
runwayml_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
@ -799,6 +802,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.add(key)
elif value.get("litellm_provider") == "tencent":
tencent_models.add(key)
elif value.get("litellm_provider") == "runwayml":
runwayml_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
@ -1091,6 +1096,7 @@ models_by_provider: dict = {
"zai": zai_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"tencent": tencent_models,
"runwayml": runwayml_models,
"mistral": mistral_chat_models,
"azure_ai": azure_ai_models,
@ -1787,6 +1793,7 @@ if TYPE_CHECKING:
from .llms.nvidia_nim.embed import (
NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig,
)
from .llms.gdc.chat.transformation import GDCGeminiConfig as GDCGeminiConfig
# Type stubs for lazy-loaded config instances
openaiOSeriesConfig: OpenAIOSeriesConfig
@ -1801,6 +1808,9 @@ if TYPE_CHECKING:
from .llms.deepseek.chat.transformation import (
DeepSeekChatConfig as _DeepSeekChatConfig,
)
from .llms.tencent.chat.transformation import (
TencentChatConfig as _TencentChatConfig,
)
from .llms.sap.chat.transformation import (
GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig,
)
@ -1843,6 +1853,7 @@ if TYPE_CHECKING:
# Type stubs for lazy-loaded config classes (to help mypy understand types)
VLLMConfig: Type[_VLLMConfig]
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
TencentChatConfig: Type[_TencentChatConfig]
GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig]
GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig]
AzureOpenAIO1Config: Type[_AzureOpenAIO1Config]

7
litellm/_lazy_imports_registry.py
View file

@ -284,6 +284,7 @@ LLM_CONFIG_NAMES = (
"LiteLLMProxyChatConfig",
"VLLMConfig",
"DeepSeekChatConfig",
"TencentChatConfig",
"LMStudioChatConfig",
"LmStudioEmbeddingConfig",
"NscaleConfig",
@ -323,6 +324,7 @@ LLM_CONFIG_NAMES = (
"SnowflakeEmbeddingConfig",
"AmazonNovaChatConfig",
"SonioxAudioTranscriptionConfig",
"GDCGeminiConfig",
)
# Types that support lazy loading via _lazy_import_types
@ -1095,6 +1097,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
),
"VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"),
"DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
"TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"),
"LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
"LmStudioEmbeddingConfig": (
".llms.lm_studio.embed.transformation",
@ -1157,6 +1160,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.dashscope.chat.transformation",
"DashScopeChatConfig",
),
"GDCGeminiConfig": (
".llms.gdc.chat.transformation",
"GDCGeminiConfig",
),
"ModelScopeChatConfig": (
".llms.modelscope.chat.transformation",
"ModelScopeChatConfig",

30
litellm/a2a_protocol/main.py
View file

@ -129,6 +129,33 @@ def _set_agent_id_on_logging_obj(
litellm_logging_obj.model_call_details["agent_id"] = agent_id
_A2A_COST_PARAM_KEYS = ("cost_per_query", "input_cost_per_token", "output_cost_per_token")
def _set_litellm_params_on_logging_obj(
kwargs: dict[str, Any],
litellm_params: dict[str, Any],
) -> None:
"""
Merge the agent's pricing params into model_call_details["litellm_params"]
so A2ACostCalculator can read them.
The non-streaming path reuses the proxy-built logging object, whose
litellm_params already carries metadata / proxy_server_request / user-key
context, so merge the pricing keys in rather than replacing the dict.
"""
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
return
cost_params = {key: litellm_params[key] for key in _A2A_COST_PARAM_KEYS if litellm_params.get(key) is not None}
if not cost_params:
return
existing = logging_obj.model_call_details.get("litellm_params") or {}
logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params}
def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
"""
Extract agent info and set model/custom_llm_provider for cost tracking.
@ -477,6 +504,9 @@ async def asend_message(
completion_tokens=completion_tokens,
)
# Merge agent pricing params into the logging obj so cost is calculated
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
# Set agent_id on logging obj for SpendLogs tracking
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)

7
litellm/a2a_protocol/utils.py
View file

@ -121,8 +121,13 @@ class A2ARequestUtils:
Returns:
Tuple of (prompt_tokens, completion_tokens, total_tokens)
"""
# Count input tokens
# Count input tokens. Dump the message to a dict first so extraction hits
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens = A2ARequestUtils.count_tokens(input_text)

10
litellm/caching/disk_cache.py
View file

@ -59,8 +59,9 @@ class DiskCache(BaseCache):
def increment_cache(self, key, value: int, **kwargs) -> int:
# get the value
init_value = self.get_cache(key=key) or 0
value = init_value + value # type: ignore
cached_value = self.get_cache(key=key)
init_value = cached_value if isinstance(cached_value, int) else 0
value = init_value + value
self.set_cache(key, value, **kwargs)
return value
@ -76,8 +77,9 @@ class DiskCache(BaseCache):
async def async_increment(self, key, value: int, **kwargs) -> int:
# get the value
init_value = await self.async_get_cache(key=key) or 0
value = init_value + value # type: ignore
cached_value = await self.async_get_cache(key=key)
init_value = cached_value if isinstance(cached_value, int) else 0
value = init_value + value
await self.async_set_cache(key, value, **kwargs)
return value

3
litellm/constants.py
View file

@ -460,6 +460,7 @@ LITELLM_CHAT_PROVIDERS = [
"openai",
"openai_like",
"bytez",
"gdc",
"xai",
"custom_openai",
"text-completion-openai",
@ -507,6 +508,7 @@ LITELLM_CHAT_PROVIDERS = [
"text-completion-codestral",
"text-completion-inception",
"deepseek",
"tencent",
"sambanova",
"maritalk",
"cloudflare",
@ -728,6 +730,7 @@ openai_compatible_providers: List = [
"volcengine",
"codestral",
"deepseek",
"tencent",
"deepinfra",
"perplexity",
"xinference",

46
litellm/cost_calculator.py
View file

@ -52,6 +52,9 @@ from litellm.llms.databricks.cost_calculator import (
from litellm.llms.deepseek.cost_calculator import (
cost_per_token as deepseek_cost_per_token,
)
from litellm.llms.tencent.cost_calculator import (
cost_per_token as tencent_cost_per_token,
)
from litellm.llms.fireworks_ai.cost_calculator import (
cost_per_token as fireworks_ai_cost_per_token,
)
@ -219,7 +222,7 @@ def _cost_per_token_custom_pricing_helper(
output_cost = completion_tokens * output_cost_per_token
return input_cost, output_cost
elif custom_cost_per_second is not None:
output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore
output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
return 0, output_cost
return None
@ -625,6 +628,8 @@ def cost_per_token(
return gemini_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
elif custom_llm_provider == "deepseek":
return deepseek_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "tencent":
return tencent_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "perplexity":
return perplexity_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "xai":
@ -657,29 +662,27 @@ def cost_per_token(
data_residency=data_residency,
)
if model_info.get("input_cost_per_second", None) is not None and response_time_ms is not None:
input_cost_per_second = model_info.get("input_cost_per_second")
if input_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; response time: %s",
model,
model_info.get("input_cost_per_second", None),
input_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
prompt_tokens_cost_usd_dollar = (
model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
if model_info.get("output_cost_per_second", None) is not None and response_time_ms is not None:
output_cost_per_second = model_info.get("output_cost_per_second")
if output_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - output_cost_per_second: %s; response time: %s",
model,
model_info.get("output_cost_per_second", None),
output_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
completion_tokens_cost_usd_dollar = (
model_info["output_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
verbose_logger.debug(
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
@ -1495,6 +1498,7 @@ def completion_cost(
custom_llm_provider=custom_llm_provider,
litellm_model_name=model,
data_residency=data_residency,
litellm_logging_obj=litellm_logging_obj,
)
elif call_type == _MCP_CALL_TYPE:
from litellm.proxy._experimental.mcp_server.cost_calculator import (
@ -2297,6 +2301,7 @@ def handle_realtime_stream_cost_calculation(
custom_llm_provider: str,
litellm_model_name: str,
data_residency: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
) -> float:
"""
Handles the cost calculation for realtime stream responses.
@ -2332,14 +2337,25 @@ def handle_realtime_stream_cost_calculation(
input_cost_per_token += _input_cost_per_token
output_cost_per_token += _output_cost_per_token
break # exit if we find a valid model
total_cost = input_cost_per_token + output_cost_per_token
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results):
total_cost += handle_realtime_transcription_cost_calculation(
transcription_cost = (
handle_realtime_transcription_cost_calculation(
results=results,
custom_llm_provider=custom_llm_provider,
litellm_model_name=litellm_model_name,
)
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
else 0.0
)
total_cost = input_cost_per_token + output_cost_per_token + transcription_cost
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=input_cost_per_token,
completion_tokens_cost_usd_dollar=output_cost_per_token,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=total_cost,
additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None,
)
return total_cost

6
litellm/exceptions.py
View file

@ -1165,12 +1165,18 @@ class ModifyResponseException(Exception):
request_data: Dict[str, Any],
guardrail_name: Optional[str] = None,
detection_info: Optional[Dict[str, Any]] = None,
original_response: Optional[Any] = None,
):
self.message = message
self.model = model
self.request_data = request_data
self.guardrail_name = guardrail_name
self.detection_info = detection_info or {}
# The LLM response that was blocked (post-call). Carries the real token
# usage the upstream call consumed, so the synthetic block response can
# report it instead of discarding it. None for pre-call blocks (the LLM
# was never invoked).
self.original_response = original_response
super().__init__(message)

22
litellm/experimental_mcp_client/client.py
View file

@ -520,13 +520,28 @@ class MCPClient:
# Return empty list instead of raising to allow graceful degradation
return []
@staticmethod
def error_tool_result(exc: Exception) -> MCPCallToolResult:
"""The error result ``call_tool`` returns when it swallows a failure (no re-execution)."""
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{type(exc).__name__}: {str(exc)}")],
isError=True,
)
async def call_tool(
self,
call_tool_request_params: MCPCallToolRequestParams,
host_progress_callback: Optional[Callable] = None,
raise_on_error: bool = False,
) -> MCPCallToolResult:
"""
Call an MCP Tool.
Args:
raise_on_error: When True, re-raise the underlying exception instead of returning an
``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect
an upstream 401 so it can re-mint the exchanged token and retry once; every other
caller keeps the default and gets graceful ``isError`` degradation.
"""
verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'")
@ -579,11 +594,10 @@ class MCPClient:
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out."
)
if raise_on_error:
raise
# Return a default error result instead of raising
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{error_type}: {str(e)}")], # Empty content for error case
isError=True,
)
return self.error_tool_result(e)
async def list_prompts(self) -> List[Prompt]:
"""List available prompts from the server."""

10
litellm/fine_tuning/main.py
View file

@ -256,8 +256,6 @@ def create_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
# Prepare Azure-specific parameters for extra_body
extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
@ -442,7 +440,7 @@ def cancel_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
@ -457,8 +455,6 @@ def cancel_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job(
api_base=api_base,
@ -616,8 +612,6 @@ def list_fine_tuning_jobs(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs(
api_base=api_base,
@ -759,8 +753,6 @@ def retrieve_fine_tuning_job(
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
response = azure_fine_tuning_apis_instance.retrieve_fine_tuning_job(
api_base=api_base,

6
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -61,13 +61,15 @@ class AzureSentinelLogger(CustomBatchLogger):
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
audit_stream_name (str, optional): Stream name from DCR for audit logs.
If not provided, audit logs use the standard stream name.
If not provided, will use AZURE_SENTINEL_AUDIT_STREAM_NAME env var or the standard stream name.
"""
self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv("AZURE_SENTINEL_DCR_IMMUTABLE_ID")
resolved_stream_name = stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
resolved_audit_stream_name = (
audit_stream_name or os.getenv("AZURE_SENTINEL_AUDIT_STREAM_NAME") or resolved_stream_name
)
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
resolved_tenant_id = tenant_id or os.getenv("AZURE_SENTINEL_TENANT_ID") or os.getenv("AZURE_TENANT_ID")
resolved_client_id = client_id or os.getenv("AZURE_SENTINEL_CLIENT_ID") or os.getenv("AZURE_CLIENT_ID")

14
litellm/integrations/datadog/datadog.py
View file

@ -354,14 +354,14 @@ class DataDogLogger(
Raises:
Raises a NON Blocking verbose_logger.exception if an error occurs
"""
if not self.log_queue:
verbose_logger.exception("Datadog: log_queue does not exist")
return
batch_to_send = self.log_queue[:]
self.log_queue = []
try:
if not self.log_queue:
verbose_logger.exception("Datadog: log_queue does not exist")
return
batch_to_send = self.log_queue[:]
self.log_queue = []
verbose_logger.debug(
"Datadog - about to flush %s events on %s",
len(batch_to_send),

145
litellm/integrations/prometheus.py
View file

@ -4,6 +4,7 @@
from __future__ import annotations
import asyncio
import math
import os
import sys
from datetime import datetime, timedelta
@ -65,6 +66,26 @@ if TYPE_CHECKING:
else:
AsyncIOScheduler = Any
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
def _get_budget_metrics_per_request_timeout() -> float:
raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
if raw is None:
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
try:
parsed = float(raw)
except ValueError:
parsed = None
if parsed is None or not math.isfinite(parsed) or parsed <= 0:
verbose_logger.debug(
"[Non-Blocking] Prometheus: invalid PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT=%r; using default %ss.",
raw,
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT,
)
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
return parsed
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -593,6 +614,21 @@ class PrometheusLogger(CustomLogger):
labelnames=[],
)
########################################
# MCP Tool Call Metrics
########################################
self.litellm_mcp_tool_calls_total = self._counter_factory(
name="litellm_mcp_tool_calls_total",
documentation="Total MCP tool calls, segmented by tool and server name",
labelnames=self.get_labels_for_metric("litellm_mcp_tool_calls_total"),
)
self.litellm_mcp_tool_call_spend_metric = self._counter_factory(
name="litellm_mcp_tool_call_spend_metric",
documentation="Total spend on MCP tool calls, segmented by tool and server name",
labelnames=self.get_labels_for_metric("litellm_mcp_tool_call_spend_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
@ -1300,6 +1336,13 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
# MCP tool call metrics
self._increment_mcp_tool_call_metrics(
standard_logging_payload=standard_logging_payload,
enum_values=enum_values,
response_cost=response_cost,
)
# increment litellm_proxy_total_requests_metric for all successful requests
# (both streaming and non-streaming) in this single location to prevent
# double-counting that occurs when async_post_call_success_hook also increments
@ -1521,6 +1564,49 @@ class PrometheusLogger(CustomLogger):
amount=float(provider_cache_creation_tokens),
)
def _increment_mcp_tool_call_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
response_cost: float,
) -> None:
metadata = standard_logging_payload.get("metadata")
if not isinstance(metadata, dict):
return
mcp_meta = metadata.get("mcp_tool_call_metadata")
if not isinstance(mcp_meta, dict):
return
mcp_enum_values = UserAPIKeyLabelValues(
mcp_tool_name=mcp_meta.get("name"),
mcp_server_name=mcp_meta.get("mcp_server_name"),
hashed_api_key=enum_values.hashed_api_key,
api_key_alias=enum_values.api_key_alias,
team=enum_values.team,
team_alias=enum_values.team_alias,
user=enum_values.user,
end_user=enum_values.end_user,
)
mcp_label_context = PrometheusLabelFactoryContext(mcp_enum_values)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_mcp_tool_calls_total,
"litellm_mcp_tool_calls_total",
mcp_enum_values,
label_context=mcp_label_context,
)
if response_cost > 0:
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_mcp_tool_call_spend_metric,
"litellm_mcp_tool_call_spend_metric",
mcp_enum_values,
label_context=mcp_label_context,
amount=response_cost,
)
async def _increment_remaining_budget_metrics(
self,
user_api_team: Optional[str],
@ -1542,7 +1628,15 @@ class PrometheusLogger(CustomLogger):
_user_spend = _metadata.get("user_api_key_user_spend", None)
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
results = await asyncio.gather(
# Bound the per-request budget-metric emission so that slow Redis/DB
# lookups under load cannot consume the whole LoggingWorker watchdog
# (LOGGING_WORKER_MAX_TIME_PER_COROUTINE, default 20s) and get the entire
# success-logging event cancelled. Budget gauges are also refreshed by the
# periodic cron every PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES,
# so dropping one slow per-request emission only loses sub-cron real-time
# detail, not correctness.
budget_metrics_timeout = _get_budget_metrics_per_request_timeout()
gather_coro = asyncio.gather(
self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
@ -1569,6 +1663,16 @@ class PrometheusLogger(CustomLogger):
),
return_exceptions=True,
)
try:
results = await asyncio.wait_for(gather_coro, timeout=budget_metrics_timeout)
except asyncio.TimeoutError:
verbose_logger.debug(
"[Non-Blocking] Prometheus: per-request budget metric emission "
"exceeded %ss under load; skipping (values are refreshed by the "
"periodic budget-metrics cron job).",
budget_metrics_timeout,
)
return
for i, r in enumerate(results):
if isinstance(r, Exception):
verbose_logger.debug(
@ -1939,6 +2043,43 @@ class PrometheusLogger(CustomLogger):
return False
@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> Optional[str]:
"""
Best-effort provider for the client-side failure path.
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
"""
litellm_params = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
model = litellm_params.get("model") or request_data.get("model")
if not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
except litellm.exceptions.BadRequestError:
return None
except Exception as e: # noqa: BLE001 - metrics labeling must never break request/failure handling
verbose_logger.debug(
"prometheus: unexpected error inferring api_provider from model=%s: %s",
model,
e,
)
return None
async def async_post_call_failure_hook(
self,
request_data: dict,
@ -1974,6 +2115,7 @@ class PrometheusLogger(CustomLogger):
_metadata = request_data.get("metadata", {}) or {}
model_id = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider = self._extract_api_provider_from_request_data(request_data)
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
@ -1995,6 +2137,7 @@ class PrometheusLogger(CustomLogger):
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
api_provider=api_provider,
stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None),
)
_label_ctx = PrometheusLabelFactoryContext(enum_values)

25
litellm/integrations/s3_v2.py
View file

@ -54,6 +54,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: Optional[str] = None,
s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
@ -92,6 +93,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files=s3_strip_base64_files,
s3_use_key_prefix=s3_use_key_prefix,
s3_use_virtual_hosted_style=s3_use_virtual_hosted_style,
s3_server_side_encryption=s3_server_side_encryption,
)
verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}")
@ -145,6 +147,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: Optional[str] = None,
params_source: Optional[dict] = None,
):
"""
@ -194,6 +197,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style
)
self.s3_server_side_encryption = params.get("s3_server_side_encryption") or s3_server_side_encryption
return
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
@ -273,6 +278,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
try:
import base64
import hashlib
import requests
@ -317,14 +323,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# Calculate SHA256 hash of the content
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
content_md5 = base64.b64encode(
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
).decode()
# Prepare the request
headers = {
"Content-Type": "application/json",
"Content-MD5": content_md5,
"x-amz-content-sha256": content_hash,
"Content-Language": "en",
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**(
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
if self.s3_server_side_encryption
else {}
),
}
req = requests.Request("PUT", url, data=json_string, headers=headers)
prepped = req.prepare()
@ -447,6 +462,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
try:
import base64
import hashlib
import requests
@ -482,14 +498,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# Calculate SHA256 hash of the content
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
content_md5 = base64.b64encode(
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
).decode()
# Prepare the request
headers = {
"Content-Type": "application/json",
"Content-MD5": content_md5,
"x-amz-content-sha256": content_hash,
"Content-Language": "en",
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**(
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
if self.s3_server_side_encryption
else {}
),
}
req = requests.Request("PUT", url, data=json_string, headers=headers)
prepped = req.prepare()

178
litellm/integrations/websearch_interception/handler.py
View file

@ -91,6 +91,7 @@ class WebSearchInterceptionLogger(CustomLogger):
messages: List[Dict],
tools: Optional[List[Dict]],
custom_llm_provider: Optional[str],
kwargs: Optional[dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""
Short-circuit web-search-only requests by executing the search directly.
@ -120,21 +121,26 @@ class WebSearchInterceptionLogger(CustomLogger):
if self.enabled_providers is not None and provider_str not in self.enabled_providers:
return None
# Only short-circuit for providers without native Anthropic Messages
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
# includes a follow-up LLM call to synthesize the answer from search
# results. Short-circuiting those would skip that synthesis step and
# return raw search text — a regression for existing users.
# Only short-circuit for providers whose Anthropic Messages agentic loop
# does not run web_search itself. Providers that have a
# BaseAnthropicMessagesConfig which handles web search natively (bedrock,
# vertex_ai, azure_ai, anthropic) already perform the search plus a
# follow-up LLM synthesis step; short-circuiting those would skip that
# synthesis and return raw search text — a regression for existing users.
#
# github_copilot has a BaseAnthropicMessagesConfig (added for thinking
# passthrough) but does not handle web_search natively, so its config
# returns handles_web_search_natively() == False and we still short-circuit
# web-search-only requests against it.
try:
provider_enum = LlmProviders(provider_str)
anthropic_config = ProviderConfigManager.get_provider_anthropic_messages_config(
model=model, provider=provider_enum
)
if anthropic_config is not None:
if anthropic_config is not None and anthropic_config.handles_web_search_natively():
verbose_logger.debug(
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
"(provider has native Anthropic Messages support, using agentic loop)"
"(provider handles web search natively via the agentic loop)"
)
return None
except (ValueError, Exception):
@ -171,7 +177,10 @@ class WebSearchInterceptionLogger(CustomLogger):
# Execute search — keep the structured SearchResponse so the native
# block can carry per-result url/title/page_age.
try:
search_result_text, structured = await self._execute_search(query)
if kwargs is None:
search_result_text, structured = await self._execute_search(query)
else:
search_result_text, structured = await self._execute_search(query, kwargs=kwargs)
except Exception as e:
verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}")
search_result_text, structured = f"Search failed: {e}", None
@ -931,7 +940,7 @@ class WebSearchInterceptionLogger(CustomLogger):
query = tool_call["input"].get("query")
if query:
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
search_tasks.append(self._execute_search(query))
search_tasks.append(self._execute_search(query, kwargs=kwargs))
else:
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query")
# Add empty result for tools without query
@ -1004,7 +1013,9 @@ class WebSearchInterceptionLogger(CustomLogger):
)
return patch, structured_results
async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
async def _execute_search(
self, query: str, kwargs: Optional[dict[str, Any]] = None
) -> Tuple[str, Optional[SearchResponse]]:
"""
Execute a single web search using router's search tools.
@ -1026,36 +1037,13 @@ class WebSearchInterceptionLogger(CustomLogger):
)
llm_router = None
# Determine search provider from router's search_tools
search_tool = self._select_search_tool_from_router(llm_router=llm_router)
search_provider: Optional[str] = None
if llm_router is not None and hasattr(llm_router, "search_tools"):
if self.search_tool_name:
# Find specific search tool by name
matching_tools = [
tool
for tool in llm_router.search_tools
if tool.get("search_tool_name") == self.search_tool_name
]
if matching_tools:
search_tool = matching_tools[0]
search_provider = search_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"with provider '{search_provider}'"
)
else:
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
"falling back to first available or perplexity"
)
# If no specific tool or not found, use first available
if not search_provider and llm_router.search_tools:
first_tool = llm_router.search_tools[0]
search_provider = first_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
)
search_litellm_params: dict[str, Any] = {}
if search_tool is not None:
await self._authorize_search_tool(search_tool=search_tool, kwargs=kwargs)
search_litellm_params = dict(search_tool.get("litellm_params", {}) or {})
search_provider = search_litellm_params.get("search_provider")
# Fallback to perplexity if no router or no search tools configured
if not search_provider:
@ -1068,7 +1056,12 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
)
result = await litellm.asearch(query=query, search_provider=search_provider)
search_kwargs = {
key: value
for key, value in search_litellm_params.items()
if key != "search_provider" and value is not None
}
result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs)
# Format using transformation function
search_result_text = WebSearchTransformation.format_search_response(result)
@ -1081,6 +1074,107 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}")
raise
async def _authorize_search_tool(
self,
search_tool: dict[str, Any],
kwargs: Optional[dict[str, Any]],
) -> None:
search_tool_name = search_tool.get("search_tool_name")
if not isinstance(search_tool_name, str) or not search_tool_name:
return
user_api_key_auth = self._get_user_api_key_auth_from_kwargs(kwargs)
if user_api_key_auth is None:
return
from litellm.proxy.auth.auth_checks import (
can_key_call_search_tool,
can_team_call_search_tool,
get_team_object,
)
await can_key_call_search_tool(
search_tool_name=search_tool_name,
valid_token=user_api_key_auth,
)
team_id = getattr(user_api_key_auth, "team_id", None)
if team_id:
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
team_object = await get_team_object(
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None),
proxy_logging_obj=proxy_logging_obj,
)
await can_team_call_search_tool(
search_tool_name=search_tool_name,
team_object=team_object,
)
@staticmethod
def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any:
if not kwargs:
return None
for metadata_key in ("metadata", "litellm_metadata"):
metadata = kwargs.get(metadata_key)
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
return metadata["user_api_key_auth"]
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
return None
for metadata_key in ("metadata", "litellm_metadata"):
metadata = litellm_params.get(metadata_key)
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
return metadata["user_api_key_auth"]
return None
def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]:
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools = list(getattr(llm_router, "search_tools") or [])
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
def _select_search_tool_from_list(
self,
search_tools: list[dict[str, Any]],
source: str,
) -> Optional[dict[str, Any]]:
if self.search_tool_name:
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
if matching_tools:
search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"from {source} with provider '{search_provider}'"
)
return matching_tools[0]
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, "
"falling back to first available or perplexity"
)
if search_tools:
first_tool = search_tools[0]
search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool from {source} "
f"with provider '{search_provider}'"
)
return first_tool
return None
async def _execute_chat_completion_agentic_loop(
self,
model: str,
@ -1140,7 +1234,7 @@ class WebSearchInterceptionLogger(CustomLogger):
if query:
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
search_tasks.append(self._execute_search(query))
search_tasks.append(self._execute_search(query, kwargs=kwargs))
else:
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query")
# Add empty result for tools without query

2
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2173,7 +2173,7 @@ def exception_type( # type: ignore
litellm_response_headers = _get_response_headers(original_exception=original_exception)
try:
error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
if model:
if model or custom_llm_provider:
if hasattr(original_exception, "message"):
error_str = (
redact_string(str(original_exception.message))

4
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -36,6 +36,8 @@ OPTIONAL_KWARGS_KEYS = frozenset(
"aws_bedrock_project_id",
"tpm",
"rpm",
"itpm",
"otpm",
"use_xai_oauth",
}
)
@ -74,6 +76,7 @@ def get_litellm_params(
proxy_server_request=None,
acompletion=None,
aembedding=None,
allm_passthrough_route=None,
preset_cache_key=None,
no_log=None,
input_cost_per_second=None,
@ -116,6 +119,7 @@ def get_litellm_params(
# Build base dict with explicit parameters (always included)
litellm_params = {
"acompletion": acompletion,
"allm_passthrough_route": allm_passthrough_route,
"api_key": api_key,
"force_timeout": force_timeout,
"logger_fn": logger_fn,

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -446,6 +446,8 @@ def get_llm_provider(
# bytez models
elif model.startswith("bytez/"):
custom_llm_provider = "bytez"
elif model.startswith("gdc/"):
custom_llm_provider = "gdc"
elif model.startswith("lemonade/"):
custom_llm_provider = "lemonade"
elif model.startswith("heroku/"):
@ -650,6 +652,10 @@ def _get_openai_compatible_provider_info(
api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" # type: ignore
dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY")
elif custom_llm_provider == "tencent":
api_base = api_base or get_secret("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1"
dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY")
elif custom_llm_provider == "fireworks_ai":
# fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
(

2
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -106,6 +106,8 @@ def get_supported_openai_params(
return litellm.VLLMConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "deepseek":
return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "tencent":
return litellm.TencentChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
return litellm.CohereChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "maritalk":

29
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,7 +1,23 @@
from typing import Dict, Optional
from typing import Any, Dict, Iterator, Optional
from litellm.types.utils import StandardCallbackDynamicParams
_CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata")
def iter_client_callback_metadata_dicts(
kwargs: dict[str, Any],
) -> Iterator[tuple[str, dict[str, Any]]]:
litellm_params = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):
nested = litellm_params.get("metadata")
if isinstance(nested, dict):
yield "litellm_params.metadata", nested
for key in _CLIENT_CALLBACK_METADATA_SLOTS:
candidate = kwargs.get(key)
if isinstance(candidate, dict):
yield key, candidate
def _is_env_reference(value: object) -> bool:
return isinstance(value, str) and "os.environ/" in value
@ -55,6 +71,7 @@ _supported_callback_params = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"turn_off_message_logging",
]
_request_blocked_callback_params = {
@ -87,19 +104,13 @@ def initialize_standard_callback_dynamic_params(
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
metadata = (kwargs.get("metadata") or {}).copy()
litellm_params = kwargs.get("litellm_params") or {}
if isinstance(litellm_params, dict):
metadata.update(litellm_params.get("metadata") or {})
if isinstance(metadata, dict):
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
if param in _request_blocked_callback_params:
continue
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source="metadata")
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

10
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1530,6 +1530,7 @@ class Logging(LiteLLMLoggingBaseClass):
and litellm_params.get(CallTypes.aembedding.value, False) is not True
and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
and litellm_params.get(CallTypes.atranscription.value, False) is not True
and litellm_params.get(CallTypes.allm_passthrough_route.value, False) is not True
)
def _is_assembled_stream_success(self, result=None) -> bool:
@ -4722,7 +4723,7 @@ class StandardLoggingPayloadSetup:
api_base: Optional[str] = None,
) -> StandardLoggingModelInformation:
model_cost_name = _select_model_name_for_cost_calc(
model=None,
model=base_model if custom_pricing else None,
completion_response=init_response_obj, # type: ignore
base_model=base_model,
custom_pricing=custom_pricing,
@ -5268,6 +5269,11 @@ def get_standard_logging_object_payload(
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
raw_response_cost = kwargs.get("response_cost")
response_cost: float = raw_response_cost or 0.0
@ -5389,7 +5395,7 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
print(json.dumps(payload, indent=4)) # noqa: T201
print(json.dumps(payload, indent=4), flush=True) # noqa: T201
def get_standard_logging_metadata(

2
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -656,7 +656,7 @@ def convert_to_model_response_object(
message: Optional[Message] = None
finish_reason: Optional[str] = None
if _should_convert_tool_call_to_json_mode(
if tool_calls is not None and _should_convert_tool_call_to_json_mode(
tool_calls=tool_calls,
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
):

9
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -5035,15 +5035,18 @@ def _bedrock_tools_pt(tools: List, model: Optional[str] = None) -> List[BedrockT
]
"""
from litellm.llms.bedrock.common_utils import (
get_bedrock_base_model,
bedrock_converse_supports_strict_tools,
normalize_json_schema_custom_types_to_object,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
_valid_json_schema_root_types = frozenset(("array", "boolean", "integer", "null", "number", "object", "string"))
# Only Claude on Bedrock honours strict tool schemas; other families
# (Nova, Llama, GPT-OSS) reject the strict field outright.
supports_strict_tools = bool(model and get_bedrock_base_model(model).startswith("anthropic"))
# (Nova, Llama, GPT-OSS) reject the strict field outright. Opus 4.7/4.8
# also reject `strict` on Bedrock Converse (see #31582) — their validator
# maps toolSpec to the native Anthropic tool shape, which has no strict
# field, even though Anthropic's native API accepts it as a top-level key.
supports_strict_tools = bool(model and bedrock_converse_supports_strict_tools(model))
tool_block_list: List[BedrockToolBlock] = []
for tool_idx, tool in enumerate(tools):
# Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding)

6
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -72,6 +72,9 @@ def _process_image_response(response: Response, url: str) -> str:
async def async_convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
@ -95,6 +98,9 @@ async def async_convert_url_to_base64(url: str) -> str:
def convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(

18
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -131,8 +131,26 @@ class SensitiveDataMasker:
return masked_data
def mask(self, data: object) -> object:
if isinstance(data, Mapping):
return self.mask_dict(dict(data))
if isinstance(data, list):
return self._mask_sequence(
data,
0,
DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
None,
False,
)
return data
_default_masker = SensitiveDataMasker()
_error_masker = SensitiveDataMasker(visible_prefix=4, visible_suffix=0)
def mask_sensitive_structure(data: object) -> object:
return _error_masker.mask(data)
def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]:

48
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -7,6 +7,7 @@ from litellm.types.llms.openai import (
ChatCompletionAudioDelta,
)
from litellm.types.utils import (
CacheCreationTokenDetails,
ChatCompletionAudioResponse,
ChatCompletionMessageToolCall,
Choices,
@ -541,6 +542,12 @@ class ChunkProcessor:
web_search_requests: Optional[int] = None
completion_tokens_details: Optional[CompletionTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
# Anthropic emits the cache-creation TTL breakdown (5m/1h split) only on
# the `message_start` event; the later `message_delta` carries the flat
# cache-creation count but drops the nested breakdown. prompt_tokens_details
# is last-wins, so without preserving this separately the 1h breakdown is
# lost and 1h cache writes get billed at the 5m rate.
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
for chunk in chunks:
usage_chunk: Optional[Usage] = None
if "usage" in chunk:
@ -594,7 +601,18 @@ class ChunkProcessor:
"web_search_requests",
)
prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"]
prompt_tokens_details = cast(
Optional[PromptTokensDetailsWrapper],
usage_chunk_dict["prompt_tokens_details"],
)
cache_creation_token_details = self._capture_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
prompt_tokens_details = self._attach_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
completion_tokens = self._reset_anthropic_cursor_completion_tokens(
chunks=chunks,
@ -613,6 +631,34 @@ class ChunkProcessor:
prompt_tokens_details=prompt_tokens_details,
)
@staticmethod
def _capture_cache_creation_token_details(
prompt_tokens_details: Optional[PromptTokensDetailsWrapper],
current: Optional[CacheCreationTokenDetails],
) -> Optional[CacheCreationTokenDetails]:
incoming = cast(
Optional[CacheCreationTokenDetails],
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if incoming is not None:
return incoming
return current
@staticmethod
def _attach_cache_creation_token_details(
prompt_tokens_details: Optional[PromptTokensDetailsWrapper],
cache_creation_token_details: Optional[CacheCreationTokenDetails],
) -> Optional[PromptTokensDetailsWrapper]:
if prompt_tokens_details is None or cache_creation_token_details is None:
return prompt_tokens_details
existing = cast(
Optional[CacheCreationTokenDetails],
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if existing is not None:
return prompt_tokens_details
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
@staticmethod
def _reset_anthropic_cursor_completion_tokens(
chunks: list[dict[str, Any] | ModelResponse],

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1884,7 +1884,7 @@ class CustomStreamWrapper:
await self.fetch_stream()
if is_async_iterable(self.completion_stream):
async for chunk in self.completion_stream: # type: ignore[union-attr]
async for chunk in self.completion_stream: # pyright: ignore[reportOptionalIterable] # is_async_iterable guard proves __aiter__
if chunk == "None" or chunk is None:
continue # skip None chunks

21
litellm/llms/a2a/chat/transformation.py
View file

@ -10,7 +10,7 @@ import httpx
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from ..common_utils import (
A2AError,
@ -312,6 +312,25 @@ class A2AConfig(BaseConfig):
# Set ID from response
model_response.id = response_json.get("id", str(uuid.uuid4()))
# A2A agents don't return token usage; estimate it so per-token pricing
# produces real cost and callers don't receive usage of 0/0/0.
try:
from litellm.utils import token_counter
prompt_tokens = token_counter(model="gpt-3.5-turbo", messages=messages)
completion_tokens = token_counter(model="gpt-3.5-turbo", text=text, count_response_tokens=True)
setattr(
model_response,
"usage",
Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
),
)
except Exception: # noqa: BLE001 - best-effort estimate; a tokenizer hiccup must not break the response
pass
return model_response
def get_model_response_iterator(

199
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -48,7 +48,10 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -70,6 +73,170 @@ class AnthropicMessagesHandler(BaseTranslation):
super().__init__()
self.adapter = LiteLLMAnthropicMessagesAdapter()
@staticmethod
def _build_streaming_usage_response(
responses_so_far: list[Any],
request_data: Optional[dict],
) -> Optional[ModelResponse]:
chunks = tuple(response for response in responses_so_far if isinstance(response, (str, bytes)))
if not chunks:
return None
try:
return AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks(
all_chunks=chunks,
model=str((request_data or {}).get("model") or ""),
)
except (AttributeError, TypeError, ValueError):
return None
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Optional[list[Any]] = None,
) -> list[bytes]:
"""
Build an Anthropic SSE sequence delivering the guardrail block message
and terminating the stream cleanly.
- ``stream_started`` False (buffered / pre-stream): nothing has been
sent, so emit a complete standalone message (message_start ->
content_block_* -> message_delta -> message_stop) via
FakeAnthropicMessagesStreamIterator, the same converter the
/v1/messages pre-stream block handler uses.
- ``stream_started`` True (sampling / detect-only end-of-stream): real
chunks were already sent, so *continue* the in-progress message --
close the open content block, append the block message as a new text
block, then end the message. Emitting a second ``message_start`` here
would make Anthropic clients reject the stream.
"""
if stream_started:
return self._block_continuation_chunks(exc, responses_so_far or [])
return self._standalone_block_chunks(exc)
def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
import uuid
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage,
)
from litellm.types.utils import AnthropicMessagesResponse
block_response = AnthropicMessagesResponse(
id=f"msg_{uuid.uuid4()}",
type="message",
role="assistant",
content=[{"type": "text", "text": exc.message}],
model=exc.model,
stop_reason="end_turn",
usage=blocked_response_usage(getattr(exc, "original_response", None)),
)
return list(FakeAnthropicMessagesStreamIterator(response=block_response))
def _block_continuation_chunks(self, exc: "ModifyResponseException", responses_so_far: list[Any]) -> list[bytes]:
"""Continue an already-started message: close the open content block,
append the block message as a new text block, then end the message --
without a second message_start."""
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage,
)
def _sse(event_type: str, payload: dict) -> bytes:
return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
output_tokens = blocked_response_usage(getattr(exc, "original_response", None))["output_tokens"]
open_index, max_index = self._content_block_state(responses_so_far)
new_index = (max_index + 1) if max_index is not None else 0
chunks: list[bytes] = []
if open_index is not None:
chunks.append(_sse("content_block_stop", {"type": "content_block_stop", "index": open_index}))
chunks += [
_sse(
"content_block_start",
{
"type": "content_block_start",
"index": new_index,
"content_block": {"type": "text", "text": ""},
},
),
_sse(
"content_block_delta",
{
"type": "content_block_delta",
"index": new_index,
"delta": {"type": "text_delta", "text": exc.message},
},
),
_sse("content_block_stop", {"type": "content_block_stop", "index": new_index}),
_sse(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": output_tokens},
},
),
_sse("message_stop", {"type": "message_stop"}),
]
return chunks
@staticmethod
def _content_block_state(
responses_so_far: list[Any],
) -> tuple[Optional[int], Optional[int]]:
"""From the SSE chunks already sent to the client, return (open
content-block index or None, highest content-block index seen or None).
A single streamed item may bundle multiple SSE events (raw bytes) or be
an already-parsed event dict, so every event across every item is
considered -- matching how ``get_streaming_string_so_far`` reads the
same stream."""
open_indices: set[int] = set()
max_index: Optional[int] = None
for item in responses_so_far:
for data in AnthropicMessagesHandler._iter_sse_events(item):
event_type = data.get("type")
index = data.get("index")
if not isinstance(index, int):
continue
if event_type == "content_block_start":
open_indices.add(index)
max_index = index if max_index is None else max(max_index, index)
elif event_type == "content_block_stop":
open_indices.discard(index)
open_index = max(open_indices) if open_indices else None
return open_index, max_index
@staticmethod
def _iter_sse_events(item: Any) -> list[dict]:
"""Yield the event-data dicts in one stream chunk.
Handles both formats this stream can carry (see
``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle
several events separated by a blank line -- and an already-parsed event
``dict``."""
if isinstance(item, dict):
return [item]
if not isinstance(item, (bytes, bytearray)):
return []
events: list[dict] = []
for block in item.decode("utf-8", errors="replace").split("\n\n"):
for line in block.split("\n"):
line = line.strip()
if not line.startswith("data:"):
continue
try:
parsed = json.loads(line[len("data:") :].strip())
except json.JSONDecodeError:
continue
if isinstance(parsed, dict):
events.append(parsed)
return events
def _translate_to_openai(self, data: dict) -> ChatCompletionRequest:
"""Translate Anthropic request to OpenAI chat completion format."""
(
@ -406,6 +573,8 @@ class AnthropicMessagesHandler(BaseTranslation):
Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
"""
from litellm.integrations.custom_guardrail import ModifyResponseException
has_ended = self._check_streaming_has_ended(responses_so_far)
if has_ended:
# build the model response from the responses_so_far
@ -430,25 +599,35 @@ class AnthropicMessagesHandler(BaseTranslation):
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
_guardrailed_inputs = (
await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
try:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=guardrail_inputs,
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
)
except ModifyResponseException as e:
if e.original_response is None:
e.original_response = built_response or self._build_streaming_usage_response(
responses_so_far, request_data
)
raise
else:
verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices")
return responses_so_far
string_so_far = self.get_streaming_string_so_far(responses_so_far)
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
inputs={"texts": [string_so_far]},
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
try:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs={"texts": [string_so_far]},
request_data=request_data if request_data is not None else {},
input_type="response",
logging_obj=litellm_logging_obj,
)
except ModifyResponseException as e:
if e.original_response is None:
e.original_response = self._build_streaming_usage_response(responses_so_far, request_data)
raise
return responses_so_far
def _prepare_request_data(

65
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -78,7 +78,7 @@ async def _prepare_context_managed_request(
system: Optional[Any],
context_management_spec: Any,
litellm_metadata: Optional[Dict],
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
llm_router: Any,
user_api_key_auth: Any = None,
) -> Optional[PolyfillResult]:
@ -95,7 +95,7 @@ async def _prepare_context_managed_request(
# silently drop intermediate turns.
polyfill_will_run = _polyfill_will_run(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if polyfill_will_run:
@ -117,7 +117,7 @@ async def _prepare_context_managed_request(
system=working_system,
context_management_spec=context_management_spec,
litellm_metadata=litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=llm_router,
user_api_key_auth=user_api_key_auth,
)
@ -143,18 +143,19 @@ async def _prepare_context_managed_request(
def _polyfill_will_run(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> bool:
"""Return True when ``compact_20260112`` will run via the polyfill dispatcher.
Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or
effective ``drop_params`` short-circuits the polyfill. The pre-processing
skip only applies when the dispatcher will actually invoke
``apply_compact_20260112`` (which has its own compaction-block slicing).
Mirrors the gating in ``_run_polyfill_if_enabled``: an empty spec or an
explicit ``context_management`` entry in ``additional_drop_params``
short-circuits the polyfill. The pre-processing skip only applies when the
dispatcher will actually invoke ``apply_compact_20260112`` (which has its
own compaction-block slicing).
"""
edits = _normalize_spec_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if edits is None:
return False
@ -169,7 +170,7 @@ def _polyfill_will_run(
def _spec_has_non_compact_edits(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> bool:
"""Return True when the spec includes edits other than ``compact_20260112``.
@ -180,7 +181,7 @@ def _spec_has_non_compact_edits(
"""
edits = _normalize_spec_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if edits is None:
return False
@ -195,10 +196,22 @@ def _spec_has_non_compact_edits(
)
def _context_management_explicitly_dropped(additional_drop_params: Optional[list[str]]) -> bool:
"""True when the caller opted out of context_management via ``additional_drop_params``.
``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
``drop_params`` only exists to drop genuinely unsupported params.
"""
if not isinstance(additional_drop_params, list):
return False
return "context_management" in additional_drop_params
def _normalize_spec_edits(
*,
context_management_spec: Any,
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
) -> Optional[List[Dict[str, Any]]]:
"""Return the normalized ``edits`` list, or ``None`` if the polyfill won't run.
@ -208,8 +221,7 @@ def _normalize_spec_edits(
if not context_management_spec:
return None
effective_drop_params = drop_params if drop_params is not None else litellm.drop_params
if effective_drop_params:
if _context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.experimental_pass_through.context_management.dispatcher import (
@ -230,22 +242,23 @@ async def _run_polyfill_if_enabled(
system: Optional[Any],
context_management_spec: Any,
litellm_metadata: Optional[Dict],
drop_params: Optional[bool],
additional_drop_params: Optional[list[str]],
llm_router: Any,
user_api_key_auth: Any = None,
) -> Optional[PolyfillResult]:
"""Run the async context_management polyfill if a spec is present.
Returns ``None`` when the spec is empty or drop_params is on. Raises
``AnthropicContextManagementError`` so the /v1/messages endpoint can
emit an Anthropic-format 400. All other exceptions are best-effort
swallowed (matches v0 behavior).
Returns ``None`` when the spec is empty or ``context_management`` is
listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
does not disable the polyfill because context_management is a supported
param). Raises ``AnthropicContextManagementError`` so the /v1/messages
endpoint can emit an Anthropic-format 400. All other exceptions are
best-effort swallowed (matches v0 behavior).
"""
if not context_management_spec:
return None
effective_drop_params = drop_params if drop_params is not None else litellm.drop_params
if effective_drop_params:
if _context_management_explicitly_dropped(additional_drop_params):
return None
try:
@ -274,7 +287,7 @@ async def _run_polyfill_if_enabled(
# emits an Anthropic-format error.
if _spec_has_non_compact_edits(
context_management_spec=context_management_spec,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
):
raise AnthropicContextManagementError(
status_code=500,
@ -533,7 +546,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
) -> Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]:
"""Handle non-Anthropic models asynchronously using the adapter"""
context_management = kwargs.pop("context_management", None)
drop_params: Optional[bool] = kwargs.get("drop_params", None)
additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None)
litellm_router = kwargs.pop("litellm_router", None)
if litellm_router is None:
try:
@ -555,7 +568,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
system=system,
context_management_spec=context_management,
litellm_metadata=proxy_litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=litellm_router,
user_api_key_auth=user_api_key_auth,
)
@ -661,7 +674,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# ``compact_20260112`` editor can ``await`` the summarization model);
# bridge to it via ``run_async_function``.
context_management = kwargs.pop("context_management", None)
drop_params: Optional[bool] = kwargs.get("drop_params", None)
additional_drop_params: Optional[list[str]] = kwargs.get("additional_drop_params", None)
# Deliberately do NOT auto-attach the proxy ``llm_router`` here:
# ``run_async_function`` spawns a new event loop in a worker thread
# to bridge to the async dispatcher, but the proxy router's httpx
@ -696,7 +709,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
system=system,
context_management_spec=context_management,
litellm_metadata=proxy_litellm_metadata,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
llm_router=litellm_router,
user_api_key_auth=user_api_key_auth,
)

8
litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
View file

@ -193,6 +193,14 @@ class AgenticAnthropicStreamingIterator:
raise StopAsyncIteration
async def aclose(self) -> None:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
aclose_if_supported,
)
await aclose_if_supported(self._inner)
await aclose_if_supported(self._follow_up_iterator)
async def _process_agentic_hooks(self) -> None:
"""Rebuild the Anthropic response from collected SSE bytes and call hooks."""
if self._hook_processing_done:

3
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -148,6 +148,7 @@ async def _try_websearch_short_circuit(
tools: Optional[List[Dict]],
custom_llm_provider: Optional[str],
stream: Optional[bool],
kwargs: Optional[dict] = None,
) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
"""
Attempt to short-circuit a web-search-only request.
@ -177,6 +178,7 @@ async def _try_websearch_short_circuit(
messages=messages,
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
if response is not None:
anthropic_response = cast(AnthropicMessagesResponse, response)
@ -292,6 +294,7 @@ async def anthropic_messages(
tools=tools,
custom_llm_provider=custom_llm_provider,
stream=original_stream,
kwargs={**kwargs, "metadata": metadata},
)
if short_circuit_response is not None:
return short_circuit_response

56
litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
View file

@ -17,7 +17,9 @@ How it works:
import uuid
from typing import Any, AsyncIterator, Dict, List, Optional, Union
import litellm
import litellm.constants as _c
from litellm.litellm_core_utils.url_utils import validate_url
from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -76,16 +78,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
raise ValueError("advisor tool definition must include a 'model' field specifying the advisor model")
_raw_max_uses = advisor_tool.get("max_uses")
max_uses: int = ADVISOR_MAX_USES if _raw_max_uses is None else int(_raw_max_uses)
# Optional routing overrides for the advisor sub-call (e.g. proxy routing).
# If not set in the tool definition, litellm resolves from env vars.
# The advisor tool is caller-controlled; only honor a client-supplied
# api_base/api_key when the proxy has enabled clientside credentials,
# otherwise let litellm resolve from server config.
advisor_api_key: Optional[str] = None
advisor_api_base: Optional[str] = None
if _allow_client_side_advisor_credentials():
advisor_api_key = advisor_tool.get("api_key")
advisor_api_base = advisor_tool.get("api_base")
advisor_api_key, advisor_api_base = _resolve_advisor_credentials(advisor_tool)
# Build the synthetic tool definition the provider will receive.
synthetic_advisor_tool = _make_synthetic_advisor_tool()
@ -186,6 +179,49 @@ def _allow_client_side_advisor_credentials() -> bool:
return general_settings.get("allow_client_side_credentials") is True
def _resolve_advisor_credentials(advisor_tool: dict) -> tuple[Optional[str], Optional[str]]:
"""Resolve the (api_key, api_base) override for the advisor sub-call.
A caller-supplied ``api_base`` is only honored alongside a caller-supplied
``api_key``: without one, ``AnthropicModelInfo.get_auth_header()`` falls
back to the proxy's own Anthropic credentials, which would then be sent to
the caller-chosen ``api_base``. A caller-supplied ``api_base`` is also
required to be https with TLS verification on, and SSRF-validated so it
can't target a private/internal/cloud-metadata address, mirroring
``proxy.auth.auth_utils.check_complete_credentials``. https with TLS
verification is required because ``validate_url`` only rewrites the
connection to a DNS-pinned IP for http, or for https with
``litellm.ssl_verify`` disabled; otherwise it returns the URL unchanged
and relies on certificate validation to block DNS rebinding, so this
closes the same gap without threading the pinned URL through the whole
``anthropic_messages()`` call chain.
"""
if not _allow_client_side_advisor_credentials():
return None, None
api_key: Optional[str] = advisor_tool.get("api_key")
api_base: Optional[str] = advisor_tool.get("api_base")
if api_base is None:
return api_key, None
if not api_key:
raise ValueError(
"advisor tool definition sets 'api_base' without 'api_key'. A "
"caller-supplied api_base is only honored alongside a "
"caller-supplied api_key, so the proxy's own credentials are "
"never sent to a caller-chosen destination."
)
if not api_base.startswith("https://"):
raise ValueError(f"advisor tool definition sets 'api_base'={api_base!r}, which must use the https scheme.")
if getattr(litellm, "ssl_verify", True) is False:
raise ValueError(
"advisor tool definition sets 'api_base' but the proxy has TLS verification "
"disabled (litellm.ssl_verify=False), so a caller-supplied api_base can't be "
"safely validated against DNS rebinding."
)
if getattr(litellm, "user_url_validation", True):
validate_url(api_base)
return api_key, api_base
def _make_synthetic_advisor_tool() -> Dict:
"""Build a regular tool definition the executor provider can understand."""
return {

99
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -1,8 +1,13 @@
import asyncio
import json
from datetime import datetime
from typing import Any, AsyncIterator, List, Union
from typing import Any, AsyncIterator, List, Protocol, Union, runtime_checkable
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
@ -12,6 +17,93 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ = PassThroughEndpointLogging()
INCOMPLETE_STREAM_ERROR_MESSAGE = (
"Provider stream ended before emitting a message_stop event; "
"the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."
)
def _is_message_stop_chunk(chunk: object) -> bool:
if isinstance(chunk, dict):
return chunk.get("type") == "message_stop"
if isinstance(chunk, (bytes, bytearray)):
return any(line == b"event: message_stop" for line in chunk.splitlines())
return False
def _is_provider_error_chunk(chunk: object) -> bool:
if isinstance(chunk, dict):
return chunk.get("type") == "error"
if isinstance(chunk, (bytes, bytearray)):
return any(line == b"event: error" for line in chunk.splitlines())
return False
def _is_terminal_stream_chunk(chunk: object) -> bool:
return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
def _incomplete_stream_error_sse_event() -> bytes:
payload = json.dumps(
{
"type": "error",
"error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
}
)
return f"event: error\ndata: {payload}\n\n".encode()
class AnthropicMessagesStreamHiddenParams(TypedDict):
additional_headers: dict[str, str]
@runtime_checkable
class SupportsAclose(Protocol):
async def aclose(self) -> None: ...
async def aclose_if_supported(stream: object) -> None:
if isinstance(stream, SupportsAclose):
await stream.aclose()
_RESPONSE_HEADERS_ADAPTER: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
def anthropic_messages_stream_hidden_params(
response_headers: httpx.Headers,
) -> AnthropicMessagesStreamHiddenParams:
return AnthropicMessagesStreamHiddenParams(
additional_headers=_RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers))
)
class AnthropicMessagesStreamingResponse:
"""
Wraps the /v1/messages SSE byte stream so upstream provider response
headers (e.g. Bedrock's x-amzn-requestid / x-amzn-trace-id) survive as
``_hidden_params["additional_headers"]``, which the proxy forwards to
clients as ``llm_provider-*`` response headers. Bare async generators
cannot carry attributes, so header context was previously dropped.
"""
def __init__(
self,
completion_stream: AsyncIterator[bytes],
hidden_params: AnthropicMessagesStreamHiddenParams,
) -> None:
self.completion_stream = completion_stream
self._hidden_params = hidden_params
def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
return self
async def __anext__(self) -> bytes:
return await self.completion_stream.__anext__()
async def aclose(self) -> None:
await aclose_if_supported(self.completion_stream)
class BaseAnthropicMessagesStreamingIterator:
"""
@ -102,13 +194,18 @@ class BaseAnthropicMessagesStreamingIterator:
This method provides the common logic for both Anthropic and Bedrock implementations.
"""
collected_chunks = []
saw_terminal_event = False
async for chunk in completion_stream:
if self.completion_start_time is None:
self.completion_start_time = datetime.now()
saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk)
encoded_chunk = self._convert_chunk_to_sse_format(chunk)
collected_chunks.append(encoded_chunk)
yield encoded_chunk
if not saw_terminal_event:
yield _incomplete_stream_error_sse_event()
# Handle logging after all chunks are processed
await self._handle_streaming_logging(collected_chunks)

33
litellm/llms/azure/responses/transformation.py
View file

@ -205,7 +205,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
#########################################################
########## DELETE RESPONSE API TRANSFORMATION ##############
#########################################################
def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str) -> str:
def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str, path_suffix: str = "") -> str:
"""
Constructs a URL for the API request with the response_id in the path.
"""
@ -218,14 +218,14 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
encoded_response_id = encode_url_path_segment(response_id, field_name="response_id")
new_path = f"{path}/{encoded_response_id}"
new_path = f"{path}/{encoded_response_id}{path_suffix}"
# Reconstruct the URL with all original components but with the modified path
constructed_url = urlunparse(
(
parsed_url.scheme, # http, https
parsed_url.netloc, # domain name, port
new_path, # path with response_id added
new_path,
parsed_url.params, # parameters
parsed_url.query, # query string
parsed_url.fragment, # fragment
@ -288,7 +288,9 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
) -> Tuple[str, Dict]:
url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) + "/input_items"
url = self._construct_url_for_response_id_in_path(
api_base=api_base, response_id=response_id, path_suffix="/input_items"
)
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
@ -322,27 +324,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
This function handles URLs with query parameters by inserting the response_id
at the correct location (before any query parameters).
"""
from urllib.parse import urlparse, urlunparse
# Parse the URL to separate its components
parsed_url = urlparse(api_base)
# Insert the response_id and /cancel at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
encoded_response_id = encode_url_path_segment(response_id, field_name="response_id")
new_path = f"{path}/{encoded_response_id}/cancel"
# Reconstruct the URL with all original components but with the modified path
cancel_url = urlunparse(
(
parsed_url.scheme, # http, https
parsed_url.netloc, # domain name, port
new_path, # path with response_id and /cancel added
parsed_url.params, # parameters
parsed_url.query, # query string
parsed_url.fragment, # fragment
)
cancel_url = self._construct_url_for_response_id_in_path(
api_base=api_base, response_id=response_id, path_suffix="/cancel"
)
data: Dict = {}

356
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -15,6 +15,7 @@ from typing import Any, Dict
from urllib.parse import quote
import httpx
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
@ -38,6 +39,30 @@ from litellm.secret_managers.main import get_secret_str
AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"
class AzureDocumentIntelligenceLine(BaseModel):
content: str | None = None
class AzureDocumentIntelligencePage(BaseModel):
pageNumber: int | None = None
width: float | None = None
height: float | None = None
unit: str | None = None
lines: tuple[AzureDocumentIntelligenceLine, ...] = ()
class AzureDocumentIntelligenceAnalyzeResult(BaseModel):
content: str | None = None
pages: tuple[AzureDocumentIntelligencePage, ...] = ()
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
class AzureDocumentIntelligenceOperation(BaseModel):
status: str | None = None
analyzeResult: AzureDocumentIntelligenceAnalyzeResult | None = None
class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"""
Azure Document Intelligence OCR transformation configuration.
@ -67,11 +92,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
(1-based, e.g. "1-3,5,7-9"). To keep the public request shape
aligned with Mistral OCR, callers pass `pages` using Mistral
semantics — a list of 0-based integers — or a pre-formatted
Azure-style string. Other Mistral-specific params (e.g.
Azure-style string. Azure DI also exposes a `features` query
parameter enabling add-on capabilities (e.g. "keyValuePairs",
"languages"), passed as a list of feature names or a
comma-separated string. Other Mistral-specific params (e.g.
`include_image_base64`) are not supported by Azure DI and are
ignored during transformation.
"""
return ["pages"]
return ["pages", "features"]
def map_ocr_params(
self,
@ -85,16 +113,18 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Translates Mistral-style `pages` (list[int], 0-based) into Azure's
`pages` query string (1-based, e.g. "1,2,3" or "1-3,5"). A raw
string that already matches Azure's format is passed through
unchanged.
unchanged. `features` (list[str] or comma-separated string) is
normalized into Azure's comma-joined `features` query string.
"""
pages = non_default_params.get("pages")
if pages is None:
return optional_params
normalized = self._normalize_pages_param(pages)
if normalized:
optional_params["pages"] = normalized
return optional_params
features = non_default_params.get("features")
normalized_pages = self._normalize_pages_param(pages) if pages is not None else ""
normalized_features = self._normalize_features_param(features) if features is not None else ""
return {
**optional_params,
**({"pages": normalized_pages} if normalized_pages else {}),
**({"features": normalized_features} if normalized_features else {}),
}
@staticmethod
def _normalize_pages_param(pages: Any) -> str:
@ -140,6 +170,39 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
raise ValueError("`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'.")
@staticmethod
def _normalize_features_param(features: object) -> str:
"""
Convert a caller-provided `features` value to Azure DI's query-string
form (comma-joined feature names, e.g. "keyValuePairs,languages").
Accepted inputs:
- list[str]: feature names like ["keyValuePairs", "languages"].
- str: a single feature name or comma-separated names.
"""
invalid_features_error = ValueError(
f"Invalid `features` for Azure Document Intelligence: {features!r}. "
f"Expected a list of feature names or a comma-separated string like "
f"'keyValuePairs' or 'keyValuePairs,languages'."
)
if isinstance(features, str):
raw_tokens = features.split(",")
elif isinstance(features, list):
if len(features) == 0:
return ""
raw_tokens = [feature for feature in features if isinstance(feature, str)]
if len(raw_tokens) != len(features):
raise invalid_features_error
else:
raise invalid_features_error
tokens = tuple(token.strip() for token in raw_tokens)
feature_pattern = re.compile(r"^[A-Za-z][A-Za-z0-9]*$")
if not all(feature_pattern.match(token) for token in tokens):
raise invalid_features_error
return ",".join(tokens)
def validate_environment(
self,
headers: Dict,
@ -228,13 +291,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
)
# Azure DI accepts `pages` as a query param (1-based, e.g. "1-3,5").
# Azure DI accepts `pages` (1-based, e.g. "1-3,5") and `features`
# (comma-joined names, e.g. "keyValuePairs") as query params.
# `optional_params` has already been normalized in `map_ocr_params`.
pages = optional_params.get("pages") if optional_params else None
if pages:
url += f"&pages={quote(str(pages), safe=',-')}"
features = optional_params.get("features") if optional_params else None
pages_query = f"&pages={quote(str(pages), safe=',-')}" if pages else ""
features_query = f"&features={quote(str(features), safe=',')}" if features else ""
return url
return f"{url}{pages_query}{features_query}"
def _extract_base64_from_data_uri(self, data_uri: str) -> str:
"""
@ -328,27 +393,15 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
return OCRRequestData(data=data, files=None)
def _extract_page_markdown(self, page_data: Dict[str, Any]) -> str:
"""
Extract text from Azure DI page and format as markdown.
Azure DI provides text in 'lines' array. We concatenate them with newlines.
Args:
page_data: Azure DI page object
Returns:
Markdown-formatted text
"""
lines = page_data.get("lines", [])
if not lines:
return ""
# Extract text content from each line
text_lines = [line.get("content", "") for line in lines]
# Join with newlines to preserve structure
return "\n".join(text_lines)
def _transform_azure_page(self, azure_page: AzureDocumentIntelligencePage) -> OCRPage:
page_number = azure_page.pageNumber if azure_page.pageNumber is not None else 1
markdown = "\n".join(line.content or "" for line in azure_page.lines)
dimensions = self._convert_dimensions(
width=azure_page.width if azure_page.width is not None else 8.5,
height=azure_page.height if azure_page.height is not None else 11,
unit=azure_page.unit if azure_page.unit is not None else "inch",
)
return OCRPage(index=page_number - 1, markdown=markdown, dimensions=dimensions)
def _convert_dimensions(self, width: float, height: float, unit: str) -> OCRPageDimensions:
"""
@ -526,6 +579,52 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
retry_after = self._get_retry_after(response=response)
await asyncio.sleep(retry_after)
def _get_polling_target(self, raw_response: httpx.Response) -> tuple[str, Dict[str, str]]:
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
poll_headers = {"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")}
return operation_url, poll_headers
def _transform_completed_response(self, model: str, raw_response: httpx.Response) -> OCRResponse:
"""
Transform a completed Azure Document Intelligence analyze operation
into the Mistral OCR response shape, preserving Azure-native
`analyzeResult` fields (`content`, `tables`, `keyValuePairs`) as
top-level response fields.
"""
operation = AzureDocumentIntelligenceOperation.model_validate(raw_response.json())
verbose_logger.debug(f"Azure Document Intelligence response status: {operation.status}")
if operation.status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}")
analyze_result = (
operation.analyzeResult if operation.analyzeResult is not None else AzureDocumentIntelligenceAnalyzeResult()
)
mistral_pages = [self._transform_azure_page(azure_page) for azure_page in analyze_result.pages]
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
content=analyze_result.content,
tables=analyze_result.tables,
keyValuePairs=analyze_result.keyValuePairs,
)
def transform_ocr_response(
self,
model: str,
@ -552,11 +651,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"unit": "inch",
"lines": [{"content": "text", "boundingBox": [...]}]
}
]
],
"tables": [...],
"keyValuePairs": [...]
}
}
Mistral OCR format:
Mistral OCR format (with Azure-native fields preserved):
{
"pages": [
{
@ -567,7 +668,10 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
],
"model": "azure_ai/doc-intelligence/prebuilt-layout",
"usage_info": {"pages_processed": 1},
"object": "ocr"
"object": "ocr",
"content": "Full document text...",
"tables": [...],
"keyValuePairs": [...]
}
Args:
@ -578,86 +682,17 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Returns:
OCRResponse in Mistral format
"""
try:
# Check if we got 202 Accepted (async operation started)
if raw_response.status_code == 202:
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
if raw_response.status_code != 202:
return self._transform_completed_response(model=model, raw_response=raw_response)
# Get Operation-Location header
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")
}
# Get timeout from kwargs or use default
timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT
# Poll until operation completes
raw_response = self._poll_operation_sync(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=timeout_secs,
)
# Now parse the completed response
response_json = raw_response.json()
verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}")
# Check if request succeeded
status = response_json.get("status")
if status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}")
# Extract analyze result
analyze_result = response_json.get("analyzeResult", {})
azure_pages = analyze_result.get("pages", [])
# Transform pages to Mistral format
mistral_pages = []
for azure_page in azure_pages:
page_number = azure_page.get("pageNumber", 1)
index = page_number - 1 # Convert to 0-based index
# Extract markdown text
markdown = self._extract_page_markdown(azure_page)
# Convert dimensions
width = azure_page.get("width", 8.5)
height = azure_page.get("height", 11)
unit = azure_page.get("unit", "inch")
dimensions = self._convert_dimensions(width=width, height=height, unit=unit)
# Build OCR page
ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions)
mistral_pages.append(ocr_page)
# Build usage info
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
# Return Mistral OCR response
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error(f"Error parsing Azure Document Intelligence response: {e}")
raise e
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response = self._poll_operation_sync(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(model=model, raw_response=completed_response)
async def async_transform_ocr_response(
self,
@ -680,81 +715,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
Returns:
OCRResponse in Mistral format
"""
try:
# Check if we got 202 Accepted (async operation started)
if raw_response.status_code == 202:
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
if raw_response.status_code != 202:
return self._transform_completed_response(model=model, raw_response=raw_response)
# Get Operation-Location header
operation_url = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs (see sync path). VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")
}
# Get timeout from kwargs or use default
timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT
# Poll until operation completes (async)
raw_response = await self._poll_operation_async(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=timeout_secs,
)
# Now parse the completed response
response_json = raw_response.json()
verbose_logger.debug(f"Azure Document Intelligence response status: {response_json.get('status')}")
# Check if request succeeded
status = response_json.get("status")
if status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {status}")
# Extract analyze result
analyze_result = response_json.get("analyzeResult", {})
azure_pages = analyze_result.get("pages", [])
# Transform pages to Mistral format
mistral_pages = []
for azure_page in azure_pages:
page_number = azure_page.get("pageNumber", 1)
index = page_number - 1 # Convert to 0-based index
# Extract markdown text
markdown = self._extract_page_markdown(azure_page)
# Convert dimensions
width = azure_page.get("width", 8.5)
height = azure_page.get("height", 11)
unit = azure_page.get("unit", "inch")
dimensions = self._convert_dimensions(width=width, height=height, unit=unit)
# Build OCR page
ocr_page = OCRPage(index=index, markdown=markdown, dimensions=dimensions)
mistral_pages.append(ocr_page)
# Build usage info
usage_info = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
# Return Mistral OCR response
return OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error(f"Error parsing Azure Document Intelligence response (async): {e}")
raise e
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response = await self._poll_operation_async(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(model=model, raw_response=completed_response)

13
litellm/llms/base_llm/anthropic_messages/transformation.py
View file

@ -114,6 +114,19 @@ class BaseAnthropicMessagesConfig(ABC):
"""
return True
def handles_web_search_natively(self) -> bool:
"""
Whether the upstream this config routes to executes ``web_search`` tools
itself as part of its Anthropic Messages agentic loop.
The web-search interception handler short-circuits web-search-only
requests (running the search itself and returning synthetic results) only
for providers that do NOT. Providers whose agentic loop already performs
the search plus a follow-up synthesis step (bedrock, vertex_ai, ...)
return True so those requests flow through untouched.
"""
return True
def get_async_streaming_response_iterator(
self,
model: str,

29
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -2,7 +2,10 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, List, Optional
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import AllMessageValues
@ -98,6 +101,30 @@ class BaseTranslation(ABC):
"""
return responses_so_far
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Optional[list[Any]] = None,
) -> Optional[list[bytes]]:
"""
Build the streaming chunks that deliver a guardrail block message and
cleanly terminate the stream in this provider's wire format.
``stream_started`` is True when real chunks were already sent to the
client: the result must *continue* the in-progress message (e.g. close
the open content block and append the block message) rather than start
a new one, which clients reject. ``responses_so_far`` provides the prior
chunks needed to do so. When False, nothing has been sent and a
standalone block message is emitted.
Returns None when the format has no safe terminator; the caller then
re-raises ``exc`` so the proxy can surface a clean error instead.
Override in provider subclasses that support synthesizing a block
stream.
"""
return None
def get_structured_messages(self, data: dict) -> Optional[List["AllMessageValues"]]:
"""
Convert request data to OpenAI-spec structured messages.

92
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -1,10 +1,100 @@
from __future__ import annotations
from typing import Any, List
import json
from typing import Any, List, Optional
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues
def _anthropic_stream_chunk_events(item: Any) -> list[dict]:
if isinstance(item, dict):
return [item]
if isinstance(item, bytes):
chunk = item.decode("utf-8", errors="replace")
elif isinstance(item, str):
chunk = item
else:
return []
events: list[dict] = []
for block in chunk.split("\n\n"):
for line in block.splitlines():
stripped = line.strip()
if not stripped.startswith("data:"):
continue
payload = stripped[len("data:") :].strip()
if not payload or payload == "[DONE]":
continue
try:
parsed = json.loads(payload)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict):
events.append(parsed)
return events
def _usage_from_anthropic_stream_chunks(original_response: list[Any]) -> Optional[AnthropicUsage]:
input_tokens = 0
output_tokens = 0
found_usage = False
for item in original_response:
for event in _anthropic_stream_chunk_events(item):
event_type = event.get("type")
if event_type == "message_start":
message = event.get("message") or {}
usage_obj = message.get("usage") or {}
elif event_type == "message_delta":
usage_obj = event.get("usage") or {}
else:
usage_obj = {}
if not isinstance(usage_obj, dict):
continue
if usage_obj.get("input_tokens") is not None:
input_tokens = int(usage_obj.get("input_tokens") or 0)
found_usage = True
if usage_obj.get("output_tokens") is not None:
output_tokens = int(usage_obj.get("output_tokens") or 0)
found_usage = True
if not found_usage:
return None
return AnthropicUsage(input_tokens=input_tokens, output_tokens=output_tokens)
def blocked_response_usage(original_response: Optional[Any]) -> AnthropicUsage:
"""
Token usage for a synthetic guardrail-blocked response.
A post-call block replaces the LLM's response with the violation message,
but the upstream call already consumed tokens -- report that real usage
(carried on ``ModifyResponseException.original_response``) rather than
discarding it. Pre-call blocks never invoked the LLM (no original_response),
so usage is zero.
"""
usage_obj: Any = None
if isinstance(original_response, list):
stream_usage = _usage_from_anthropic_stream_chunks(original_response)
if stream_usage is not None:
return stream_usage
elif isinstance(original_response, dict):
usage_obj = original_response.get("usage")
elif original_response is not None:
usage_obj = getattr(original_response, "usage", None)
def _tokens(key: str, fallback_key: str) -> int:
if isinstance(usage_obj, dict):
return int(usage_obj.get(key, usage_obj.get(fallback_key, 0)) or 0)
return int(getattr(usage_obj, key, getattr(usage_obj, fallback_key, 0)) or 0)
return AnthropicUsage(
input_tokens=_tokens("input_tokens", "prompt_tokens"),
output_tokens=_tokens("output_tokens", "completion_tokens"),
)
def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None)
if per is not None:

3
litellm/llms/base_llm/ocr/transformation.py
View file

@ -70,6 +70,9 @@ class OCRResponse(LiteLLMPydanticObjectBase):
model: str
document_annotation: Any | None = None
usage_info: OCRUsageInfo | None = None
content: str | None = None
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
object: str = "ocr"
model_config = {"extra": "allow"}

28
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -76,6 +76,7 @@ from litellm.utils import (
from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_converse_supports_parallel_tool_use_config,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_claude_4_5_on_bedrock,
@ -1106,18 +1107,28 @@ class AmazonConverseConfig(BaseConfig):
if cache_control is None:
return None
cache_point = CachePointBlock(type="default")
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None:
if is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
cache_point = self._build_cache_point_block(cache_control, model)
if block_type == "system":
return SystemContentBlock(cachePoint=cache_point)
else:
return ContentBlock(cachePoint=cache_point)
@staticmethod
def _build_cache_point_block(control: Optional[dict], model: Optional[str] = None) -> CachePointBlock:
"""Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict.
``type`` is always ``"default"`` (the only value Bedrock's Converse API
accepts). ``ttl`` is only honored for models that support extended TTL
caching (Claude 4.5 family on Bedrock).
"""
cache_point = CachePointBlock(type="default")
if isinstance(control, dict) and "ttl" in control:
ttl = control["ttl"]
if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
return cache_point
def _transform_system_message(
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
@ -1241,7 +1252,7 @@ class AmazonConverseConfig(BaseConfig):
# Handle parallel_tool_calls configuration
parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None)
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
if parallel_tool_use_config is not None and bedrock_converse_supports_parallel_tool_use_config(model):
for key, value in parallel_tool_use_config.items():
if (
key in additional_request_params
@ -1526,7 +1537,8 @@ class AmazonConverseConfig(BaseConfig):
if cache_injection_points and len(bedrock_tools) > 0:
for point in cache_injection_points:
if point.get("location") == "tool_config":
bedrock_tools.append({"cachePoint": {"type": "default"}})
cache_point = self._build_cache_point_block(point.get("control"), model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
break
bedrock_tool_config: Optional[ToolConfigBlock] = None

2
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1558,7 +1558,7 @@ class AWSEventStreamDecoder:
text = chunk_data["outputText"]
# ai21 mapping
elif "ai21" in self.model: # fake ai21 streaming
text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore
text = chunk_data["completions"][0]["data"]["text"]
is_finished = True
finish_reason = "stop"
######## /bedrock/converse mappings ###############

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py
View file

@ -51,10 +51,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
Qwen2 uses "text" field, but we also support "generation" field for compatibility.
"""
try:
if hasattr(raw_response, "json"):
response_data = raw_response.json()
else:
response_data = raw_response
response_data = raw_response.json()
# Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
generated_text = response_data.get("generation", "") or response_data.get("text", "")

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py
View file

@ -175,10 +175,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
Transform Qwen3 Bedrock response to OpenAI format
"""
try:
if hasattr(raw_response, "json"):
response_data = raw_response.json()
else:
response_data = raw_response
response_data = raw_response.json()
# Extract the generated text - Qwen3 uses "generation" field
generated_text = response_data.get("generation", "")

45
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -5,6 +5,7 @@ from functools import partial
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args
import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -24,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
_get_httpx_client,
)
from litellm.types.llms.bedrock import GuardrailConfigBlock
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
@ -37,6 +39,38 @@ else:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
_GUARDRAIL_CONFIG_VALIDATOR: "TypeAdapter[GuardrailConfigBlock]" = TypeAdapter(GuardrailConfigBlock)
_GUARDRAIL_CONFIG_EXPECTED_FORMAT = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
try:
guardrail_config = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config)
except ValidationError as e:
raise BedrockError(
status_code=400,
message="Invalid guardrailConfig={}. Expected format: {}. Error: {}".format(
raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT, e
),
)
if "guardrailIdentifier" not in guardrail_config:
raise BedrockError(
status_code=400,
message="guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {}".format(
raw_guardrail_config, _GUARDRAIL_CONFIG_EXPECTED_FORMAT
),
)
trace = guardrail_config.get("trace")
candidate_headers = {
"X-Amzn-Bedrock-GuardrailIdentifier": guardrail_config.get("guardrailIdentifier"),
"X-Amzn-Bedrock-GuardrailVersion": guardrail_config.get("guardrailVersion"),
"X-Amzn-Bedrock-Trace": trace.upper() if trace is not None else None,
}
return {name: value for name, value in candidate_headers.items() if value is not None}
class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
def __init__(self, **kwargs):
@ -390,7 +424,16 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
return headers
raw_guardrail_config = optional_params.pop("guardrailConfig", None)
if raw_guardrail_config is None:
return headers
existing_header_names = frozenset(name.lower() for name in headers)
guardrail_headers = {
name: value
for name, value in _bedrock_invoke_guardrail_headers(raw_guardrail_config).items()
if name.lower() not in existing_header_names
}
return {**headers, **guardrail_headers}
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]

95
litellm/llms/bedrock/common_utils.py
View file

@ -4,9 +4,11 @@ from __future__ import annotations
Common utilities used across bedrock chat/embedding/image generation
"""
import contextlib
import functools
import json
import os
import re
from typing import (
TYPE_CHECKING,
Any,
@ -683,39 +685,72 @@ def get_bedrock_base_model(model: str) -> str:
return model
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
return any(
(litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True
for candidate in (model, get_bedrock_base_model(model))
)
def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model is a Claude 4.5 model on Bedrock.
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
Check if the model supports Bedrock prompt caching with an extended '1h' TTL
(in addition to the default 5m TTL).
Backed by the ``cache_creation_input_token_cost_above_1hr`` field in
``model_prices_and_context_window.json`` instead of a hardcoded list of
model-name patterns, so newly released models pick up support as soon as
their pricing entry ships, with no code change required here.
"""
model_lower = model.lower()
claude_4_5_patterns = [
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
"haiku-4.5",
"haiku_4.5",
"haiku-4-5",
"haiku_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
"sonnet-4.6",
"sonnet_4.6",
"sonnet-4-6",
"sonnet_4_6",
"opus-4.6",
"opus_4.6",
"opus-4-6",
"opus_4_6",
"opus-4.7",
"opus_4.7",
"opus-4-7",
"opus_4_7",
]
return any(pattern in model_lower for pattern in claude_4_5_patterns)
return any(
(litellm.model_cost.get(candidate) or {}).get("cache_creation_input_token_cost_above_1hr") is not None
for candidate in (model, get_bedrock_base_model(model))
)
_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$")
def bedrock_converse_supports_strict_tools(model: str) -> bool:
"""
Whether ``toolSpec.strict`` can be forwarded to Bedrock Converse for ``model``.
Non-Anthropic Bedrock families (Nova, Llama, GPT-OSS) reject the field
outright. Anthropic models forward it unless their entry in
``model_prices_and_context_window.json`` sets
``bedrock_converse_supports_strict_tools: false`` — Bedrock routes those
(Opus 4.7/4.8, see #31582) through a stricter validator that rejects the
``strict`` key on ``toolSpec`` even though Anthropic's native API accepts
it as a top-level tool field.
"""
base = get_bedrock_base_model(model)
if not base.startswith("anthropic"):
return False
flag = _get_bedrock_converse_strict_tools_flag(base)
return flag if flag is not None else True
def _get_bedrock_converse_strict_tools_flag(base_model: str) -> Optional[bool]:
candidates = dict.fromkeys((base_model, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base_model)))
for candidate in candidates:
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
custom_llm_provider="bedrock",
)
flag = model_info.get("bedrock_converse_supports_strict_tools")
if isinstance(flag, bool):
return flag
model_cost_key = model_info.get("key")
if isinstance(model_cost_key, str):
local_flag = (
_get_local_model_cost_map().get(model_cost_key, {}).get("bedrock_converse_supports_strict_tools")
)
if isinstance(local_flag, bool):
return local_flag
return None
def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None:

69
litellm/llms/bedrock/realtime/handler.py
View file

@ -5,6 +5,7 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
"""
import asyncio
import contextlib
import json
from typing import Any, Optional
@ -12,6 +13,7 @@ from litellm._logging import _redact_string, verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig
@ -58,9 +60,7 @@ class BedrockRealtime(BaseAWSLLM):
InvokeModelWithBidirectionalStreamOperationInput,
)
from aws_sdk_bedrock_runtime.config import Config
from smithy_aws_core.identity.environment import (
EnvironmentCredentialsResolver,
)
from smithy_aws_core.identity import StaticCredentialsResolver
except ImportError:
raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime")
@ -81,11 +81,36 @@ class BedrockRealtime(BaseAWSLLM):
verbose_proxy_logger.debug(f"Bedrock Realtime: Connecting to {endpoint_uri} with model {model}")
credentials = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
if credentials is None:
raise BedrockError(
status_code=401,
message=(
"No AWS credentials found for Bedrock realtime. Set aws_* params in litellm_params "
"or configure credentials in the environment"
),
)
frozen_credentials = credentials.get_frozen_credentials()
# Initialize Bedrock client with aws_sdk_bedrock_runtime
config = Config(
endpoint_uri=endpoint_uri,
region=aws_region_name,
aws_credentials_identity_resolver=EnvironmentCredentialsResolver(),
aws_access_key_id=frozen_credentials.access_key,
aws_secret_access_key=frozen_credentials.secret_key,
aws_session_token=frozen_credentials.token,
aws_credentials_identity_resolver=StaticCredentialsResolver(),
)
bedrock_client = BedrockRuntimeClient(config=config)
@ -156,12 +181,19 @@ class BedrockRealtime(BaseAWSLLM):
session_state: dict,
):
"""Forward messages from client WebSocket to Bedrock stream."""
try:
from aws_sdk_bedrock_runtime.models import (
BidirectionalInputPayloadPart,
InvokeModelWithBidirectionalStreamInputChunk,
)
from aws_sdk_bedrock_runtime.models import (
BidirectionalInputPayloadPart,
InvokeModelWithBidirectionalStreamInputChunk,
)
async def send_to_bedrock(bedrock_message: str) -> None:
event = InvokeModelWithBidirectionalStreamInputChunk(
value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8"))
)
await bedrock_stream.input_stream.send(event)
verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}")
try:
while True:
# Receive message from client
message = await client_ws.receive_text()
@ -176,19 +208,15 @@ class BedrockRealtime(BaseAWSLLM):
# Send transformed messages to Bedrock
for bedrock_message in transformed_messages:
event = InvokeModelWithBidirectionalStreamInputChunk(
value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8"))
)
await bedrock_stream.input_stream.send(event)
verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}")
await send_to_bedrock(bedrock_message)
except Exception as e:
verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True)
# Close the Bedrock stream input
try:
for close_message in transformation_config.session_close_messages():
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
except Exception:
pass
async def _forward_bedrock_to_client(
self,
@ -206,6 +234,10 @@ class BedrockRealtime(BaseAWSLLM):
output = await bedrock_stream.await_output()
result = await output[1].receive()
if result is None:
verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
break
if result.value and result.value.bytes_:
bedrock_response = result.value.bytes_.decode("utf-8")
verbose_proxy_logger.debug(f"Bedrock Realtime: Received from Bedrock: {bedrock_response[:200]}")
@ -252,6 +284,7 @@ class BedrockRealtime(BaseAWSLLM):
except Exception as e:
verbose_proxy_logger.debug(f"Bedrock to client forwarding ended: {e}", exc_info=True)
finally:
# Close the client WebSocket
try:
await client_ws.close()

146
litellm/llms/bedrock/realtime/transformation.py
View file

@ -4,14 +4,18 @@ This file contains the transformation logic for Bedrock Nova Sonic realtime API.
Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format.
"""
import base64
import json
import uuid as uuid_lib
from typing import Any, List, Optional, Union
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.openai import (
OpenAIRealtimeContentPartDone,
OpenAIRealtimeDoneEvent,
@ -35,6 +39,17 @@ from litellm.types.realtime import (
from litellm.utils import get_empty_usage
class BedrockContentEnd(BaseModel):
stopReason: Optional[str] = None
TRIGGER_AUDIO_SAMPLE_RATE_HERTZ = 16000
TRIGGER_AUDIO_BYTES_PER_SECOND = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2
TRIGGER_LEADING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2)
TRIGGER_TRAILING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND * 3)
TRIGGER_AUDIO_CHUNK_SIZE = 1024
class BedrockRealtimeConfig(BaseRealtimeConfig):
"""Configuration for Bedrock Nova Sonic realtime transformations."""
@ -43,6 +58,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self.prompt_name = str(uuid_lib.uuid4())
self.content_name = str(uuid_lib.uuid4())
self.audio_content_name = str(uuid_lib.uuid4())
self.prompt_started = False
self.client_audio_streamed = False
# Default configuration values
# Inference configuration
@ -247,6 +264,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
prompt_start = {"event": {"promptStart": prompt_start_config}}
messages.append(json.dumps(prompt_start))
self.prompt_started = True
# Send system prompt if provided
instructions = session_config.get("instructions")
@ -304,8 +322,22 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling input_audio_buffer.append")
self.client_audio_streamed = True
messages: List[str] = []
if hasattr(self, "_audio_content_started") and self._audio_content_sample_rate != self.input_sample_rate_hertz:
mismatched_content_end = {
"event": {
"contentEnd": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
}
}
}
messages.append(json.dumps(mismatched_content_end))
delattr(self, "_audio_content_started")
self.audio_content_name = str(uuid_lib.uuid4())
# Check if we need to start audio content
if not hasattr(self, "_audio_content_started"):
audio_content_start = {
@ -329,6 +361,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
}
messages.append(json.dumps(audio_content_start))
self._audio_content_started = True
self._audio_content_sample_rate = self.input_sample_rate_hertz
# Send audio chunk
audio_data = json_message.get("audio", "")
@ -383,7 +416,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling conversation.item.create")
messages: List[str] = []
item = json_message.get("item", {})
item_type = item.get("type")
@ -392,6 +424,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if item_type == "function_call_output":
return self.transform_conversation_item_create_tool_result_event(json_message)
messages: list[str] = []
# Handle regular message
if item_type == "message":
content = item.get("content", [])
@ -443,6 +477,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
"""
Transform response.create event to Bedrock format.
Nova Sonic only starts generating after it detects user speech, so text-only
sessions never get a response on their own. Injecting a short spoken "ready"
utterance (followed by silence) makes the model respond to the pending
interactive text input. Sessions where the client streams its own audio rely
on Nova Sonic's built-in turn detection instead.
Args:
json_message: OpenAI response.create message
@ -450,8 +490,53 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling response.create")
# Bedrock starts generating automatically, no explicit trigger needed
return []
if not self.prompt_started or self.client_audio_streamed:
return []
messages: list[str] = []
if not hasattr(self, "_audio_content_started"):
trigger_content_start = {
"event": {
"contentStart": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
"type": "AUDIO",
"interactive": True,
"role": "USER",
"audioInputConfiguration": {
"mediaType": self.input_media_type,
"sampleRateHertz": TRIGGER_AUDIO_SAMPLE_RATE_HERTZ,
"sampleSizeBits": self.input_sample_size_bits,
"channelCount": self.input_channel_count,
"audioType": self.input_audio_type,
"encoding": self.input_encoding,
},
}
}
}
messages.append(json.dumps(trigger_content_start))
self._audio_content_started = True
self._audio_content_sample_rate = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ
messages.extend(self._response_trigger_audio_messages())
return messages
def _response_trigger_audio_messages(self) -> list[str]:
pcm = TRIGGER_LEADING_SILENCE + ready_trigger_pcm() + TRIGGER_TRAILING_SILENCE
return [
json.dumps(
{
"event": {
"audioInput": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
"content": base64.b64encode(pcm[offset : offset + TRIGGER_AUDIO_CHUNK_SIZE]).decode(),
}
}
}
)
for offset in range(0, len(pcm), TRIGGER_AUDIO_CHUNK_SIZE)
]
def transform_response_cancel_event(self, json_message: dict) -> List[str]:
"""
@ -467,6 +552,35 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Send interrupt signal if needed
return []
def session_close_messages(self) -> list[str]:
"""
Build the Bedrock events that gracefully close the session
(contentEnd for any open audio content, promptEnd, sessionEnd).
Returns:
List of Bedrock format messages (JSON strings)
"""
if not self.prompt_started:
return []
messages: list[str] = []
if hasattr(self, "_audio_content_started"):
audio_content_end = {
"event": {
"contentEnd": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
}
}
}
messages.append(json.dumps(audio_content_end))
delattr(self, "_audio_content_started")
messages.append(json.dumps({"event": {"promptEnd": {"promptName": self.prompt_name}}}))
messages.append(json.dumps({"event": {"sessionEnd": {}}}))
self.prompt_started = False
return messages
def transform_realtime_request(
self,
message: str,
@ -837,10 +951,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Optional[ALL_DELTA_TYPES],
]:
"""
Transform Bedrock promptEnd event to OpenAI response.done.
Transform a Bedrock end-of-response event (promptEnd, completionEnd, or an
END_TURN contentEnd) to OpenAI response.done.
Args:
event: Bedrock promptEnd event
event: Bedrock event that ends the response
current_response_id: Current response ID
current_conversation_id: Current conversation ID
@ -848,7 +963,18 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Tuple of (events, reset_output_item_id, reset_response_id, reset_delta_type)
"""
verbose_logger.debug("Handling promptEnd")
return self._response_done_events(current_response_id, current_conversation_id)
def _response_done_events(
self,
current_response_id: Optional[str],
current_conversation_id: Optional[str],
) -> tuple[
List[OpenAIRealtimeEvents],
Optional[str],
Optional[str],
Optional[ALL_DELTA_TYPES],
]:
if not current_response_id or not current_conversation_id:
return [], None, None, None
@ -1084,6 +1210,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
current_delta_chunks,
)
returned_messages.extend(events)
if BedrockContentEnd.model_validate(event["contentEnd"]).stopReason == "END_TURN":
(
done_events,
current_output_item_id,
current_response_id,
current_delta_type,
) = self._response_done_events(current_response_id, current_conversation_id)
returned_messages.extend(done_events)
elif "toolUse" in event:
events, tool_call_id, tool_name = self.transform_tool_use_event(
@ -1093,7 +1227,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Store tool call info for potential use
verbose_logger.debug(f"Tool use event: {tool_name} (ID: {tool_call_id})")
elif "promptEnd" in event:
elif "promptEnd" in event or "completionEnd" in event:
(
events,
current_output_item_id,

208
litellm/llms/bedrock/realtime/trigger_audio.py Normal file
View file

@ -0,0 +1,208 @@
"""
Pre-rendered spoken "ready" trigger audio (16kHz, 16-bit, mono PCM), generated with Amazon Polly.
Amazon Nova Sonic v1 only starts generating after it hears the user speak, so text-only realtime
sessions inject this short utterance to trigger a response (same approach as Pipecat's
AWSNovaSonicLLMService assistant-response trigger).
"""
import base64
import gzip
from functools import lru_cache
READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64 = (
"H4sIANGpRWoC/517dXQcR9Bnw+Duis3MzMwkc8xsxxTZMTMzM0PMmJhBjpmZmWKQZSaxFrQ80H0lJXf3vXf/nev17ExPQ3Xh"
"r2wPQv8/f/D/uMP/zxv8f64YkSyiWU1AIpCcRSqyICtQCApF4SgSRaHsKCfKjfKhgqgIKo5KobKoPKqEqqFaqC5qgBqjFqg1"
"ao86o+6oF+qDYtAgNBSNQKPQODQeTUZT0Uw0B81D89ECtAgtRcvRMrQCrQRahVajNWgtWo/+QBvQJrQRbQbagraibWg70Da4"
"2/rf7zbozxyxCcaug1krYZXFaCGai2ajGWgKmgi7jYFdh6ABqB/qjXqgLqgd8NUcNQUe66CaqDJwXQaVhDMUgNNkh7NZ4bSc"
"69zLXTyVJ/Kv/BN/z9/w5/w+0B1+g1/kZ/gpfoL/zQ/xA/wvvovv5Jv5Fmjr+Tq+lq/iS/kioPl8Fp+eRRP4KKBhfBDvw3vw"
"Lrw978jb8Fa8JW8B1JQ34PV5Q2iZ1wa8Ea/Da/Ja0Gpm9TTmzXgT3hzGt+a/wNz2vBPM78TbwtyW0NuIV+fVeCWgCrwyUHmg"
"yll9NWClOrwurNA0a42GMLYerFoN3pf7j8rwYkBFeWFehOcHKshz8dw8Ow/nUTyCh/IwaDYuc4mrPATIBs8WeJMNxmTnOWF0"
"Tp4DnsK4lYscc8405mNulsFc0FKYg9lZKjQX3KXDcxJck+E5haWxRPYd2k/2hX2G9gkonr1nX6H3C7RkGOmB1QjsGckLAK+1"
"QAqdeQwfD7Jdz3fzk/we/4f/5D4uoVyoGOizOdjbMLCw+WBBO1Esuo7uoZfoE/qJnCgARq7iUByGs+G8OGcWReAQbMUCDqBk"
"FI9eoGvoGFjVSrCdAWAr9VBhsHsHf8EP87m8F8gV88dsBxvFajKJvTS3mjFmOdNnXDMWG12NUoZPf6jv0afpnfRKejadaena"
"B+219kqL0xK0oBamV9E761P1I3qcLhhVjL7GEuOckWAUMHube02nWZ/NY7dYBFjJZR4JNnsRqbgr3o7f4WykNZlJDpEH5AfR"
"SRgtQMvTarQJbUZb0JZA7Wjr/1o7+gs8N6K1aRmal9qoSRLJC3KF7CHLyUTSl7QhtUkxkptkIyEklOQghUgF0oh0IINg/WVk"
"O9lLjsI+u8kfZC4ZQ/qQZqQ6jM9DchIbkQglfpyKP+Cb+DDejBfiEbgDrogx/gftQWNRbaTx23wBjwYbuMwmsfLsq7nZ7Ghi"
"84IxHs7r1A/rA/Wiery2UeukRWiPg4uD0UEeuBSYFWgQEAPP/dv9k/xd/K387fy9/DP9x/2p/mqB1QEjMCvIg/M1m75Tr2m8"
"NuaZ5dhrNoaHQnzIDpzUIzdJLXqS5hEmCVcFJpQXu4gjxelAk8XBYgexgiiLP4XLwmphkNBAyCkk0yt0Be1HK1JCX5JdZDSp"
"D2d7htfhLmAXX9FR0H5XVAPlQDp4/xN+BezsCHj6Xrge4cfA9y/wS/w6nPUxWN97/h2ihIObXISoWASiXwxEtDPIjRriTTiA"
"B5N3pD19RH8RbgilxWXiZ7GI1EUaLU2QxkojpcHSUGmGtEO6KX2UXkvbpYaSQ7wq/iXuEleKQ8XS4gdhsVBXEISjoNd3ZBzJ"
"S27hCbgMfgeRTUGHeXfuY6tZFfbQ7G+mGmOMJH2UTvRNWnZtfjA8eBjk+srfx+/x7fQ18/m8+739veW8yPvec8nzl2ejZzXQ"
"Ns9pz2dPiLeld4+3iO+pb6t/RWBr8IL2UY8wm7Pp/AC6huPIF/pOuC1ul4bKuZVDSnF1kfpUDbPUs/xq6WmpaZEtD9TJaqi6"
"QymlHJAj5KnSM7GYOF64SYvTtSSC7MB1cALE6WbIC/GyG8QaESJKAfDm2TyWx3EZInAryA9LQfZvwOor4c54PF4B8juGn+A0"
"HEWqgAf0JkPIeDKFTCOryA5yiaQSG42mI+hf9DvNLvQXrgg2sZ+4V/wilpH6SFulT1JReYJ8WU6QsynFlbxASPkiX5S3yNPk"
"0fJkebm8RO4ox0lNpbNia/Gj0BEsYxxYhZ1cJOdJE3IWM8h0vfgpFsVGmaeNaOOL3kK/ohXV2gfnBL74R/q/+rr57N6V3ibe"
"gGevp4fnm3uuu7T7bcaJjJkZTTMsGfddm11jXF1dfVyzXBdcpTOeZFx2f/HU8j33rww207Obz9l0VJqcoBXEQ1KyXEgtYgla"
"TlgL2ZbZ/rGl2GwheUKUkHhbN9t1a6S1iWWM2kepJxeTUoQLdD5pgWuiWnwWe282Ac3vNhZBNBllTDBWGDuNNwYzmpixZgTb"
"ysqBhJtDDOyL43E0nCw/HUPP0jTwlipCKaGQQIR4eotupcNoPcrJY3KP3CEmaUKnUZ22F24KMeItcbKUXf5DrqUsUxxKe3Wc"
"+lBNVQVLW8twy2kLs2S3NrY2tFqsryytLN/VieoLpaEyV/ZLG6VGUlDcKo4VO4uR4mOhs3CI/iQyyY3t/CmLNfcaF/TT2sXg"
"+0BUoLd/r4/4fvde80ieTu6/M1Jc0a5LzvzOBY4P9qH2bPaE9Pj0T+mivaZ9sv20XbfncNR3vHY8dHbPWOQp6v9VSzXX4wpi"
"HWWK5Zw1xtbd9t46yeq0PLHkt26y9rXlDukSEh1yz5ZgPWwppw6VN4qKsIL0wQPQJH6WtQH9Kmwge8em8ZFoNP6NtKDXAf5E"
"Co/AunLQzPiZi26gFYTzgiSGiYeEdHqKNMKneW0WahY2Ruo+7Yi2SzsBkT6P3lZfol/Tdb2+McLYblw2TNDEA/MhU5FANgnD"
"5BWWniEnw6ZHDIqcGZk7ck7EzfAe4ZXDK4WvCufhvSPORFyJGBnxPJyHpYW+DEm1mdYm1o2WoDpCjVOqKmvlEvJtaaqUXWom"
"NhLiSS5cmzczF+rZtBzBsYEE/xV/G/8nXz/feu87j+zp5l6TkTfjlquFq4DrunOUs6bzo2OsQ3Vst3ezlwdJNrBXg2uUXU8v"
"YRccp5293WX85fSd/C4tIt9UW1jnW7mljWWk2k5NV1Msw21KaL6wa2F/hrUPXWOba/lFmShlE2/SPeQNfoZsaDS/xtYyCRAS"
"Q71IN+qhbYRswl+0Ab1N/OQcfSa0l47Km5X3ynilh/xSjBIakivoL/6aDWcCm21+MmoYO/WC+m1tnrZUO6p91urq8/THeg8j"
"YHw28/Ih6DO+TW+KcfJE4KVw6O2wHeHFImIi9PDocCNsadiksPSwjLA3YTfCqoZ1Cz0fcsrmsi6ydrXOtn6z5rAxqwJetNBa"
"w/qPpYflndoRYtVpeYpUW/TRu2QktqKebJtxR7se8Phue85nNHGZjuoOh32MvYy9kP1D+iOgielqesv0kukF0wdCs6ZPT7+V"
"Xsf+1Z5gz+ko6Ryc8dHr19aiuaKpbrC1BaA02rpcfaNUV7tbGtkehFQJ9YSk2aKsK9SH8hWxibCFzEM9WG/Dqbm02UYCy0W+"
"CbOlmfJ4eZf0U7whrpAaKVUsH6wOW/mQt7bBtlfWn5azqqjYxao0G+pu1tK7a39px/U4Y7hZzaxvJGoDg7kCp/yJ/seBGE00"
"jplTeE9cmhYSp0mfpXjJK54RHgmzxDxyjCU8pFrYwvBLYUbIV2tXdYjslC5IyZJPPqUsVwOWYrZFIQmhxcECK4d+tcVa21kr"
"2mJDDobVi4iNiIooGUZCcljnqcWUSPmBGCGEkv68jtle7x787v/V53UnuD44CtuT0r+m77B3dMQ4tzvDnDmcC+3v01ukR6Tt"
"TU1NiUlpnKKkzk4bm97GUdn5wFHa2cQx3jnE0zJoQy7Rp26x9rP+Zsmh7lJKKHOVEeoPJUbpLn8SL4rVwffO0R0oYAwPCv6u"
"vqf+/noV/ogchlKFC7eFH+JB+YCabmliZdZctiW2DiH3Q1qFjg2Js2ZT44VpaKJRVOOBtYGMwG/BaUF74GwgIrg9uF7bodcF"
"9HZTb2M8Z+GkjlhfGaJ6lRVyZbEEnYC9fBZCdKT4WZYtsyDnzVRtymepqFQTOLog51Z3WIbZzoa8CG0YViwsLLRqyGXbUlux"
"0Blh9cIPRoRHvo14H/4+9KXtpkVTHssnpcLSafEvYSe1ksm8tnldEwN7vf3d91yHXYMztrtPex54FnoKu486v9u/p81Lq2Zf"
"51jhDHfddaxOb5w2K+Vz0vDk2sk1UtypQ1MrplZKr5ihBYeRnmopm2RrrtaX5otLxFNSYbWvdZ/NbZmjZJPzilNoE4xMR6CJ"
"d7a7jO+e9pYdJctETTwtxNB89K6QJEdaZ4WsCWsUPjl8emT5qKURR0K3qIwW5tX1ZsHHwdMQ15jWMVjMH+mb7X3oUwLvA9O1"
"3kYF3pRUFB/LLdVcahl5s5CPHAL0nIRvkXByGM1k7dkhXhDF4TkkkgSIRNcLprRV2WM5bb1pfWxZot5TvZarNlfIr6GvQ3rb"
"zlrXWivbqobst32wVrW+VAurMapb3Q8Svw6RppVYXahEz5JpaIy5WGvk/8tb1fsNUEwzb0fvBPdJl+o8YGfpetqxlE7JD5OV"
"ZHeyJd2bylJbptLEbgkzEub8nJrUPqUa2F2T9J0Zn/RE8aytctigkHVyfmrisUKI8kW9Z+mulpSGkXGsvDnUSAi6vH9nrHYd"
"zNjjL8C60fFSXmm+MFxoJc6QhlpWhA4Mt0SWiwyNWBs+MnS+db9yQtjO8xjN9ViwpSFGV+2Cv5D3c0ZOT/NAV8PkB/FG8oHO"
"EHaJQ+W2ynO5gtiQLIQKWoUKeTeqz18YX4I5AgW110YaTyd++kpoK42RXyjvLZutO61rbKlwHWfbZntqq2UbaTmtDJDPSivE"
"Z+Im+Z1yXm2reuWncmu5lrxIvqx8VzR5tXwEUO99uh3FsbV6seBNX3VvVY+asTVjbUaM+45nrPuOa7Kjpb1C6viU96lbU3+H"
"bLAqvaj9YdqQVG9S++SMpEpJ+ZOvJfZPupU8zLFBmyuWCskRut26Rub4LkvlbmGuckC9r1SUDpJu5qLACt8Zz09XlPOKc73n"
"czBonsPRtBO5C7V/FD0rb7A5w/dGXc/2W9THsDO2CEsZOU24Txriv9EP/oYdMO/q6wKPvfc8m7xlghfMmmSdWFdC0kBxn9hf"
"6iwtkBLFPmJNcZjgpnXJYtSLbdE2Bry+rv7DOkVf6BbpqhQQZ0tHpRRptdoSLOpL6PzQyyFNbYssJRVBuihECcWFC5Ku/KME"
"lFFKafmlRKSOUi55plLUkmLxWH9YTXWaPFdcCRmpBA81rgU/Brr48/sEr83TMyObe2LGtYyKGdQlgVfOs/9m/9W+1RnpLOTa"
"6yzuLOOw2KemD087n5or7XJqVNrxlI2OzcEmwjXLGltTWz7rXsmHR6D2Qg85XGmu9FduSHVJvLbYW9Q9OKOFu6N/oLGTj0WD"
"+RjeEkXjz2SfOF5Ntk0NC4scEtUj8n3IA6Wl+E4whfeCIrYQPuKnrKYR0P4MouDvwWjdyk+SseIieaKsgLzaiGWl3+UcgL/t"
"4kmxoxCPu3HNGKiN97cEVHAv8IFtIa3EpVJuqb+0Uv6hlLXeCu0R3i78aHjZ8KuhD6wB5Z3UX2wijhFVmaiXFEWeJM2VqskN"
"lGfqaUtXywBLJ8tjS0OrWx0sj4DYeAQ/Rk9YUb2Xv41HdI/wxPk3BqoHznkPuG46cju+2rO7Jrlfuld5Orh7uMY6UtNcKcVT"
"XqRMs/d2fAFLy5WamPwlMT4h2rmcpakTbLct9eQUWhg9QOWkebarIdstneUV4lK0UkvwXM8o5U7y7AycMP38lrkkuMbPgkvM"
"R/ik/MV6JvRC2OKw26HlQ2pZtsrvpMUKs3yw3Fbmih5eTTvvzevt6m+vjTFGQs1o5fXYXwyjFrgHnSp2lgeotS0n1HxKfekG"
"nYldPC8aiUrxDuYzbV1wj57MMvA3GifIYkF5gtJC/R3QbnnrGdvvIYNCRoSsty2zrJEjxDHCNGGcmENi4haxkVhQbCkmin+K"
"1aQSck8lQemozlOGSSWFFqQCNnkzNkDPHsjh++p54tnofe4p7NnnphkhzucO2RHpPORIcrRycecl++r0hmkd0i6nXUvLbk+2"
"b007n9wv0f+z18/eSdczTqCnwEV1eSjNiYuwYby1MMYSb8WW5/IF8Q46qKV6DrivePr79wSjjFxGpDbKN80bHqhp3ECquFZ5"
"py5XO6uJ6lJLH6jdHljKWT+ABMrL+ehD9lgrGiyiLTT2mR+Nq3p+QJUHg80NiU/A2+lyyCo7lA9ST7G4oJFtpAytJ8wX2pN1"
"bKNeWauqt2Nf8HlaR9SEUmJJKb/cTj6gRFjyWX22waHZQh/aBloqKWmARLLJY+Rlsk1OETeLzaTfpUZyqNxdKg4I5Kh0XEwX"
"2gr5hFPUiVvxFMMZXO2zeX96Xnp+95z0rPBU8sx1j3P5HGccVR0THCcc+ZzvHbvBzkTHqfS/0lhq7rRHqS9SHiUfT56WJCVO"
"TinnsbM4saFcU/ydKLgl7k5LyonKCbWepaKlitKcxurEl99d2X3Jt16/ZlbVW/o2ua+5a/h6Bp+bY8k46aO62/YjZKtttGW0"
"ck8pad0Y2jS8avhj2yC5Iz5ohhvT9C/aJL2u3kZfpGXTbgS1wOBAC20qy0PGCNehIh1J+9PG9D3dJBQUDNwY70EneF+eyErx"
"bmgk/5v7UVN8nRQUZovD5EbKW6WQpT6g7t8tFdWDUGVvV+up1dRR6gGlkdJPOSvHS3OkKOmq6Befi7PEOKjLiJhIV5BsuBeP"
"NQO6qB/S2gUf+Gv5Zd8nTzvPM3fpjDEZbd0lM+yueq5RjvaOk/Yj6eUcOR2b06um9Ujbl/otJZAyJmVHct/k8Ylm6iNfPL8m"
"vBaX0p1ordnabEZqWVjotfAjEQNDk6RkZg9sDEzQn7JCZCfpx5r4H7lyu2q7b/mi9EW8HDWB1gq9hWV0jlBL2WcLhPYM/xg+"
"OmyRdbtUmJbDyxGF+nMiyYcKGPmCBX2LQKtj/cP10eCPV9l9VpdNNZuzs+gQHSzekcbJ/SQH7Yr38VH8Mxol7FBmWpLU29IY"
"KuMfqBn9JtUCpDrBgi1vlbbyWOkPaay8Us1u7R1SPjTNWly9IX2Vsit31G7WrdZfLS3Ur1I5KVaYT0fSKjSO7MLvmUNvHGwW"
"OOO/4nvtb+Xv7D/rO+dxZSxxDnDUdPZ3+jOKeVhGWsZOxxzHLOcNRzVHFfu+tI72nvYSzvN2nLYh+U5i/bRX3oJsjbBOGMP3"
"aFv9/Q2H8FvInPDVYTVsWP1NmihZAOk9JUfJCuGm+EkYgmP1OP8dL/Jv1xawW7gj+Qc/RJfZIfYnukvbyolqYVunkCe2RdZK"
"lpxqbzla2gtoNsz6QE0QFdzD1PQocx27z/2skDlK9wQHBI8Ex2pD9InGZvMMX49aoVb8EPOa7dkPvpjWlG+rpS3V1WxyEbCT"
"dKmv+tLa1bbF2tkSq4yS8ggzSXVynJwRFkh15GxSSZqEuqC9uK2wTFovj5SvSjOlN2JRsZ+whBakaaQSbUkvkTr4BDtrtDRK"
"GgWMMMOr/2oc02toAwMj/O39NPA60Dp4NoACjfzcN8g/33/eX9Pfz7/LPzJQIdDN/8i71PvIq/na+6f6fb7FvrK+VpDB6wXL"
"6T2NXUayXkq36KqxyVzBf6ApuC9ug/fg2aQY7Uo70qm0mNBHHCqFyPelUCm/+EYYLo6Ucsqq3FzaJVYRK4klxThhidBYaCa0"
"FiYLc+HuJq1I15NhZBcJp9VobjqHfMF/4K34LN6HS+JDaACajY4CXv0FHectuMRdzMds/CNY4ho2iDVi9Vg1VpR9M/+B6h2z"
"7mw2W88WsXXsFrPwxfwHb4I+o214FqlOT9FywkqhipgkXpNmyW65PlS4awFreJXcaj91rjpWbawSNZtaDjDtVSW7ckPeJJ+Q"
"H8lz5ZbyFSlacon7xV/FIuJrYa1QWfhEe9PDgJhHYILvAzYcjt7zXHwy1MjXzbNmZ/O2Ud9I0LkeYvyp59evaCF6V/2HVl+7"
"FFS1odo+bZWWR3MGfcHcWh+oD19oH7Uw3dSi9G76Dv2+Lhn9jM9GTXOYudzMxY6z3jyBt0NnUGmcgJeTNyQDNL2KZhOmCjeF"
"/cJzIVVIFz5BZXUCetqAVOsKOj0K8hxKB9ESdCqpSsaTg2Qy0fGvuDy+jQlZgp+hTegaKoPD8E70htfgE3gldB81Rvl5GkM8"
"hl/ij/g4OFNP/op3Qh3Rc76fS+hPFIeeo1j0BMWji8BREfwPfoR34qW4J/6EB0HkOIcL4J7ER6uLTQERHRMKiUfF/NI06ZM0"
"UE6Rdfmn7JQny4XlgDRIVpQIpZscKp0SPdJguZvwinQVeymX5FhiomJ0txAmuPAEEi3cEeNJB3bfIKiqsFFYgzqbETw77sH7"
"m3HGbUZxU15AfxJYbj4jt9DcQE1Pu8AEQA+GVs/XxP3I89pX3dfWc8izxVfQ9ynjJkTWFs4Qd3bPNvfbjJ0ZroyJbuQu7C7p"
"669PRXdISTyC1TNv80pCbmWEJcaaCnk3TGksL5AXKw3UfOohiGOfxIOCHw9jHY1nxjeWD6Ww7sbrYOFAw0CJYIhe1jzDCnCF"
"tTE/mJv5e9yUWoQVgiFsFpk4TmwmCDQnrSnMEE+Jd6gPcLLf+M2MZTX4HLbTKA45aoB2XCuiT9ZT9KrGZOOm0d7MzZqxsUzh"
"VxChX+gGmkSO0q/SHMs72y3bQ8vfag+L23Y47GB4IIyF/hXSPWSibYxtrO2sLbstzXJWraO+VLyyJNcBlBgqD5BbSNuEUPqV"
"nKIvwVZa0f34CLrOh/BjLM50QcyYpY/TfIFFvl7e7p47npKemu5Ql+z43V4rfaq9rn2m470jxWHY+9gD6Rn2ac69rnoZ21yf"
"nE8dCY7Rjp7OEMc654yMyoFpbANgg799Yd5xwSn4jFLWFmodofwqrKbJQnO1ZsjQsCahf1vbK1ukW2J1MVY4KcwFO9kudMOD"
"zc6Qp8ELeAymGHGXPkILaJfN3XiesEDcKRSnPch0spdeEqNkopyXndJj0SL0IrPwc2yjOv1OP+EdbIveReun7zQT2F2WYTzT"
"OgVZ4FZwmE7NLhA5Nppe44KBWU/+FY0kPehPuoLaqIcspZWFdWKsfMyywXbEdkBdIbeVN1h+C8uIbBDVOqJ9SJp1sG1m2KjI"
"6KgGkRfCZodE2wxrrDVo/cP6wTrXGmrppNjhLKOEN0J7abTUWcT0JW/DnpqLWCLfxAexNvqXwFH/BH/dAA52CTz2j/HW8XQG"
"lLnD293bx1PD43V/gadmnuvuhxnRbu555Z3pu+BZ7lrveuh65on3fvTMz3DZ36UfcO7z3te7Glv09to37atxlX8lqXK0Jdq6"
"3XLOEms9YIsIuRA6I+x0+NDwAaHRNq5slffJR2Qmq1I+YQr6aAzQl+kB46d5wLip19RcgRrB6dpdoxl7xfbwKMRRbfwb/pPs"
"p5XFNGmjlCB2E24SCy1KI0FTC2lu3IntN44bb4x044QxXN+rXQxUDZQIJPsbBotqHfW7ekv9oD7COGpW4xfRX+QSXSh4BK9Q"
"XzwjSvI4Ja/6UPkgf5RqSM2lF1DlfJcaSYOkgdI0eZTUWxorFZEHKK3VDLWL5ara1nLU8tmaIyQyJNVW0NbaehckftzaDk7K"
"1dNKXXmrVFBKEqPEAO1FC5ICOMh/5cO4G07iN8eZd40YyIHVjdzGMWO1EW6YelP9qH5Hf6/XNsrqY7SZmk+rb9zTYoJ3/N99"
"Df2T/C8Dq7SPgak+X0bLjK/uEr6R/k3+Hd5ZGcmO2/aSzrYZZXzegKwtDxjeBb6+2p/sJCkF3nxTXCMa4i35rGVZSPbQ1qGX"
"bLr1hWWv2lTNrf6i3lZ+in+SQzzNzM5izR1mE7OF/jTo9yf6EwOdtK96DVMzC7IE8xY7juKJKnaUvksDZIu8Vm6jvFA2qyfU"
"HGpRpadUV4wWooU8wgTaj1RBSWyYed/IYVY0b5g5TMmw6o81ppc3j7E/+GP+FOJ1EFUnOWgd4bQ4X5okHYT4c0s8DDX1KDmn"
"PA+qyGNCUaGTUFvYQJNJKGSFEXgjqoVa8OmQOynqie6iUtiNp5Ph9BJUtyOl7vJ+ZaQ6Sa1guWIRrUnWN9bxttfW65aClli1"
"r+pRHsjT5ZJSM/EFrUQfkglkAN6B7vETPJ2/52/5CI54bV6GV0P90Xv0GvVG1RFFQV4V18VLcF4UZAvZUraJSXwcY6YXrOpc"
"cF+wmjZYuwvect230fvYG+W96W3stXuXeZO9g72jfdhnerODx2ieGH/LwIGA7o13L3ZJbp/ns39y8F6wTGCd76gvEDxi1kHh"
"uBtksHD8lZhCWfmJMkph8g95qDJFWaeUV+Yrk2SvtFE8S+/i+ugyyG8f97L3RiftRKBioFPwnvZOf6pP174G04O59QqmzNei"
"vWgMesGroI54EJ0LFv6LcJdmh5PXhXiiCDPpFvIJsmsRXodnwNkqsjNmS3Oi4dSHAzZYaNSGDH/Q/N2MMFeb81lfNA5bSDw+"
"iC/j4WQlHSLUESPF08JZ+gQQcZTAaFPhDBVpA9IJf4As3BNvwX6UyntwkS/lW/gD/pKf5PlQTkTQRJDtZfwDHyOlSVFSjvwk"
"u2gh+ZY63BqrtBRvCLklXW1giwm5Z30ia/QsLS19scyw7lW3i4/xKvQNmbSJ1EJsSbsjJ3sC2cxEGvbgDngef8Pqs3GsJMqJ"
"LbQVOYiK8aGsNHei4kSmS/BBHg8WWYZ15dPRH3yXWc7YqjXWFL2+cc7w6D+D4wO5/B/8CwNbg9eDawNvfMu8y71nfZ/8yYGQ"
"YJVAuH+dL+hL9r8OSNru4NZAG8Cgyb7K/iHglaavju+NZ61nmfeQ/0bgQ4AGogNnA5u0O0ZDbgHc4uQd0QXcX2gnXZRySylC"
"vBAQfpME+bX0u5hBF5I5+Djglt7oGy/MT5qTjIL6ccizTfUJxndjmPGrflOL0IdB/OsAqKY72sdX8UkgwdV0hDBAuEcdZB/J"
"DnKaKPygfahIOuI/0G20D61Cy3gO/sP8Zo40k81TZln203SZO808LIZl4624xsuiQ+gn6g54NgeZSl4Dzi1L29OF9A96gQ6m"
"PWkZWp6eIevJIjKQNCTpeAh+DzY1DC1Dh9F2NBhFoIN8B//ER6FkZMWv0Snwyc+oLt6N47EXNFyG5APMlRcyZRM8F88B9DUN"
"VUWf+Tn0lGymi2htfBAsoh3ZQI+LE0VVmItr4MWkgrhPOif+IswhGjbwG1oBcvAq6sFVcB2chseSAnQJ4fgFmouao79RNL6B"
"r+LO+BmvxPvBiS5Bj4pdfAnbY3KzIH/Bz4NdTDaHGSWMQ8ZaswfrYYYZT7VnWjc9ylhotDEW6wO1O5AZLmiadlcrpcnatuCc"
"YKvg8uCtYB1tp7Zfq6sVB0RcWOurubRhejP9kvY4GAjm1dZoKcHdwa+B9oE431jfLX/rYDDYJZgW6BS4F6ivXTR6A25dzXez"
"KrwsjhS+iYa0X7oIOL28RJQNUJ8fl03psLhfKCu8A83NJs/xO/SUt2A1TQm8DvKfqZjTjRCjqNHT/MEOg9ev5PNA3m9REO8E"
"NFBHWAzzFtDFQmnxunhPdAoXaCnamJ6morALPH0XyLcxmoI+ITvUAU4+lecDC1vMG6NnaD9YksZrQ3XQAktkA7lOupAK5DeC"
"6T+0uTBCoLBCY1qf7qC1AIlFCvkA2dwFNC2CHTzG1fFKpPNlwM02fpd7uJcfBPlrrBTE0rloCrlLCN2FV+MreAf9IVSVTHGU"
"aNI0ekEoJFWXR0itxLpCPL0D9b5T2C10oJOIieuQaWQumUU+YahV0W9oLWIoDb1EfVB+tIB35Im8NVqCBiEVjeBNAft/56Mh"
"lqSDN3xjn1kDfhsqkY38LcvDyrOS7G+WDFSUXTTzmQ3NoeYnM948aYabF4zZxjJjr+ExrhlrjMZGFcOlNzfyGtkhC/cGSRcy"
"DF3XX+g39WqGYUjmPWORscAoYqwyJhjbjQ+Aek9BpXkCaiMXRH9Zp8ZnvYeeDlXSDn2LKfEcqCiKBT4eo5NkidBa7CEmCYOE"
"CUJZMZdUVfoilhJLCoOpi1yCk4aSebgfeKmfLWBlWS3WkxVgDyDmTjefm9HMyabwhXw7j+V+QNeHcQhpCxIqC1X3JFIeasZH"
"oOupJCdxQUSeDh6ajNvh2agMYmDzSZDVXvO2/ALgjHdmPkZ5V96Mn4PqcxeL5PGQFRag0Wge1K+FoaJ9g29BDLgD9ZJK+pGl"
"pDv47Rt8AE/HM3B9XBG/RVsgOiXye4ATN4D91AWPSmSPGeEvWSrLz/vw8lCZKVCpreU/uZ1f5M/5bIjLR8FXn5G+OD9+hr/R"
"cDFB/CKMoflJFdpPWC01kz4Jg+hkUonE0F1CmnCbRpGf6ADEj1nATUWcG5Xgw1lr9hL2ymCbWV7GjarGF+Oc2Zc1YGvNw0Zl"
"o7MRajrNtuyC6TdqGaVAQwUg0iWZVcwWRub/kSpnjDBaGYn6bEBWrfWaemn9ofZKOwTe30JbrH3VtukrdFnvpaUHPwYfapHG"
"buO23l+bERS1QfpOI8FoYVzRNgUnB4fq1c3b5gTDDk8tgh20WP2AEWkc0ioHjwRqB84ECgdzabX0aVBHm8FhwU3BNvpHcw/4"
"wmIWYdYyv7OLWBDaCXPJSbCNENyZVhIThLx0HqrH/+RV8ESSg0xGvVl1UzZfQ4Um8pHsF7OYEdSnGtPMOnD2juwzVO5fzSfs"
"B98OmX0haohKodVoEG5F/oQa+ig2UDbcEGrwAmQcXgQV8Vq+HuJFgJ/mRfmvLNlMNXuy2+wHm8/c5iPTY66ASroxj+ClOGdN"
"wFIqo2gUwKWon06k+0k0vSdYAH80VprLJQBhF5WvKbvUi8ouaZyYU2wAttxYGiEuojvxLaizk7CJo/FRHsZGm6/MBawo38QO"
"md+N341NRjtTZArbbJrGTyPJKGHOMLeaG8w+ZoaxzmhvdDfyg34GgCY362+1G1qaVhF0lK7N1eppWCukjdUua7u07oCE/9Yu"
"aY+04lB7RulYb6iv1f36Ez1WZxDF8xtljUHGTqODEWF81csY0fB8yzhvtATNn9bP68eMp8ZRw69P1wfoY2BORV3TdmvttSda"
"DtD/St2mH9WG6E0MK1TFPVgFVpltZFd5PDqJd+D7+DTW8ABikmq0CN1EOpM+pAy5jffjXTgP3gY5qAh6D9k0yA6yADvPtrAB"
"rB3ryiaxdNYKUGVR3gAiZHPIyHlxBdwCb8Yy+YMECKI7yQVyk1Sko2lniml7cgnk2BM8OTeJxhPAj1uDd37OnIcW8t58HQ/y"
"pqgGikQPOEJ7kIBDwC/forKQM7dDbGgC8fsFROcA3g94oL+gizHSXvEBDRO4sEQ+qvRRo5RYQFfHxXrKR7WKmiAFhexCTaG/"
"9Ctg+TNCfTIMFyHT6Sc6nwZxKJoBEaQmPoXjkIx+5QIgiWdZ/z9tOrcD6nrKPjAXa80L8sesGgsCpkgzK7LZLJrNM+1GqpHd"
"3GdeNqeapc32Rro+3GgMz83NP4yuoPPFRnmwv1um1exnLDGY8c58b640J5oPjUuGag4yR5uFoAYZZa6A3wwjxGxh7jYlFsZe"
"mH+aE0wCWPU31oodBQ4UvoddAzk35GtAxicgwiWznFAfVASMUZUfYe1ZDXYKeJ3F3piNzApmN/O8mWJeMTuayGxjPgZLvccO"
"sBOsMh8Lc74AbmwNES832Uzy0xi6mt6nNkDN+4UzQpywWTgEGNovHBGWC4uEacIGYYXQWsgvvIEs5qTRQlXhDh0A6GkT/Uwj"
"hDDhHO1LR9A9lIKMswkfICcvAmT1lHrobcjZeagVcFs8OUQWkpLkOF6J/8Br8C84ATDUTPQraomaoCoomZ+BU+3kY/gQPpoP"
"4Pn5a+ZmYZBn4yD6XmYzWQfWh/0JVrcSbG4Q6wTeHcdk7mGxEOcNFs3nQBwewpvwipBHdJ4TneLt+Ax+E3Lpab6cl+aJIL/T"
"7B82mjnN2SCjbmZuwBxTjOZGTfCmTB/abQwxphnfjJVQHeaFODLf3GN2gsxaAmTdgSUCqlzDMmP/FPan2d8caNrYNjaDOSBa"
"5THXmB3YWeBvBLsPntWTPWCD4BTPYN5A0GIhfoyf4934PqaZ7VhuyC4ePoEfg3eH2G98EFqBHMD9L5BNjqJeeBXuiL+gE6g5"
"5KjF5CHIrT5g0t+JQAfR6bQLIJVcgFcH0aX0OX0GuptBp9FdoEFF8NFXUKXsAVzzmJo0lxAuIEDK72gcdVMXzSOUFpLpXnqc"
"vqcBKgspNJ4mgo7O0GP0Mj0H1610Dm1NI6lB/iEXyRrItDlJCkSFvoCCnOgaZNHeKDs6C3KtDNXObfYXG8PysX/MhWZb8zez"
"u1nHDBiPjT+NbXANMweDV3SGqjjaXG5WAinMYb8wysLheoF9Z59YP3bHRCChC2wR42Cru83rUEHbANEkmTpg+UKAbxLN/FDn"
"bGI3IDdeYW1YX9DgHL4L9D6DTQdrqMj/gHqrFlSqMvjqKf6RL+CxLDfkkO6sC8g0lV0xx5onzPqgncfA7XKzgfnSfAkYys2G"
"slxgSRfZYO7ml3l2/gkqqFjI6REoFOLcfrivgOYDTUTd0Ep0ErBlFTwI98R9cCwOJ+1ISzIOsA0nrcAb9tBVdBlIuq1wVYgR"
"GgrFhYrCDuG94BO2CNWFdFpDOCDsgTc24Ss9CXrjtKTwnY6lw0GjW0CTo0GfPWgL2px2h5o9B5UgTn4mxyBG9iZNSSo+D1G4"
"J7ZAvmuOCiGM4gCbNOGP2E6wycxqPYaVY5/MJ6YXrLQAE1kEK8JGwpt1bDf7BtF5KoxvxkuAbxREf6HJyAOy6QxW+BdKRfkQ"
"Bvm9ZHY+BCLgHrbRPGyeZD7+hhfgCeZ4sybrwXsgO7/BcrLGbDHvDRK5xW+x9Swc8Noy9JaH8f3sKqvAu/OZPIT3Y7VZL5bE"
"7oDH5mdW0Fpb8N+hrAlzmcUga5wDPxwPmf4j+MV4sKUVIPvi/A5vC7IuDRX/Rn4B4v8OyP3loHpdwHOh8+g71HgR6AO3gh4a"
"44E4HY0EnRTEx7AbqqKWuDTuhr/g/qQu8eG1gPFU0gFQZyNSk+g4P0S5O+Qc5LPGgPoXkPPQJpI2gP0WgH/NIc2BOoB/7YAZ"
"MaQaaUbGQN5aQlqRPKQErNiF1CGYnMDH8Vf8Cf+N22IRJ0EV2BgXww4UA/KrApzfBn7qgd084gSeqwO37yDuzIOYdgEs9A6c"
"sy54+C98OOS3pTyGV+HR0NOfb+Vn+QNAv1P5Zs4ArRZBtVBhVAxVg6ozFs6dAtcZaCj6A71AOXADQD/zwQZi0D9QzXaAKDoD"
"0FFtFI/64SlYwNXQd94eEdwf94J3ZaFyqYycaDzOh7cgBSE0AOJKdvwTdUZfuJN3hrwbD/VtEcTBBzqDhb8EVDoD6qVBaCfS"
"kIyj8DW0FWT+DZXANjivHR0DPrzoF1wV18YI5JAT+gfgybg3rgnv2wFPeyBLL4GePhDJNuFrED+24EOABg7B3TnQzVl4vxIw"
"xd+AfXeCL+2E380wFmo4sPIL+B6+iY/i61B7fcXpMP4ESP0baPcFfgn19k5A6/vwZbwY/4qbAw4cj3/HkwBDExyGa+HfQEIN"
"sQR6aYy742kQSfNhDjh7CnjuQOBPxSWhuh6AO8HJ7qPHyAZzauMfYGFX4akYrgZzn0DN74c5Q2F9N5xVwiNhz93AcyWwtNX4"
"FQ7CGWbiIVBP3McZ2AOn7I1H47v4LT6Jx+GyIIVecP4FuD2OAC1UwIPh7S+YAo/FYeQ6OPkUsNhieAR+CLXINuCtLm4C1xP4"
"COyVDeuoAIxsB/uXwCnoOHoEVpADZP8CUNAZdAc9QXHIA56bBHyGg9YzUC7cFFaoB3G6GlBFeOoBZy8Ke/qRgq1YQ270APR4"
"Al1Cr9EhqHJmgkedh7sD4Ge7AGUdQgkoACgsDr1D6VmaL4Jz47xYhhq1DkgtJ1h9KPSaIJGnMOYruoXuocuQxcehsWAzM9Bc"
"NAZNh+sk8M3xsGLm10br0Z/oAljwKRj7Cj2E2BMLfFyD5+3ob5h9Aew4HnoWoo5gZ3HAwQPwoxhA6KPA0r6g56g9VHkR8HQQ"
"bHUVaoUk8KwY4Pwi6oWyoVyoDRoIz0MAL3ohducCPFEc+fg/PJVbsr7LskFzAJ7rhPqiquAPAioP/tMcVYZxAYhxGbw8zHXy"
"q4BAbvJX/BvEtDi+B/LzTOi5Dv47iQ/iXSDa9uUT+TTeizeCyDkK3o4E/LUavPYoP86vwHUt5KVrsO9tvgXmDOBzwdO381kw"
"PhQwc33o6Qy45ht7zt6yCMApeXgG5P9tkJdUHgkY4x92GCiOBSGuMvYOMtMz5oPa2M88gPzesIcQo++xF5AHr8P1NvzugAz4"
"AOgji4fK8SWsYGcCz8utXIV6pwjE+/K8NuxUCCJsLdi9HUScXLBvDog+9Xh96E/KmpON5wSk9Ra4OQVR28841Lqn2XZATKks"
"FNZKhVG3YQcJcFQYd7LPEN3DYJ98gJZdgNMMVpiXhCedpQDHCE4j8gCMT8j6gkmHmvYDcPg16ykF8lECy/wXZD9gqcx/S9Zh"
"fjjwLMFqHDCdwA0WAs0LY3U4T6YMMr+I4jDHBOlk7v+NMajJ3bDmOzj9VxhpByl9YjdBMq9h/ZfsFrsE0jnFjgBCPMk2sFWA"
"HLextWw5mwd36yB7zQe0PIdNAyy/AbLRQshQw9kU6N/H/mDLoKoZAW0zzN4I2Ws0GwLzFrAlMHoYIJe+bCqMmQH9fWHWDMjA"
"2wCH7oCxu2DXJ+wV6O4+aOwVcPMJatFPwNFd6HsE7Snc3QVOrwCufQTvnwLPz2HOY/ae/QSJp8GJ3kCzg4R0kIcGJwsyK+gw"
"G2iVQZ+Y9YWZyhFPgLVTs74ZSwRJPIP93rMvgFlT4PoeUFYCzMWcgEw1kLKPabCeAjoUuBfeuOGdBTK2BTSX+RUagvUj4UmC"
"0UF4p4AeEcjfAfIPQnOBTtJhxwSgL6DTz7DHBzhB5lnuw1nuAd1kf4PMTwFie5aF2U6BPG7AGd8D3YOnk5D/n8Cch9B/HGqb"
"83DuNzD7HNh+LEjkJTxfh/7tgFxOgA7Psr2AGdYB0jkNY87CmL9g5EUY9Qxs9Ro7A5b5AlZ4AatfgBkXofch+ETmzifg/Y0s"
"zu5D73XofZxlH09h/AewoR8gtUzJpcPZvf/5WVrWd3SMUZCaAXWuASdnIBkdbA+BNEx4MqE/CM0DErHDDCfc+0G6Bsz8l4ws"
"+8xcReaZI3WwXpYlvUSQpi9rTS9I8SM0b9a85CzNpWRpiAK+/Q7c2WFU5peCOoxxQKOgEQY7uWCcEzTmyfKLn2BdicBFpgc4"
"4ZoEazmybMYH751gHQ6Yk8m9BnNSss6dBL0JWfvHAb2DnTNl8SVrpWSYw4CLf79htIK1RPIoiFyleTGoggtlURmIKqXA54tA"
"1V8BYkwT3hjiSQ2oResDzmzD2/JWvDn8doZ4GcMH8t+zrkP5CIiM0wHrTAGaDL9zAN8tgjYPouVivgqq240QRxfyJXC/EbDR"
"Zoihq/kGiMKxgJNO8YOAy49CXL4FNdRtoJtQDb/g7wD3v4Xfp0AvIYa/4E/+o8fQnkM9/5rH8w8Q239Cre+CPGEHvOOG2O/M"
"ujqA0uDXBe8/8s8w7itP4Mk8Ba6JWTM8PIn/gJYIfZlfun6AFd/Avp8z/44aKB5mpMCoVOj5B3Z9Du/igJeHwN8d4OE1PL3L"
"4vMb7O3nBmA9g5tQqQazyOAUWSFXyUgEnGaDKisfygF1iIwsKArlRjmhT0SZowXokSEneoBnDZ4yv0PWss4QhDU1HgB+0+HZ"
"CyfL/NtqB/xqWd/oemBfjWd+s4wgQ2a+C3IF8imGJy/wYoV9MneKhL0iUQHA4uWBiqKCgDCrZH2pXAdQaUVUKeu75frQakNG"
"rQitMqoAWLMkUGkYWzyLMr91LoTyojyQo3PBunlh5QhYNxKFoXDYQYLz/PvVNAb+vf/x6wPOU7Iknwb3buA+Fe7c8NYP5M3i"
"WoNz/vsHAW4lsIqUJbd/v8KWQH4yXKX/VpeyrlAI//elNs36Jf/d/89vuDMl+b+/7/73G+/MHTL34P9jv/97/ffd/3369/d/"
"AYxHlHJ2PgAA"
)
@lru_cache(maxsize=1)
def ready_trigger_pcm() -> bytes:
return gzip.decompress(base64.b64decode(READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64))

16
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2084,12 +2084,18 @@ class BaseLLMHTTPHandler:
initial_response: Union[AsyncIterator, AnthropicMessagesResponse]
if stream:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
AnthropicMessagesStreamingResponse,
anthropic_messages_stream_hidden_params,
)
completion_stream = anthropic_messages_provider_config.get_async_streaming_response_iterator(
model=model,
httpx_response=response,
request_body=request_body,
litellm_logging_obj=logging_obj,
)
stream_hidden_params = anthropic_messages_stream_hidden_params(response.headers)
if not self._has_agentic_completion_hook(logging_obj):
# No callback overrides async_should_run_agentic_loop, so the
@ -2097,7 +2103,10 @@ class BaseLLMHTTPHandler:
# and rebuilding the response from SSE at end-of-stream to call
# hooks that all return (False, {}). Stream through directly and
# skip that per-chunk + end-of-stream overhead.
return completion_stream
return AnthropicMessagesStreamingResponse(
completion_stream=completion_stream,
hidden_params=stream_hidden_params,
)
from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
AgenticAnthropicStreamingIterator,
@ -2114,7 +2123,10 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
)
return initial_response
return AnthropicMessagesStreamingResponse(
completion_stream=initial_response,
hidden_params=stream_hidden_params,
)
else:
initial_response = anthropic_messages_provider_config.transform_anthropic_messages_response(
model=model,

0
litellm/llms/gdc/__init__.py Normal file
View file

0
litellm/llms/gdc/chat/__init__.py Normal file
View file

285
litellm/llms/gdc/chat/transformation.py Normal file
View file

@ -0,0 +1,285 @@
"""
GDC Gemini chat completion transformation
"""
import json
import os
import re
import threading
from typing import Any, Final
from urllib.parse import urlsplit
import litellm
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
class GDCGeminiConfig(OpenAILikeChatConfig):
supports_vertex_params: bool = True # Tell LiteLLM utilities not to strip vertex_ params
_GDCH_CREDENTIAL_TYPE: Final[str] = "gdch_service_account"
_PATH_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-zA-Z0-9_-]+$")
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._creds_lock = threading.Lock()
self._gdch_creds_cache: dict = {}
def get_supported_openai_params(self, model: str) -> list:
return [
"vertex_project",
"vertex_location",
] + super().get_supported_openai_params(model)
def _resolve_project(self, optional_params: dict, litellm_params: dict) -> str | None:
return (
litellm_params.get("vertex_project")
or litellm_params.get("vertex_ai_project")
or getattr(litellm, "vertex_project", None)
or optional_params.get("vertex_project")
or optional_params.get("vertex_ai_project")
)
def _resolve_location(self, optional_params: dict, litellm_params: dict) -> str | None:
return (
litellm_params.get("vertex_location")
or litellm_params.get("vertex_ai_location")
or getattr(litellm, "vertex_location", None)
or optional_params.get("vertex_location")
or optional_params.get("vertex_ai_location")
)
def _effective_project(self, api_base: str, optional_params: dict, litellm_params: dict) -> str | None:
match = re.search(r"/v1/projects/([^/]+)", api_base)
if match:
return match.group(1)
return self._resolve_project(optional_params, litellm_params)
def _validate_path_id(self, value: str, field: str, model: str) -> str:
if not self._PATH_ID_PATTERN.match(value):
raise litellm.utils.AuthenticationError(
message=f"{field} must be a plain identifier of letters, digits, hyphens or underscores.",
llm_provider="gdc",
model=model,
)
return value
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
stream: bool | None = None,
) -> str:
api_base = api_base or litellm.gdc_api_base or litellm.api_base
if not api_base:
raise litellm.utils.AuthenticationError(
message="api_base/host is required for GDC Gemini. Please set it or pass it.",
llm_provider="gdc",
model=model,
)
if not api_base.startswith("http"):
api_base = f"https://{api_base}"
api_base = api_base.rstrip("/")
if "/v1/projects/" in api_base:
return api_base
project = self._resolve_project(optional_params, litellm_params)
if not project:
raise litellm.utils.AuthenticationError(
message="project is required for GDC Gemini. Please pass vertex_project.",
llm_provider="gdc",
model=model,
)
location = self._resolve_location(optional_params, litellm_params)
if not location:
raise litellm.utils.AuthenticationError(
message="location is required for GDC Gemini. Please pass vertex_location.",
llm_provider="gdc",
model=model,
)
project = self._validate_path_id(project, "vertex_project", model)
location = self._validate_path_id(location, "vertex_location", model)
return f"{api_base}/v1/projects/{project}/locations/{location}/chat/completions"
def _read_env_bool(self, val: Any, env_var: str, default: bool = True) -> bool | str:
def _parse(s: str) -> bool | str:
cleaned = s.strip().lower()
if cleaned in ("false", "0", "no", "off"):
return False
if cleaned in ("true", "1", "yes", "on"):
return True
return s
if val is not None:
if isinstance(val, str):
return _parse(val)
return val
_env_val = os.getenv(env_var)
if _env_val is None:
return default
return _parse(_env_val)
def _fetch_auth(self, gdch_creds: Any, ssl_verify: bool | str) -> None:
import requests
from google.auth.transport import requests as auth_requests
auth_session = requests.Session()
auth_session.verify = ssl_verify
auth_request = auth_requests.Request(session=auth_session)
gdch_creds.refresh(auth_request)
def _cached_fetch_token(self, creds: Any, audience: str, ssl_verify: bool | str, api_key: str | None = None) -> str:
# Key cache by both audience and credential identity to prevent cross-caller contamination
cache_key = (audience.rstrip("/"), api_key or str(id(creds)))
with self._creds_lock:
if cache_key not in self._gdch_creds_cache:
self._gdch_creds_cache[cache_key] = creds.with_gdch_audience(audience.rstrip("/"))
gdch_creds = self._gdch_creds_cache[cache_key]
if not getattr(gdch_creds, "valid", False) or not getattr(gdch_creds, "token", None):
self._fetch_auth(gdch_creds, ssl_verify)
token = gdch_creds.token
return token
def _load_creds_from_key(self, api_key: str) -> tuple[Any, bool]:
import google.auth
try:
json_obj = json.loads(api_key)
except json.JSONDecodeError:
return None, False
if not isinstance(json_obj, dict) or json_obj.get("type") != self._GDCH_CREDENTIAL_TYPE:
raise ValueError(
"GDC only accepts a GDCH service account credential as a JSON api_key "
'(expected "type": "gdch_service_account"). Other Google credential types are '
"rejected so their token or external-account endpoints cannot drive server-side requests."
)
creds, _ = google.auth.load_credentials_from_dict(json_obj)
return creds, True
def validate_environment(
self,
headers: dict,
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
import google.auth.exceptions
api_base = api_base or litellm.gdc_api_base or litellm.api_base
if not api_base:
raise litellm.utils.AuthenticationError(
message="api_base/host is required for GDC Gemini. Please set it or pass it.",
llm_provider="gdc",
model=model,
)
if not api_key:
raise litellm.utils.AuthenticationError(
message="api_key is required for GDC Gemini. Please pass your service account string or token as the api_key.",
llm_provider="gdc",
model=model,
)
project = self._effective_project(api_base, optional_params, litellm_params)
if not project:
raise litellm.utils.AuthenticationError(
message="project is required for GDC Gemini. Please pass vertex_project.",
llm_provider="gdc",
model=model,
)
project = self._validate_path_id(project, "vertex_project", model)
_audience_parts = urlsplit(api_base if api_base.startswith("http") else f"https://{api_base}")
audience = f"{_audience_parts.scheme}://{_audience_parts.netloc}"
try:
creds, is_service_account = self._load_creds_from_key(api_key)
except (
google.auth.exceptions.GoogleAuthError,
ValueError,
TypeError,
KeyError,
AttributeError,
) as e:
raise litellm.utils.AuthenticationError(
message=f"Failed to load service account credentials from api_key: {str(e)}",
llm_provider="gdc",
model=model,
) from e
if creds is not None:
ssl_verify = self._read_env_bool(litellm_params.get("ssl_verify"), "SSL_VERIFY", default=True)
if self._read_env_bool(litellm_params.get("gdc_token_caching"), "GDC_TOKEN_CACHING", default=False):
token = self._cached_fetch_token(creds, audience, ssl_verify, api_key)
else:
gdch_creds = creds.with_gdch_audience(audience)
self._fetch_auth(gdch_creds, ssl_verify)
token = gdch_creds.token
headers["Authorization"] = f"Bearer {token}"
if "Authorization" not in headers and not is_service_account:
headers["Authorization"] = f"Bearer {api_key}"
# Standardize necessary metadata headers
if "content-type" not in headers and "Content-Type" not in headers:
headers["Content-Type"] = "application/json"
stale_quota_headers = tuple(h for h in headers if h.lower() == "x-goog-user-project")
for stale in stale_quota_headers:
headers.pop(stale, None)
headers["x-goog-user-project"] = f"projects/{project}"
return headers
def transform_request(
self,
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transforms the request to the GDC provider
"""
if model.startswith("gdc/"):
model = model.split("/", 1)[1]
data = super().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
# Remove extra params used for routing/auth
for param in [
"vertex_project",
"vertex_ai_project",
"vertex_location",
"vertex_ai_location",
"ssl_verify",
"gdc_token_caching",
]:
data.pop(param, None)
return data

0
litellm/llms/github_copilot/messages/__init__.py Normal file
View file

118
litellm/llms/github_copilot/messages/transformation.py Normal file
View file

@ -0,0 +1,118 @@
from typing import Any, Optional
from litellm.exceptions import AuthenticationError
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from ..authenticator import Authenticator
from ..common_utils import (
DEFAULT_GITHUB_COPILOT_API_BASE,
GetAPIKeyError,
get_copilot_default_headers,
)
_MESSAGES_PROXY_API_VERSION = "2026-06-01"
class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
GitHub Copilot implementation of Anthropic messages API.
Routes requests to Copilot's /v1/messages endpoint with appropriate authentication and headers.
"""
def __init__(self) -> None:
super().__init__()
self.authenticator = Authenticator()
def handles_web_search_natively(self) -> bool:
"""
Copilot's /v1/messages endpoint does not execute ``web_search`` tools, so
the interception handler must short-circuit web-search-only requests
instead of routing them here.
"""
return False
def should_filter_anthropic_beta_headers(self) -> bool:
"""
Copilot's /v1/messages is a native Anthropic Messages passthrough, so
``anthropic-beta`` values injected by ``_update_headers_with_anthropic_beta``
(context_management, structured outputs, ...) must reach the upstream
verbatim. The default provider-scoped filter would drop them because
github_copilot has no entry in ``anthropic_beta_headers_config.json``.
"""
return False
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> tuple[dict, Optional[str]]:
"""
Validate environment for GitHub Copilot and add Copilot-specific headers.
The caller-supplied ``api_base`` is intentionally ignored. Routing this
request anywhere other than the authenticated Copilot endpoint would
leak the Copilot bearer token to a caller-controlled URL.
"""
# Always use the Copilot endpoint resolved from the authenticated
# session, never the caller-supplied api_base. rstrip so a
# tenant-specific base with a trailing slash does not yield a
# double-slash URL once "/v1/messages" is appended downstream.
dynamic_api_base = (self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE).rstrip("/")
try:
dynamic_api_key = self.authenticator.get_api_key()
except GetAPIKeyError as e:
raise AuthenticationError(
model=model,
llm_provider="github_copilot",
message=str(e),
)
# Merge Copilot headers with provided headers
copilot_headers = get_copilot_default_headers(dynamic_api_key)
for key, value in copilot_headers.items():
if key not in headers:
headers[key] = value
headers["openai-intent"] = "messages-proxy"
headers["x-interaction-type"] = "messages-proxy"
headers["x-github-api-version"] = _MESSAGES_PROXY_API_VERSION
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
headers = self._update_headers_with_anthropic_beta(
headers, optional_params, custom_llm_provider="github_copilot"
)
return headers, dynamic_api_base
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Return the complete URL for GitHub Copilot /v1/messages endpoint.
``api_base`` here is the value already resolved by
``validate_anthropic_messages_environment`` (the authenticated Copilot
host), not the raw caller-supplied base — that one is discarded there to
avoid leaking the Copilot bearer token to a caller-controlled URL. We
reuse it to avoid a second authenticator read, falling back to a fresh
resolution only if it was not provided.
"""
resolved = (api_base or self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE).rstrip("/")
if not resolved.endswith("/v1/messages"):
resolved = f"{resolved}/v1/messages"
return resolved

2
litellm/llms/oci/chat/cohere.py
View file

@ -110,7 +110,7 @@ def adapt_messages_to_cohere_standard(
tool_calls: Optional[List[CohereToolCall]] = None
if role == "assistant" and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item]
tool_calls = []
for tc in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item]
for tc in msg["tool_calls"]: # pyright: ignore[reportOptionalIterable] # truthiness check above rules out None
raw_arguments: Any = tc.get("function", {}).get("arguments", {})
if isinstance(raw_arguments, str):
try:

6
litellm/llms/ollama/chat/transformation.py
View file

@ -363,7 +363,11 @@ class OllamaChatConfig(BaseConfig):
response_json_message["reasoning_content"] = reasoning_content
response_json_message["content"] = content
if request_data.get("format", "") == "json" and litellm_params.get("function_name") is not None:
if (
request_data.get("format", "") == "json"
and litellm_params.get("function_name") is not None
and response_json_message is not None
):
function_call = json.loads(response_json_message["content"])
message = litellm.Message(
content=None,

23
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -2,6 +2,7 @@
Support for gpt model family
"""
import json
from typing import (
TYPE_CHECKING,
Any,
@ -782,8 +783,30 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
delta["reasoning_content"] = delta.pop("reasoning")
return choices
@staticmethod
def _extract_error_from_chunk(chunk: dict) -> Optional[tuple[str, int]]:
"""OpenAI-compatible backends (vLLM, sglang) can return an HTTP 200
stream whose body carries an error payload, e.g.
``data: {"error": {"message": "...", "code": 400}}``."""
error = chunk.get("error")
if not error:
return None
if not isinstance(error, dict):
return str(error), 500
message = error.get("message")
code = error.get("code")
status_code = code if isinstance(code, int) and 400 <= code < 600 else 500
return (message if isinstance(message, str) else json.dumps(error)), status_code
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
try:
error_details = self._extract_error_from_chunk(chunk)
if error_details is not None:
error_message, error_status_code = error_details
raise OpenAIError(
status_code=error_status_code,
message=error_message,
)
choices = chunk.get("choices", [])
choices = self._map_reasoning_to_reasoning_content(choices)

2
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -716,7 +716,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
elif isinstance(content, list) and content_idx_optional is not None:
# Replace specific text item in list content
choice.message.content[content_idx_optional]["text"] = guardrail_response # type: ignore
content[content_idx_optional]["text"] = guardrail_response
async def _apply_guardrail_responses_to_output_tool_calls(
self,

15
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -45,6 +45,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
ResponsesAPIStreamEvents,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
@ -196,12 +197,13 @@ class OpenAIResponsesHandler(BaseTranslation):
return data
def extract_request_tool_names(self, data: dict) -> List[str]:
"""Extract tool names from Responses API request (tools[].name for function, tools[].server_label for mcp)."""
"""Extract tool names from Responses API request (tools[].name for function
and custom, tools[].server_label for mcp)."""
names: List[str] = []
for tool in data.get("tools") or []:
if not isinstance(tool, dict):
continue
if tool.get("type") == "function" and tool.get("name"):
if tool.get("type") in ("function", "custom") and tool.get("name"):
names.append(str(tool["name"]))
elif tool.get("type") == "mcp" and tool.get("server_label"):
names.append(str(tool["server_label"]))
@ -586,7 +588,14 @@ class OpenAIResponsesHandler(BaseTranslation):
"""
Check if the streaming has ended.
"""
return all(response.choices[0].finish_reason is not None for response in responses_so_far)
if not responses_so_far:
return False
terminal_types = {
ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
}
return responses_so_far[-1].get("type") in terminal_types
def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str:
"""

2
litellm/llms/sap/chat/transformation.py
View file

@ -155,7 +155,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
def headers(self) -> Dict[str, str]:
if self.token_creator is None:
self.run_env_setup()
access_token = self.token_creator() # type: ignore
access_token = self.token_creator() # pyright: ignore[reportOptionalCall] # run_env_setup set it or raised
return {
"Authorization": access_token,
"AI-Resource-Group": self.resource_group,

0
litellm/llms/tencent/__init__.py Normal file
View file

0
litellm/llms/tencent/chat/__init__.py Normal file
View file

68
litellm/llms/tencent/chat/transformation.py Normal file
View file

@ -0,0 +1,68 @@
"""
Translates from OpenAI's `/v1/chat/completions` to Tencent TokenHub's
OpenAI-compatible endpoint.
"""
from typing import Optional
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class TencentChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
params = super().get_supported_openai_params(model)
if supports_reasoning(model, custom_llm_provider="tencent"):
params.extend(["thinking", "reasoning_effort"])
return params
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
thinking_value = optional_params.pop("thinking", None)
reasoning_effort = optional_params.pop("reasoning_effort", None)
if thinking_value is not None:
if isinstance(thinking_value, dict):
optional_params["thinking"] = thinking_value
elif reasoning_effort is not None and reasoning_effort != "none":
optional_params["thinking"] = {"type": "enabled"}
return optional_params
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> tuple[Optional[str], Optional[str]]:
api_base = api_base or get_secret_str("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1"
dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY")
return api_base, dynamic_api_key
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if not api_base:
api_base = "https://tokenhub-intl.tencentcloudmaas.com/v1"
api_base = api_base.rstrip("/")
if api_base.endswith("/chat/completions"):
return api_base
if not api_base.endswith("/v1"):
api_base = f"{api_base}/v1"
return f"{api_base}/chat/completions"

6
litellm/llms/tencent/cost_calculator.py Normal file
View file

@ -0,0 +1,6 @@
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")

0
litellm/llms/tencent/messages/__init__.py Normal file
View file

85
litellm/llms/tencent/messages/transformation.py Normal file
View file

@ -0,0 +1,85 @@
"""
Tencent Anthropic-compatible messages transformation config.
Tencent TokenHub exposes an Anthropic-compatible Messages API endpoint
alongside its standard OpenAI-compatible chat completions endpoint.
"""
from typing import Any, Optional
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
class TencentAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
Tencent TokenHub exposes an Anthropic-compatible Messages API.
Unlike the chat completions endpoint (which uses /v1), the Anthropic
endpoint may use a different base URL. Configure via
TENCENT_ANTHROPIC_API_BASE or TENCENT_API_BASE.
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "tencent"
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("TENCENT_API_KEY") or litellm.api_key
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> str:
return (
api_base
or get_secret_str("TENCENT_ANTHROPIC_API_BASE")
or get_secret_str("TENCENT_API_BASE")
or "https://tokenhub-intl.tencentcloudmaas.com"
)
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: list[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> tuple[dict, Optional[str]]:
return super().validate_anthropic_messages_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=self.get_api_key(api_key=api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
base_url = self.get_api_base(api_base=api_base).rstrip("/")
if base_url.endswith("/v1/messages"):
return base_url
if base_url.endswith("/v1/chat/completions"):
base_url = base_url[: -len("/v1/chat/completions")]
elif base_url.endswith("/v1"):
base_url = base_url[: -len("/v1")]
return f"{base_url}/v1/messages"

Some files were not shown because too many files have changed in this diff Show more