scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 175

Merge remote-tracking branch 'upstream/main' into fix-44242-ws-inject-response-id

Browse source 
# Conflicts:
#	litellm/responses/streaming_iterator.py
This commit is contained in:
Darsh Joshi 2026-10-19 12:01:52 -04:00
commit a1fa04e50a
1312 changed files with 128104 additions and 12491 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.github/e2e-stack/assert_tests_ran.py vendored
View file

@ -1,69 +0,0 @@
import os
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Final
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
from coverage_registry.management_cases import MANAGEMENT_CASES
def main() -> int:
selected: Final = tuple(sys.argv[2:])
try:
report: Final = ET.parse(Path(sys.argv[1])).getroot()
except (ET.ParseError, OSError):
_ = sys.stdout.write("::error::could not read the test execution report\n")
return 1
cases: Final = tuple(report.iter("testcase"))
expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
passed: Final = frozenset(
case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
)
missing: Final = tuple(path for path in selected if path not in passed)
required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
passed_nodes: Final = frozenset(
prop.get("value")
for case in cases
if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
for prop in case.findall("./properties/property")
if prop.get("name") == "management_node"
)
missing_nodes: Final = required_nodes - passed_nodes
for node in sorted(missing_nodes):
_ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
for path in selected:
collected: Final = sum(case.get("file") == path for case in cases)
skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
_ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
for case in cases:
if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
continue
outcome = "skipped" if case.find("skipped") is not None else "failed"
_ = sys.stdout.write(f" {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
for prop in case.findall("./properties/property"):
name = prop.get("name", "")
value = prop.get("value", "")
if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
r"[A-Za-z0-9_.:<>-]{1,240}", value
):
_ = sys.stdout.write(f" {name}: {value}\n")
if expected_count is not None and (
len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
):
_ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
return 1
if (
selected
and not missing
and not missing_nodes
and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
):
return 0
_ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
return 1
if __name__ == "__main__":
sys.exit(main())

58
.github/scripts/assert_ci_coverage.py vendored
View file

@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
return frozenset(ast.literal_eval(values))
def _owns(workflow: str, relative_path: str) -> bool:
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
return owned_root is not None and relative_path.startswith(owned_root)
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
return frozenset[str]().union(
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
)
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
)
!= 1
)
gha_tokens: Final = _invoked_test_tokens(
scalar
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
workflow_tokens: Final = MappingProxyType(
{
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
for path in (repo_root / ".github/workflows").glob("*.y*ml")
}
)
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
circle_owned: Final = (
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
for path in sorted(circle_owned)
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
+ tuple(
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
for path in sorted(github_files)
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
)
)
browser_commands: Final = tuple(
scalar.value
@ -688,13 +710,9 @@ def main() -> int:
scalars = _all_scalars()
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
test_findings = _uncovered_tests(
allowlist,
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@ -711,13 +729,25 @@ def main() -> int:
test_findings,
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if ownership_findings:
browser_remedy = (
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
"select them and the default Playwright config must exclude them."
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
else ""
)
_report(
"integration contracts with the wrong CI owner",
ownership_findings,
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
)
if dockerfile_findings:
_report(
"Dockerfiles that no CI job builds",
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if stale_findings or test_findings or dockerfile_findings:
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
return 1
_write(

5
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -24,12 +24,9 @@ jobs:
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \

7
.github/workflows/sync-together-ai-models.yml vendored
View file

@ -41,10 +41,6 @@ jobs:
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
@ -57,8 +53,7 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
litellm/model_prices_and_context_window_backup.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"

4
.github/workflows/test-linting.yml vendored
View file

@ -539,8 +539,8 @@ jobs:
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_idp_stack
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:

180
.github/workflows/test-mcp-oauth-e2e.yml vendored
View file

@ -1,180 +0,0 @@
name: MCP OAuth happy path
on:
pull_request:
paths:
- '.github/workflows/test-mcp-oauth-e2e.yml'
- '.github/e2e-stack/**'
- 'tests/e2e/*.py'
- 'tests/e2e/pytest.ini'
- 'tests/e2e/idp_realm.json'
- 'tests/e2e/mcp/**'
- 'litellm/experimental_mcp_client/**'
- 'litellm/proxy/_experimental/mcp_server/**'
- 'litellm/proxy/auth/**'
- 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
- 'litellm/proxy/_types.py'
- 'litellm/types/mcp_server/mcp_server_manager.py'
- 'litellm/proxy/management_endpoints/*sso*.py'
- 'litellm/proxy/management_endpoints/sso/**'
- 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
- 'litellm/proxy/proxy_server.py'
- 'litellm/proxy/schema.prisma'
- 'ui/litellm-dashboard/src/app/connect/**'
- 'ui/litellm-dashboard/src/app/mcp/oauth/**'
- 'pyproject.toml'
- 'uv.lock'
workflow_dispatch:
permissions: {}
concurrency:
group: mcp-oauth-${{ github.ref }}
cancel-in-progress: true
jobs:
oauth:
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
environment: e2e-changed
timeout-minutes: 45
permissions:
contents: read
id-token: write
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U litellm"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_HOST: 127.0.0.1
DATABASE_PORT: '5432'
DATABASE_USER: litellm
DATABASE_PASSWORD: dbpassword9090
DATABASE_NAME: litellm
DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
E2E_KEYCLOAK_URL: http://127.0.0.1:8081
E2E_KEYCLOAK_ADMIN_USER: admin
E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
E2E_FIXTURE_MODE: live
E2E_PROVIDER_CACHE: '0'
E2E_MCP_OAUTH_LIVE: '1'
E2E_REQUIRED_TEST_COUNT: '4'
steps:
- name: Checkout the tested source
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
- name: Require and materialize the upstream login
env:
STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
run: |
umask 077
python3 - <<'PY'
import base64
import json
import os
import secrets
from pathlib import Path
encoded = os.environ.get("STORAGE_STATE", "")
if not encoded:
raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
state = json.loads(base64.b64decode(encoded, validate=True))
if not isinstance(state, dict) or not state.get("cookies"):
raise SystemExit("The captured login must contain browser cookies")
directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
directory.mkdir(mode=0o700)
path = directory / "linear-state.json"
path.write_text(json.dumps(state))
with open(os.environ["GITHUB_ENV"], "a") as output:
output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
value = "sk-e2e-" + secrets.token_hex(24)
print(f"::add-mask::{value}")
output.write(f"{name}={value}\n")
PY
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: ./.github/actions/setup-uv-with-retries
with:
version: '0.10.9'
- uses: ./.github/actions/cache-cargo-build
- name: Install the frozen E2E environment
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
uv run --no-sync python scripts/prisma_generate_if_needed.py
uv run --no-sync playwright install --with-deps chromium
- name: Configure license access
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
with:
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
aws-region: us-east-1
role-session-name: mcp-oauth-${{ github.run_id }}
role-duration-seconds: 900
output-env-credentials: false
output-credentials: true
- name: Load the E2E license
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
AWS_DEFAULT_REGION: us-east-1
run: |
license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
test -n "${license}"
echo "::add-mask::${license}"
echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
- name: Build the gateway consent UI at the tested commit
run: |
cd ui/litellm-dashboard
../../scripts/with_dashboard_node.sh npm ci
../../scripts/with_dashboard_node.sh npm run build
mkdir -p ../../litellm/proxy/_experimental/out
cp -r out/. ../../litellm/proxy/_experimental/out/
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
mkdir -p "${page%.html}"
mv "${page}" "${page%.html}/index.html"
done
- name: Prepare the isolated database and IdP
run: |
umask 077
bash .github/e2e-stack/start-idp.sh
uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
- name: Run every required OAuth variant without retries
run: |
umask 077
uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
--rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
--junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
> "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
- name: Report JUnit results and reject skipped or missing cases
if: always()
run: |
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
"${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
- name: Remove private login and logs
if: always()
run: |
docker rm -f e2e-keycloak >/dev/null 2>&1 || true
rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

8
.github/workflows/test-rust.yml vendored
View file

@ -16,6 +16,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"
@ -46,6 +50,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"

2
.github/workflows/test-unit.yml vendored
View file

@ -658,6 +658,7 @@ jobs:
- shard: proxy-extras
test-path: >-
tests/unit/litellm_proxy_extras
tests/unit/db_scripts
python-version: "3.12"
workers: 2
reruns: 0
@ -696,7 +697,6 @@ jobs:
tests/unit/responses/litellm_completion_transformation
tests/unit/responses/test_additional_tools.py
tests/unit/responses/test_custom_tool_call.py
tests/unit/responses/test_dispatch.py
tests/unit/responses/test_metadata_codex_callback.py
tests/unit/responses/test_no_duplicate_spend_logs.py
tests/unit/responses/test_null_test_fix.py

4
AGENTS.md
View file

@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

1
README.md
View file

@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |

32
backend/Dockerfile
View file

@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

1
backend/routes/allowlist.py
View file

@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/admin",
"/metrics",
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}

45
ci_cd/cost_map_guard.py
View file

@ -1,11 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
"""
from __future__ import annotations
@ -19,13 +19,12 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
CostMap = dict[str, object]
@ -35,7 +34,6 @@ CostMap = dict[str, object]
class Snapshot:
cost_map: str
backup: str
schema: str
def _parse_object(text: str, path: str) -> CostMap | str:
@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
def _rendered_schema(cost_map: CostMap) -> str:
try:
return render(build_schema(cost_map))
except SystemExit as error:
return str(error)
def _evaluation_output_price_failure(
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
) -> str | None:
@ -78,29 +69,13 @@ def _file_failures(
*,
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
) -> tuple[str, ...]:
schema_text: Final = _rendered_schema(head_map)
if not schema_text.startswith("{"):
return (schema_text,)
backup_failure: Final = (
()
if head.backup == head.cost_map
else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
)
schema_failure: Final = (
()
if head.schema == schema_text
else (
f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
"run `python ci_cd/generate_model_prices_schema.py` and commit the result",
)
)
return (
*backup_failure,
*schema_failure,
*(
f"{COST_MAP_PATH} does not validate against its schema: {error}"
for error in validation_errors(head_map, json.loads(schema_text))[:20]
),
*(
failure
for key, entry in head_map.items()
@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

418
ci_cd/generate_model_prices_schema.py
View file

@ -1,418 +0,0 @@
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Final, Optional
import jsonschema
REPO_ROOT = Path(__file__).parent.parent
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
JsonSchema = dict
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
BOOLEAN: JsonSchema = {"type": "boolean"}
STRING: JsonSchema = {"type": "string"}
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
WEEKDAY_PATTERN: Final = (
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
)
EXTRA_BOOLEAN_KEYS = frozenset(
{
"gemini_native_audio",
"gemini_audio_only_live",
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)
HOURS_UTC: Final[JsonSchema] = {
"description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
"oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
}
OFF_PEAK_WINDOW: Final[JsonSchema] = {
"type": "object",
"properties": {
"hours_utc": HOURS_UTC,
"weekdays": {
"type": "array",
"description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
"items": {
"oneOf": [
{"type": "integer", "minimum": 1, "maximum": 7},
{"type": "string", "pattern": WEEKDAY_PATTERN},
]
},
"minItems": 1,
},
},
"required": ["hours_utc"],
"additionalProperties": False,
}
OBJECT_KEYS: dict[str, JsonSchema] = {
"off_peak_pricing": {
"type": "object",
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
"properties": {
"hours_utc": HOURS_UTC,
"windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
"weekday_timezone": {
"type": "string",
"description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
},
"anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
"additionalProperties": False,
},
"search_context_cost_per_query": {
"type": "object",
"description": "USD cost per web search query, keyed by search context size.",
"properties": {
"search_context_size_low": NONNEG_NUMBER,
"search_context_size_medium": NONNEG_NUMBER,
"search_context_size_high": NONNEG_NUMBER,
},
"additionalProperties": False,
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
"additionalProperties": NONNEG_NUMBER,
},
"metadata": {
"type": "object",
"description": "Free-form notes about the entry (e.g. pricing derivation).",
},
"provider_specific_entry": {
"type": "object",
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
},
}
ARRAY_KEYS: dict[str, JsonSchema] = {
"supported_audio_formats": {
"type": "array",
"description": "Audio container formats the model can return.",
"items": {"type": "string", "enum": ["mp3", "wav"]},
},
"supported_endpoints": {
"type": "array",
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
"items": STRING,
},
"supported_modalities": {
"type": "array",
"description": "Input modalities the model accepts.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
},
"supported_output_modalities": {
"type": "array",
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
"items": STRING,
},
"tiered_pricing": {
"type": "array",
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
"items": {
"type": "object",
"properties": {
"range": {
"type": "array",
"description": "[min, max] prompt-token span this tier applies to.",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"max_results_range": {
"type": "array",
"description": "[min, max] result-count span this tier applies to (search models).",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
"input_cost_per_query": NONNEG_NUMBER,
},
"additionalProperties": False,
},
},
}
INTEGER_KEYS: dict[str, JsonSchema] = {
"max_tokens": {
**NONNEG_INTEGER,
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
},
"max_input_tokens": {
**NONNEG_INTEGER,
"description": "Maximum prompt/context tokens the model accepts.",
},
"max_output_tokens": {
**NONNEG_INTEGER,
"description": "Maximum tokens the model can generate in one response.",
},
"output_vector_size": {
**NONNEG_INTEGER,
"description": "Embedding dimension for embedding models.",
},
"prompt_cache_min_tokens": {
**NONNEG_INTEGER,
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
},
"token_count_framing": {
**NONNEG_INTEGER,
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
},
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
}
NUMBER_KEYS: dict[str, JsonSchema] = {
"regional_processing_uplift_multiplier_eu": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
},
"regional_processing_uplift_multiplier_us": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
},
"regional_endpoint_uplift_multiplier": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
},
}
COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
}
def cost_description(key: str) -> Optional[str]:
if key in COST_DESCRIPTIONS:
return COST_DESCRIPTIONS[key]
if key.endswith("_flex"):
return "Flex service-tier rate for the same-named base field."
if key.endswith("_priority"):
return "Priority service-tier rate for the same-named base field."
if "_above_" in key:
return "Rate applied once the prompt exceeds the token threshold in the field name."
return None
def cost_schema(key: str) -> JsonSchema:
description = cost_description(key)
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
return {
"litellm_provider": {
"type": "string",
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
},
"mode": {
"type": "string",
"description": "Primary API surface / task type of the model.",
"enum": list(modes),
},
"source": {
"type": "string",
"description": "URL of the provider pricing/model page this entry was taken from.",
},
"deprecation_date": {
"type": "string",
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
"format": "date",
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
},
"web_search_billing_unit": {
"type": "string",
"description": "Whether web search is billed per query or per prompt.",
"enum": ["per_query", "per_prompt"],
},
"bedrock_output_config_effort_ceiling": {
"type": "string",
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"token_count_fallback_model": {
"type": "string",
"minLength": 1,
"description": "Model that counts tokens for this one when the provider cannot count it.",
},
"audio_transcription_config": STRING,
"vertex_ai_audio_api": {
"type": "string",
"enum": ["lyria_predict", "lyria_interactions"],
},
}
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
if key in curated:
return curated[key]
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
return BOOLEAN
if "cost" in key:
return cost_schema(key)
return None
def build_schema(prices: dict) -> JsonSchema:
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
if unclassified:
raise SystemExit(
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
f"Add them to the key tables in {Path(__file__).name} and rerun it."
)
entry_properties = {key: classify(key, modes) for key in all_keys}
return {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "LiteLLM model_prices_and_context_window.json",
"description": (
"Schema for LiteLLM's model price and context window registry "
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
"ignore unknown fields rather than reject them."
),
"type": "object",
"properties": {
"sample_spec": {
"type": "object",
"description": (
"Documentation placeholder illustrating the entry shape; not a real model and not "
"schema-conformant (several values are prose)."
),
},
"fallback_generalizations": {
"type": "object",
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
"properties": {
"rules": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": STRING,
"pattern": STRING,
"description": STRING,
},
"required": ["name", "pattern"],
"additionalProperties": True,
},
}
},
"additionalProperties": False,
},
},
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
"$defs": {
"modelEntry": {
"type": "object",
"description": (
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
"are optional; boolean capability flags are simply omitted when unknown or false."
),
"required": ["litellm_provider"],
"properties": entry_properties,
"additionalProperties": True,
}
},
}
def render(schema: JsonSchema) -> str:
return json.dumps(schema, indent=2) + "\n"
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
return tuple(
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
for error in validator.iter_errors(prices)
)
def main() -> int:
check = "--check" in sys.argv[1:]
prices = json.loads(PRICES_PATH.read_text())
rendered = render(build_schema(prices))
errors = validation_errors(prices, json.loads(rendered))
if errors:
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
print("\n".join(errors[:20]))
return 1
if not check:
SCHEMA_PATH.write_text(rendered)
print(f"wrote {SCHEMA_PATH}")
return 0
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
print(
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
)
return 1
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
return 0
if __name__ == "__main__":
sys.exit(main())

201
db_scripts/strip_vector_store_internal_metadata.py Normal file
View file

@ -0,0 +1,201 @@
import argparse
import os
import sys
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from itertools import accumulate, repeat, takewhile
from typing import Final, Literal, TextIO
import httpx
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
from litellm.vector_stores.utils import is_litellm_written_metadata_key
DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
LIST_PAGE_SIZE: Final = "100"
class VectorStore(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
metadata: dict[str, JsonValue] | None = None
class VectorStorePage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[VectorStore, ...]
has_more: bool = False
class OpenAIErrorBody(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
message: str = ""
class OpenAIErrorResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
error: OpenAIErrorBody
@dataclass(frozen=True, slots=True)
class Options:
api_key: str
api_base: str
vector_store_ids: tuple[str, ...]
extra_keys: frozenset[str]
apply: bool
@dataclass(frozen=True, slots=True)
class StoreResult:
store_id: str
matched_keys: tuple[str, ...]
status: Literal["dry_run", "updated", "failed"]
error: str = ""
def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
return is_litellm_written_metadata_key(key) or key in extra_keys
def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
if page is None or not page.has_more or not page.data:
return None
return fetch_page(client, after=page.data[-1].id)
def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
pages: Final = accumulate(
repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
)
for page in takewhile(lambda page: page is not None, pages):
yield from page.data
def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
for store_id in store_ids:
yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
def describe_http_error(response: httpx.Response) -> str:
try:
message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
except ValidationError:
return f"HTTP {response.status_code}"
return f"HTTP {response.status_code}: {message}"
def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
try:
response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
except httpx.TransportError as error:
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
if response.is_success:
return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
keys: Final = matching_keys(store, options.extra_keys)
if not keys:
return None
if not options.apply:
return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
return delete_keys(client, store.id, keys)
def format_result(result: StoreResult) -> str:
key_list: Final = ", ".join(result.matched_keys)
count: Final = len(result.matched_keys)
match result.status:
case "dry_run":
return f"{result.store_id}: would delete {count} keys: {key_list}"
case "updated":
return f"{result.store_id}: deleted {count} keys: {key_list}"
case "failed":
return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
def clean_and_report(
client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
) -> Iterator[StoreResult | None]:
for store in stores:
result: Final = clean_store(client, store, options)
if result is not None:
print(format_result(result), file=out)
yield result
def run(options: Options, client: httpx.Client, out: TextIO) -> int:
stores: Final = (
retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
)
scanned: Final = tuple(clean_and_report(client, stores, options, out))
results: Final = tuple(result for result in scanned if result is not None)
updated: Final = sum(1 for result in results if result.status == "updated")
failed: Final = sum(1 for result in results if result.status == "failed")
mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
print(
f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
file=out,
)
return 1 if failed else 0
def parse_options(argv: Sequence[str] | None) -> Options:
parser: Final = argparse.ArgumentParser(
description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
)
parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
parser.add_argument("--api-base", default=DEFAULT_API_BASE)
parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
args: Final = parser.parse_args(argv)
if not args.api_key:
parser.error("--api-key or OPENAI_API_KEY is required")
return Options(
api_key=args.api_key,
api_base=args.api_base.rstrip("/"),
vector_store_ids=tuple(args.vector_store_id),
extra_keys=frozenset(args.extra_key),
apply=args.apply,
)
def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
options: Final = parse_options(argv)
with httpx.Client(
base_url=options.api_base,
headers={"Authorization": f"Bearer {options.api_key}"},
timeout=60.0,
transport=transport,
) as client:
try:
return run(options, client, sys.stdout)
except httpx.HTTPStatusError as error:
sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
return 2
except httpx.TransportError as error:
sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
return 2
except ValidationError:
sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
return 2
if __name__ == "__main__":
sys.exit(main())

71
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
@ -227,7 +228,7 @@ class CheckBatchCost:
"so it will no longer be polled"
)
async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
"""
Atomically flip batch_processed from false to true, returning whether this pod won
the row. Every pod and uvicorn worker schedules its own poller against the shared
@ -241,10 +242,16 @@ class CheckBatchCost:
"""
if not self._has_batch_processed_column:
return True
from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
try:
claimed: Final = await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": False},
data={"batch_processed": True},
data={
"batch_processed": True,
"status": response.status if response.status != "completed" else "complete",
"file_object": postgres_storable_json(response),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -253,7 +260,9 @@ class CheckBatchCost:
return False
return claimed > 0
async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
async def _release_job_claim(
self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
) -> None:
"""Give a claimed row back once billing it failed, so a later poll cycle retries it.
Safe to match on batch_processed=True: while this poller is active the retrieve
@ -265,7 +274,11 @@ class CheckBatchCost:
try:
await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": True},
data={"batch_processed": False},
data={
"batch_processed": False,
"status": original_status,
"file_object": json.dumps(original_file_object),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -564,8 +577,6 @@ class CheckBatchCost:
@staticmethod
def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object: Final = job.file_object
@ -718,8 +729,7 @@ class CheckBatchCost:
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass the deployment's router-registered pricing (litellm_params custom
@ -755,8 +765,9 @@ class CheckBatchCost:
failed_requests=output_file_result.failed_requests + error_file_failed_requests,
)
)
logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
logging_obj = LiteLLMLogging(
model=batch_result.models[0],
model=logging_model,
messages=[{"role": "user", "content": "<retrieve_batch>"}],
stream=False,
call_type="aretrieve_batch",
@ -787,7 +798,9 @@ class CheckBatchCost:
custom_llm_provider=str(llm_provider) if llm_provider else None,
)
if not await self._claim_job_for_costing(job):
original_status: Final = job.status
original_file_object: Final = job.file_object
if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
"in this window, so its cost is already being tracked there"
@ -806,7 +819,7 @@ class CheckBatchCost:
batch_completion_cost=batch_result.completion_cost,
)
except Exception:
await self._release_job_claim(job)
await self._release_job_claim(job, original_status, original_file_object)
raise
# Record batch duration (completed_at - created_at)
@ -1099,10 +1112,7 @@ class CheckBatchCost:
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
if (
response.status in PROVIDER_TERMINAL_BATCH_STATUSES
and response.output_file_id is not None
):
if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
try:
tracked = await self._track_completed_batch_cost(
job=job,
@ -1133,24 +1143,19 @@ class CheckBatchCost:
# Track this job for the final metrics summary
processed_models.append(tracked)
# mark the job as complete
try:
update_data: dict = {
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data=update_data,
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
if not self._has_batch_processed_column:
# nothing claimed the row, so it stays in the poll page and its lines wait for that retry
if not self._has_batch_processed_column:
try:
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data={
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
continue
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -416,7 +416,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
return value
def _postgres_storable_json(file_object: BaseModel) -> str:
def postgres_storable_json(file_object: BaseModel) -> str:
dumped: Final = file_object.model_dump_json()
if "\\u0000" not in dumped:
return dumped
@ -650,7 +650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
# FIX: Update status and file_object on every operation to keep state in sync
update_columns: Final = {
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"status": file_object.status,
"updated_by": user_api_key_dict.user_id,
}
@ -666,7 +666,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data={
"create": {
"unified_object_id": unified_object_id,
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": resolve_resource_owner_id(user_api_key_dict),
@ -702,7 +702,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_object_id": response_id,
"model_object_id": decrypted_response_id(response_id),
"file_purpose": "response",
"file_object": _postgres_storable_json(_without_model_output(response)),
"file_object": postgres_storable_json(_without_model_output(response)),
"status": PENDING_RESPONSE_STATUSES[0],
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.77"
version = "0.1.78"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.77"
version = "0.1.78"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

34
gateway/Dockerfile
View file

@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,23 @@
SET LOCAL lock_timeout = '10s';
CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
DECLARE
next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
BEGIN
IF TG_OP = 'UPDATE' THEN
NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
ELSE
NEW."reload_revision" := next_revision;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
CREATE TRIGGER "litellm_environment_variables_revision"
BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261016000000_background_interaction_settlement_settled_at_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_background_interaction_settlement_settled_at"
ON "LiteLLM_BackgroundInteractionSettlement" ("settled_at");

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
outcome String?
@@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
@@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
}
model LiteLLM_Lens {

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.109"
version = "0.4.110"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.109"
version = "0.4.110"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

1
litellm-rust/Cargo.lock generated
View file

@ -4083,6 +4083,7 @@ dependencies = [
name = "litellm-inference-responses"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-util",
"litellm-auth",

1
litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs
View file

@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
"docker_model_runner",
"ragflow",
"pinstripes",
"openinfer",
"darkbloom",
"meta",
"cognition",

2
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -24,7 +24,7 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
pub async fn create(
State(gateway): State<Arc<Gateway>>,

2
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -16,7 +16,7 @@ use wiremock::{
#[rstest]
#[case::anthropic("anthropic/test-model", "/v1/messages", true)]
#[case::azure("azure_ai/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", true)]
#[tokio::test]
async fn messages_reaches_the_provider_and_preserves_json_or_sse(
#[case] model: &str,

71
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
assert_eq!(sent, expected);
}
#[rstest]
#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
#[case::deepseek("deepseek", &[])]
#[case::minimax("minimax", &[])]
#[tokio::test]
async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
call: MessagesCall,
#[case] provider: &str,
#[case] derived: &[AnthropicBeta],
) {
let upstream = upstream([message_response()]).await;
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": safeguards}),
))
.await;
let request = only_request(&upstream).await;
let expected: BetaSet = derived
.iter()
.cloned()
.chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
.collect();
assert_eq!(sent_betas(&request), expected);
assert_eq!(
request.body_json::<Value>().unwrap()["safeguards"],
safeguards
);
}
#[rstest]
#[tokio::test]
async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
call: MessagesCall,
#[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": null}),
))
.await;
let request = only_request(&upstream).await;
assert_eq!(
sent_betas(&request),
BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
);
assert!(
!request
.body_json::<Map<String, Value>>()
.unwrap()
.contains_key("safeguards")
);
}
#[rstest]
#[tokio::test]
async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

1
litellm-rust/crates/inference-responses/AGENTS.md
View file

@ -2,3 +2,4 @@
- This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
- The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
- WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

1
litellm-rust/crates/inference-responses/Cargo.toml
View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

499
litellm-rust/crates/inference-responses/src/ids.rs Normal file
View file

@ -0,0 +1,499 @@
//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
//!
//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
//! request carrying those ids can be routed to the provider and deployment that produced them,
//! and then restored to what the provider originally issued. The native and Python paths serve
//! the same traffic, so an id minted by either one must decode on the other.
//!
//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
//! plaintext:
//!
//! | Id | Applied to | Decoded plaintext |
//! |---|---|---|
//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
//!
//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
//!
//! How they fit together:
//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
//! [`encode_response`] stamps it on every response, unless the id is already managed.
//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
//! decrypts under the API key that produced it, so the router pins replays to that deployment.
//! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
//! carry the deployment in the wrapped content alone.
//! - `cntr_` lets container calls find the provider and deployment that own a container. A
//! container id that already names a provider is left as it is.
//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
//! request input back into the provider's own bytes before the request is sent. The routing
//! decision itself stays in Python's `EncryptedContentAffinityCheck`.
//!
//! Python's quirks are kept on purpose:
//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
//! `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
//! are skipped, trailing bits are allowed, and padding stripped in transit is added back.
//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
use base64::{
Engine, alphabet,
engine::{GeneralPurpose, general_purpose::PAD},
};
use litellm_llms_types::formats::responses::ResponsesApiResponse;
use serde_json::{Map, Value};
const RESPONSE_PREFIX: &str = "resp_";
const ITEM_PREFIX: &str = "encitem_";
const CONTAINER_PREFIX: &str = "cntr_";
const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
/// non-zero trailing bits are accepted
const LENIENT: GeneralPurpose = GeneralPurpose::new(
&alphabet::STANDARD,
PAD.with_decode_allow_trailing_bits(true),
);
/// Where a response came from, as encoded into the ids LiteLLM hands back
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct IdScope {
pub provider: Option<String>,
pub model_id: Option<String>,
pub encrypted_content_affinity: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DecodedContainerId {
pub provider: Option<String>,
pub model_id: Option<String>,
pub container_id: String,
}
/// Rewrites the response id, encrypted-content items and container ids the way the Python
/// Responses path does before returning to the caller. An already managed id is left untouched
pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
if is_managed_response_id(&response.id) {
return;
}
response.id = response_id(scope, &response.id);
if let Some(model_id) = scope
.model_id
.as_deref()
.filter(|_| scope.encrypted_content_affinity)
.filter(|model_id| !model_id.is_empty())
{
response
.output
.iter_mut()
.filter_map(Value::as_object_mut)
.for_each(|item| encode_encrypted_item(item, model_id));
}
response
.output
.iter_mut()
.for_each(|item| encode_containers(item, scope));
}
/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
/// so the provider receives what it originally issued
pub fn restore_input(input: &mut Value) {
let Some(items) = input.as_array_mut() else {
return;
};
for item in items.iter_mut().filter_map(Value::as_object_mut) {
if let Some(Value::String(id)) = item.get_mut("id")
&& let Some((_, original)) = decode_encrypted_item_id(id)
{
*id = original;
}
if let Some(Value::String(content)) = item.get_mut("encrypted_content")
&& let Some(original) = unwrap_encrypted_content(content)
{
*content = original;
}
}
}
pub fn response_id(scope: &IdScope, id: &str) -> String {
let assembled = format!(
"litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
python_str(scope.provider.as_deref()),
python_str(scope.model_id.as_deref()),
);
format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
}
pub fn is_managed_response_id(id: &str) -> bool {
decode_text(&id.replace(RESPONSE_PREFIX, ""))
.is_some_and(|decoded| decoded.split(';').count() >= 3)
}
pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
format!(
"{ITEM_PREFIX}{}",
LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
)
}
pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
let (model_id, item_id) = decoded.split_once(';')?;
Some((
model_id.replace("litellm:model_id:", ""),
item_id.replace("item_id:", ""),
))
}
pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
format!(
"{WRAPPED_CONTENT_PREFIX}{};{content}",
LENIENT.encode(format!("model_id:{model_id}"))
)
}
/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
return None;
}
let (metadata, original) = content.split_once(';')?;
decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
Some(original.to_owned())
}
pub fn container_id(scope: &IdScope, id: &str) -> String {
let assembled = format!(
"litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
scope.provider.as_deref().unwrap_or_default(),
scope.model_id.as_deref().unwrap_or_default(),
);
format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
}
pub fn decode_container_id(id: &str) -> DecodedContainerId {
let unmanaged = || DecodedContainerId {
provider: None,
model_id: None,
container_id: id.to_owned(),
};
if !id.starts_with(CONTAINER_PREFIX) {
return unmanaged();
}
let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
return unmanaged();
};
let parsed = decoded
.strip_prefix("litellm:custom_llm_provider:")
.and_then(|rest| rest.split_once(';'))
.and_then(|(provider, rest)| {
let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
let container_id = rest.strip_prefix("container_id:")?;
(!container_id.is_empty() && !container_id.contains('\n')).then_some((
provider,
model_id,
container_id,
))
});
let Some((provider, model_id, container_id)) = parsed else {
return unmanaged();
};
let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
DecodedContainerId {
provider: present(provider),
model_id: present(model_id),
container_id: container_id.to_owned(),
}
}
fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
return;
};
if content.is_empty() {
return;
}
*content = wrap_encrypted_content(content, model_id);
if let Some(Value::String(id)) = item.get_mut("id")
&& !id.is_empty()
{
*id = encrypted_item_id(model_id, id);
}
}
fn encode_containers(item: &mut Value, scope: &IdScope) {
let Some(item) = item.as_object_mut() else {
return;
};
encode_container_field(item, scope);
if let Some(nested) = item
.get_mut("code_interpreter_call")
.and_then(Value::as_object_mut)
{
encode_container_field(nested, scope);
}
if item.get("type").and_then(Value::as_str) != Some("message") {
return;
}
let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
return;
};
content
.iter_mut()
.filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
.flatten()
.for_each(|annotation| encode_containers(annotation, scope));
}
fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
if let Some(Value::String(id)) = item.get_mut("container_id")
&& decode_container_id(id).provider.is_none()
{
*id = container_id(scope, id);
}
}
fn python_str(value: Option<&str>) -> &str {
value.unwrap_or("None")
}
fn padded(value: &str) -> String {
let missing = value.len() % 4;
let padding = if missing == 0 { 0 } else { 4 - missing };
format!("{value}{}", "=".repeat(padding))
}
fn decode_text(value: &str) -> Option<String> {
let alphabet: String = value
.chars()
.filter(|character| {
character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
})
.collect();
String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
IdScope {
provider: provider.map(str::to_owned),
model_id: model_id.map(str::to_owned),
encrypted_content_affinity: affinity,
}
}
fn response(id: &str, output: Value) -> ResponsesApiResponse {
serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
}
#[rstest]
#[case::openai_without_model_id(
Some("openai"),
None,
"resp_0a1b2c3d4e5f",
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
)]
#[case::openrouter_with_model_id(
Some("openrouter"),
Some("deploy-1"),
"gen-123",
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
)]
fn response_ids_match_python(
#[case] provider: Option<&str>,
#[case] model_id: Option<&str>,
#[case] id: &str,
#[case] expected: &str,
) {
assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
}
#[rstest]
#[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
#[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
#[case::two_segments("resp_YTti", false)]
#[case::three_segments("resp_YTtiO2M=", true)]
#[case::managed(
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
true
)]
fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
assert_eq!(is_managed_response_id(id), expected);
}
#[test]
fn encrypted_items_match_python() {
assert_eq!(
encrypted_item_id("deploy-1", "rs_0a1b"),
"encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
);
assert_eq!(
wrap_encrypted_content("gAAAA", "deploy-1"),
"litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
);
}
#[rstest]
#[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
#[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
#[case::not_base64("encitem_notbase64!", None)]
#[case::unmanaged("rs_0a1b", None)]
fn encrypted_item_ids_decode_like_python(
#[case] id: &str,
#[case] expected: Option<(&str, &str)>,
) {
assert_eq!(
decode_encrypted_item_id(id),
expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
);
}
#[rstest]
#[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
#[case::no_separator("litellm_enc:nosemicolon", None)]
#[case::unwrapped("gAAAA", None)]
fn encrypted_content_unwraps_like_python(
#[case] content: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
}
#[rstest]
#[case::with_model_id(
Some("deploy-1"),
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
)]
#[case::without_model_id(
None,
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
)]
fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
assert_eq!(
container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
expected
);
}
#[rstest]
#[case::raw("cntr_abc", None, None, "cntr_abc")]
#[case::managed(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
Some("openai"),
Some("m"),
"cntr_1"
)]
#[case::empty_segments(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
None,
None,
"cntr_1"
)]
#[case::python_none_provider(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
None,
None,
"x"
)]
fn container_ids_decode_like_python(
#[case] id: &str,
#[case] provider: Option<&str>,
#[case] model_id: Option<&str>,
#[case] container_id: &str,
) {
assert_eq!(
decode_container_id(id),
DecodedContainerId {
provider: provider.map(str::to_owned),
model_id: model_id.map(str::to_owned),
container_id: container_id.to_owned(),
}
);
}
#[test]
fn encodes_response_items_under_affinity() {
let mut encoded = response(
"resp_1",
json!([
{"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
{"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
{"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
{"type": "container_file_citation", "container_id": "cntr_abc"}
]}]}
]),
);
let scope = scope(Some("openai"), Some("deploy-1"), true);
encode_response(&mut encoded, &scope);
let container = container_id(&scope, "cntr_abc");
assert_eq!(encoded.id, response_id(&scope, "resp_1"));
assert_eq!(
Value::Array(encoded.output),
json!([
{
"type": "reasoning",
"id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
"encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
"summary": []
},
{"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
{"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
{"type": "container_file_citation", "container_id": container}
]}]}
])
);
}
#[rstest]
#[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
#[case::no_model_id(scope(Some("openai"), None, true))]
fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
let mut encoded = response("resp_1", output.clone());
encode_response(&mut encoded, &scope);
assert_eq!(Value::Array(encoded.output), output);
}
#[test]
fn leaves_managed_responses_untouched() {
let scope = scope(Some("openai"), Some("deploy-1"), true);
let id = response_id(&scope, "resp_1");
let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
let mut encoded = response(&id, output.clone());
encode_response(&mut encoded, &scope);
assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
}
#[test]
fn restores_encrypted_items_in_input() {
let mut input = json!([
{
"type": "reasoning",
"id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
"encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
"summary": []
},
{"role": "user", "content": "hi"}
]);
restore_input(&mut input);
assert_eq!(
input,
json!([
{"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
{"role": "user", "content": "hi"}
])
);
}
}

1
litellm-rust/crates/inference-responses/src/lib.rs
View file

@ -1,6 +1,7 @@
use litellm_host::observation::ObservationSender;
pub use litellm_inference::RouteError as Error;
pub mod ids;
pub mod route;
pub mod types;
pub mod websocket;

11
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -195,9 +195,10 @@ impl AnthropicBeta {
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
&[BetaProvider::BedrockConverse]
Self::AdvancedToolUse20251120 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -231,12 +232,11 @@ impl AnthropicBeta {
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::DangerousToolUse20260903 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
@ -244,6 +244,7 @@ impl AnthropicBeta {
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]

2
litellm-rust/crates/llms/src/anthropic/beta_headers.rs
View file

@ -76,7 +76,7 @@ mod tests {
#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, None)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
fn filter_renames_per_host(
#[case] provider: BetaProvider,

91
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -109,7 +109,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
merge_beta_headers(
headers,
provider_feature_betas(request, BetaProvider::Anthropic),
)
}
}
@ -136,12 +139,20 @@ pub(crate) fn transform_messages_request(
messages: strip_encrypted_reasoning_blocks(messages),
params: MessagesOptionalParams {
context_management,
extra: without_null_safeguards(request.params.extra),
..request.params
},
..request
})
}
fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
extra
.into_iter()
.filter(|(key, value)| key != "safeguards" || !value.is_null())
.collect()
}
pub(crate) fn update_headers_with_anthropic_beta(
headers: Headers,
request: &MessagesRequest,
@ -149,12 +160,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
merge_beta_headers(headers, feature_betas(request))
}
/// The betas a request needs on a host other than the first-party API: the Anthropic set
/// filtered and renamed through the host's `BetaProvider` policy.
pub(crate) fn update_headers_with_safeguards_beta(
headers: Headers,
request: &MessagesRequest,
provider: BetaProvider,
) -> Headers {
merge_beta_headers(
headers,
feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
)
}
/// The betas a request needs on a host with a column in the beta map, filtered and renamed
/// through that column.
pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
feature_betas(request)
.iter()
.filter_map(|beta| beta.on(provider))
on_provider(
feature_betas(request).union(mapped_host_betas(&request.params)),
provider,
)
}
fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
betas.iter().filter_map(|beta| beta.on(provider)).collect()
}
fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
carries_safeguards(params)
.then_some(AnthropicBeta::DangerousToolUse20260903)
.into_iter()
.collect()
}
@ -206,6 +239,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
.is_some_and(|config| config.format.is_some())
}
fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
}
fn messages_carry_output_config(messages: &[Message]) -> bool {
messages
.iter()
@ -1025,10 +1065,44 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
&["per-turn-control-2026-07-01"]
)]
#[case::safeguards(
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
#[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
#[case::null(BetaProvider::Anthropic, Value::Null, &[])]
#[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
#[case] provider: BetaProvider,
#[case] safeguards: Value,
#[case] expected: &[&str],
) {
assert_eq!(
provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
assert_eq!(
update_headers_with_anthropic_beta(
client.clone(),
&request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
),
client
);
}
#[rstest]
#[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
#[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@ -1067,6 +1141,11 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
&[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
)]
#[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
)]
fn request_headers_merge_the_feature_betas(
#[case] input: &[(&str, &str)],
#[case] fields: Value,

12
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -2,7 +2,7 @@ use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -12,7 +12,7 @@ use crate::{
common_utils::API_KEY_PLACEMENT,
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
},
azure_ai::common_utils::{
@ -104,7 +104,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
}
}
@ -544,6 +544,12 @@ mod tests {
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219")],
&[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
)]
#[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
fn request_headers_carry_the_anthropic_feature_betas(
#[case] fields: serde_json::Value,
#[case] forwarded: &[(&str, &str)],

21
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs
View file

@ -6,7 +6,7 @@ use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
@ -15,7 +15,7 @@ use crate::{
Error, ErrorDetail,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
@ -174,7 +174,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
None => headers,
};
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
}
fn wire_body(&self, body: Value) -> Value {
@ -356,6 +356,21 @@ mod tests {
);
}
#[rstest]
#[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
#[case::null_safeguards(Value::Null, None)]
fn safeguards_need_the_dangerous_tool_use_beta(
#[case] safeguards: Value,
#[case] expected: Option<&str>,
) {
let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
assert_eq!(
header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
expected.map(str::to_string)
);
}
#[rstest]
#[case::default(json!({}), false)]
#[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

23
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -451,4 +451,27 @@ mod tests {
json!({"type": "enabled", "budget_tokens": 1024})
);
}
#[rstest]
#[case::without_client_betas(&[])]
#[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
let body = json!({
"model": "deepseek-v4-pro",
"max_tokens": 100,
"messages": [{"role": "user", "content": "hi"}],
"safeguards": safeguards
});
let headers: Headers = client
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect();
assert_eq!(
DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
.request_headers(headers.clone(), &request_from(body.clone())),
headers
);
assert_eq!(transformed(body)["safeguards"], safeguards);
}
}

19
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -213,23 +213,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
}
/// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
/// out.
/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
let params = &request.params;
provider_feature_betas(request, BetaProvider::VertexAi).union(
[
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305),
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
.then_some(AnthropicBeta::DangerousToolUse20260903),
]
.into_iter()
.flatten()
.collect(),
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305)
.into_iter()
.collect(),
)
}

1
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -2,5 +2,4 @@
For `model_prices_and_context_window.json` validation, we should eventually:
- Remove any schema file like `model_prices_and_context_window.schema.json`
- Stop skipping this crate's tests

73
litellm-rust/crates/python-bridge/src/routes/responses/host.rs
View file

@ -1,22 +1,41 @@
use std::convert::Infallible;
use crate::routes::inference::InferenceHost;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
use litellm_inference_responses::{
Error,
ids::{self, IdScope},
route::Responses,
types::ResponsesCall,
};
use pyo3::{
gc::{PyTraverseError, PyVisit},
prelude::*,
types::PyDict,
};
pub(super) struct ResponsesPythonHost(pub InferenceHost);
pub(super) struct ResponsesPythonHost {
host: InferenceHost,
ids: IdScope,
}
impl ResponsesPythonHost {
pub fn new(host: InferenceHost) -> Self {
Self {
host,
ids: IdScope::default(),
}
}
}
pub(super) fn project(
host: &mut InferenceHost,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> PyResult<ResponsesCall> {
let call = host.project(py, arguments, "input")?;
let mut call = host.project(py, arguments, "input")?;
ids::restore_input(&mut call.input);
let optional_params = call
.params
.into_iter()
@ -50,6 +69,39 @@ pub(super) fn project(
})
}
/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
/// `litellm_metadata.model_info.id`
fn id_scope(
host: &InferenceHost,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
call: &ResponsesCall,
) -> PyResult<IdScope> {
let metadata = host.argument(py, arguments, "litellm_metadata")?;
let encrypted_content_affinity = match &metadata {
Some(metadata) => metadata
.call_method1("get", ("encrypted_content_affinity_enabled",))?
.is_truthy()?,
None => false,
};
let metadata = metadata
.map(|value| from_py::<serde_json::Value>(&value))
.transpose()?
.unwrap_or_default();
let model_id = match metadata.pointer("/model_info/id") {
None | Some(serde_json::Value::Null) => None,
Some(serde_json::Value::String(id)) => Some(id.clone()),
Some(other) => Some(other.to_string()),
};
Ok(IdScope {
provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
.map(|resolved| resolved.custom_llm_provider.to_owned())
.or_else(|| call.custom_llm_provider.clone()),
model_id,
encrypted_content_affinity,
})
}
impl PythonBinding for ResponsesPythonHost {
type Protocol = Responses;
type Failure = PyErr;
@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> Result<ResponsesCall, InvokeError<Error>> {
let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
if call
.optional_params
.get("stream")
@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
py: Python<'_>,
response: <Responses as litellm_host::protocol::Protocol>::Response,
) -> PyResult<Py<PyAny>> {
self.0.response(py, &response)
let mut response = response;
ids::encode_response(&mut response, &self.ids);
self.host.response(py, &response)
}
fn encode_stream_head(
@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
}
fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
self.0.error(py, error)
self.host.error(py, error)
}
fn host_error(error: &PyErr) -> Error {
Error::InvalidRequest(error.to_string().into())
@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
impl PythonOwned for ResponsesPythonHost {
fn close(&mut self, _: Python<'_>) {
self.0.close();
self.host.close();
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.0.traverse(visit)
self.host.traverse(visit)
}
}

2
litellm-rust/crates/python-bridge/src/routes/responses/mod.rs
View file

@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
));
}
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
}
#[pyfunction]

45
litellm/__init__.py
View file

@ -1,34 +1,13 @@
### Hide pydantic namespace conflict warnings globally ###
from __future__ import annotations
import warnings
from litellm._process_defaults import apply_process_defaults as _apply_process_defaults
_apply_process_defaults()
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
from collections.abc import Mapping, Sequence
from typing import (
Any,
@ -207,7 +186,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
post_call_rules: List[Callable[..., object]] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
@ -393,6 +372,7 @@ key_alias_pattern: str | None = None
enable_gemini_default_thinking_level_low: bool = (
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
)
gemini_include_thoughts_default: bool = True
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@ -696,6 +676,7 @@ qwen_ai_platform_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@ -762,7 +743,7 @@ def is_openai_finetune_model(key: str) -> bool:
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in model_cost_map.items():
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
@ -965,6 +946,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
publicai_models.add(key)
elif value.get("litellm_provider") == "darkbloom":
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1133,6 +1116,7 @@ model_list = list(
| moonshot_models
| publicai_models
| darkbloom_models
| openinfer_models
| v0_models
| morph_models
| lambda_ai_models
@ -1176,7 +1160,9 @@ def _build_models_by_provider() -> dict:
"edenai": edenai_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
| vertex_code_chat_models
| vertex_text_models
| vertex_code_text_models
| vertex_anthropic_models
| vertex_vision_models
| vertex_language_models
@ -1184,7 +1170,7 @@ def _build_models_by_provider() -> dict:
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
@ -1247,6 +1233,7 @@ def _build_models_by_provider() -> dict:
"moonshot": moonshot_models,
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,
@ -1415,6 +1402,7 @@ from .exceptions import (
InvalidRequestError,
BadRequestError,
ImageFetchError,
MalformedRequestFieldError as MalformedRequestFieldError,
VectorStoreSearchError,
NotFoundError,
PermissionDeniedError,
@ -1690,6 +1678,9 @@ if TYPE_CHECKING:
from .llms.perplexity.decisions.transformation import (
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
)
from .llms.nace.decisions.transformation import (
NaceDecisionsConfig as NaceDecisionsConfig,
)
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)

10
litellm/_internal_context.py
View file

@ -37,6 +37,16 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
@contextmanager
def internal_sub_call() -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
try:
yield
finally:
is_internal_call.reset(token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

2
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMRerankConfig",
),
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

2
litellm/_logging.py
View file

@ -1000,6 +1000,8 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
lg.handlers.clear() # remove any existing handlers
lg.addHandler(handler) # add JSON formatter handler
lg.propagate = False # prevent bubbling to parent/root
if (configured_level := os.getenv("LITELLM_LOG")) is not None:
logging.getLogger().setLevel(resolve_log_level(configured_level))
def _get_uvicorn_json_log_config():

24
litellm/_process_defaults.py Normal file
View file

@ -0,0 +1,24 @@
import os
import warnings
import dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
def apply_process_defaults() -> None:
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())

22
litellm/anthropic_beta_headers_config.json
View file

@ -220,36 +220,36 @@
},
"databricks": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"advanced-tool-use-2025-11-20": null,
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"code-execution-2025-08-25": null,
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": null,
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"structured-output-2024-03-01": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"oauth-2025-04-20": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05"
}
}

45
litellm/anthropic_beta_headers_manager.py
View file

@ -22,17 +22,28 @@ Environment Variables:
- LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
"""
import asyncio
import json
import os
import threading
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
class GetAnthropicBetaHeadersConfig:
@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
with _BETA_HEADERS_CONFIG_LOCK:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
return _BETA_HEADERS_CONFIG
async def aload_beta_headers_config() -> None:
if _BETA_HEADERS_CONFIG is None:
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
@cache
def _bundled_beta_headers_config() -> BetaHeadersConfig:
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
def reload_beta_headers_config() -> dict:
@ -288,6 +313,14 @@ def is_beta_header_supported(
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
provider_mapping: Final = config.get(canonical_provider, {})
return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,

8
litellm/assistants/main.py
View file

@ -129,6 +129,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_assistants=aget_assistants,
)
@ -312,6 +313,7 @@ def create_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants,
@ -461,6 +463,7 @@ def delete_assistant(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
assistant_id=assistant_id,
client=client,
async_delete_assistants=async_delete_assistants,
@ -637,6 +640,7 @@ def create_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
acreate_thread=acreate_thread,
)
@ -783,6 +787,7 @@ def get_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_thread=aget_thread,
)
@ -960,6 +965,7 @@ def add_message(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
a_add_message=a_add_message,
)
@ -1111,6 +1117,7 @@ def get_messages(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_messages=aget_messages,
)
@ -1308,6 +1315,7 @@ def run_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
arun_thread=arun_thread,
event_handler=event_handler,

19
litellm/batches/batch_utils.py
View file

@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
)
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
from litellm.utils import token_counter
@ -569,6 +569,7 @@ def extract_file_access_credentials(
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.
*sorted(ANTHROPIC_WIF_KWARGS_KEYS),
*sorted(OPENAI_WIF_KWARGS_KEYS),
)
for key in credential_keys:
if key in litellm_params:
@ -793,11 +794,17 @@ def _batch_response_was_successful(
message batch results lines report ``result.type == "succeeded"``; Bedrock
batch output lines report ``modelOutput`` (and no ``error``).
"""
if custom_llm_provider == "anthropic":
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
if custom_llm_provider == "bedrock":
return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
_response: Final[dict[str, object]] = cast( # cast-ok: batch response bodies come from provider JSON
dict[str, object], batch_job_output_file.get("response", None) or {}
envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
envelope: Final = batch_job_output_file.get(envelope_key)
if not isinstance(envelope, Mapping):
if envelope is not None:
verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
return False
response: Final = cast(Mapping[object, object], envelope) # cast-ok: mapping shape validated above
return (
response.get("type") == "succeeded"
if custom_llm_provider == "anthropic"
else response.get("status_code") == 200
)
return _response.get("status_code", None) == 200

4
litellm/batches/main.py
View file

@ -309,6 +309,7 @@ def create_batch(
timeout=timeout,
max_retries=optional_params.max_retries,
_is_async=_is_async,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -819,6 +821,7 @@ def list_batches(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -1016,6 +1019,7 @@ def cancel_batch(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

13
litellm/caching/caching.py
View file

@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -376,7 +376,7 @@ class Cache:
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def get_cache_key(self, **kwargs) -> str:
def get_cache_key(self, **kwargs: object) -> str:
"""
Get the cache key for the given arguments.
@ -410,13 +410,16 @@ class Cache:
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
if kwargs[param] is None:
continue # ignore None params
param_value = kwargs[param]
cache_key += f"{param}: {param_value}"
cache_key += f"{param}: {kwargs[param]}"
if is_semantic_cache:
cache_key += self._get_semantic_cache_tenant_scope(kwargs)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs)
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

21
litellm/caching/caching_handler.py
View file

@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
import litellm
from litellm._internal_context import post_response_phase
from litellm._internal_context import is_internal_call, post_response_phase
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
@ -346,6 +346,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
verbose_logger.debug("Checking Async Cache")
logging_obj.reset_caching_details()
cached_result = await self._retrieve_from_cache(
call_type=call_type,
kwargs=kwargs,
@ -357,7 +358,7 @@ class LLMCachingHandler:
verbose_logger.debug("Cache Hit!")
cache_hit: Final = True
end_time: Final = datetime.datetime.now()
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
api_base=kwargs.get("api_base", None),
@ -366,7 +367,7 @@ class LLMCachingHandler:
cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
self._update_litellm_logging_obj_environment(
logging_obj=logging_obj,
model=model,
model=provider_model,
kwargs=kwargs,
cached_result=cached_result,
is_async=True,
@ -381,7 +382,7 @@ class LLMCachingHandler:
call_type=call_type,
kwargs=kwargs,
logging_obj=logging_obj,
model=model,
model=provider_model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
@ -467,6 +468,7 @@ class LLMCachingHandler:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
if is_response_without_output(cached_result):
@ -511,8 +513,10 @@ class LLMCachingHandler:
custom_llm_provider=custom_llm_provider,
)
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
logging_obj.handle_sync_success_callbacks_for_async_calls(
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
):
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
start_time=start_time,
end_time=end_time,
@ -820,7 +824,12 @@ class LLMCachingHandler:
start_time (datetime): The start time of the operation.
end_time (datetime): The end time of the operation.
cache_hit (bool): Whether it was a cache hit.
An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
"""
if is_internal_call.get():
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

45
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import (
record_output_item_chunk,
record_output_text_chunk,
)
from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options
from litellm.responses.utils import (
ResponsesAPIRequestUtils,
auto_reasoning_summary,
normalize_responses_api_stream_options,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]:
return [_without_audio_input_parts_in_item(item) for item in input_items]
def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object:
match top_level_effort, nested_effort:
case {"summary": str() as summary}, str() as effort:
return {"effort": effort, "summary": summary}
case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning:
return {**reasoning, "summary": summary}
case _:
return nested_effort
def _served_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
return custom_llm_provider if isinstance(custom_llm_provider, str) else None
@ -197,7 +211,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
return not _supports_audio_input(model, litellm_params)
def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params: dict,
responses_api_request: "ResponsesAPIOptionalRequestParams",
drop_params: bool = False,
model: str | None = None,
custom_llm_provider: str | None = None,
) -> None:
"""Map optional_params into responses_api_request (mutates in place)."""
for key, value in optional_params.items():
@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self.map_reasoning_effort(value)
responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params,
responses_api_request,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
model=model,
custom_llm_provider=_served_provider(litellm_params),
)
stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
@ -1455,7 +1473,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
for key, value in _provider_metadata(raw_response.model_extra).items():
for key, value in responses_provider_metadata(raw_response.model_extra).items():
setattr(model_response, key, value)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Extract supported params from extra_body and merge into optional_params
extra_body_copy: Final = extra_body.copy()
top_level_effort: Final = cast(object, optional_params.get("reasoning_effort")) # cast-ok: upcast
for key, value in extra_body_copy.items():
if key in supported_responses_api_params:
# Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
optional_params[key] = extra_body.pop(key)
extra_body.pop(key)
optional_params[key] = (
_effort_with_requested_summary(cast(object, value), top_level_effort) # cast-ok: upcast
if key == "reasoning_effort"
else value
)
return optional_params
def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
def map_reasoning_effort(
self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None
) -> Reasoning:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(
@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
return (
Reasoning(effort=reasoning_effort, summary="detailed")
Reasoning(
effort=reasoning_effort,
summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
)
if auto_summary_enabled
else Reasoning(effort=reasoning_effort)
)
@ -2139,7 +2168,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
provider_metadata: Final = _provider_metadata(response_data)
provider_metadata: Final = responses_provider_metadata(response_data)
served_service_tier: Final = response_data.get("service_tier")
return ModelResponseStream(
choices=[

16
litellm/constants.py
View file

@ -1,4 +1,5 @@
import os
import re
import sys
from enum import Enum
from types import MappingProxyType
@ -18,6 +19,9 @@ MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_PROFILE: Final = "jwt_bearer_obo"
MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_SCOPE: Final = "https://graph.microsoft.com/.default"
# Replies depend on the caller's delegated identity, which response-cache keys do not include.
RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot", "databricks_agent"})
CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE)
GATEWAY_CLIENT_HEADER: Final = "x-gateway-client"
CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
@ -57,6 +61,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
@ -146,6 +151,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
@ -389,7 +395,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@ -1024,6 +1030,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
]
@ -1092,6 +1099,7 @@ openai_compatible_providers: Final[list] = [
"docker_model_runner",
"ragflow",
"pinstripes", # Pinstripes - JSON-configured provider
"openinfer",
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
@ -1613,6 +1621,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
# Headers to control callbacks
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
@ -1624,11 +1633,13 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
"Truncation is a DB storage safeguard. "
@ -1705,6 +1716,7 @@ AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:tran
AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com"
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com"
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key"
AWS_ERROR_TYPE_HEADER: Final = "x-amzn-errortype"
AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio"
AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription"
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL: Final = "fast-transcription"
@ -1855,6 +1867,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))
@ -1894,6 +1907,7 @@ _background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS: Final = 300.0
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

1
litellm/containers/README.md
View file

@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm

146
litellm/cost_calculator.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -147,6 +150,7 @@ from litellm.utils import (
TranscriptionResponse,
cached_get_model_info_helper,
token_counter,
unmapped_model_generalization,
)
if TYPE_CHECKING:
@ -325,6 +329,15 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return False
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -863,6 +876,16 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
return own_model_info["litellm_provider"]
def _pricing_provider_for_cost_candidate(
candidate_provider: str | None,
request_provider: str | None,
current_provider: object,
) -> str | None:
if candidate_provider != request_provider:
return candidate_provider
return current_provider if isinstance(current_provider, str) else None
def select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -1426,32 +1449,15 @@ def completion_cost(
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
Return the USD cost of a LiteLLM call.
Parameters:
completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
`messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
`litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
[OPTIONAL PARAMS]
model (str): Optional. The name of the language model used in the completion calls
prompt (str): Optional. The input prompt passed to the llm
completion (str): Optional. The output completion text from the llm
total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
Returns:
float: The cost in USD dollars for the completion based on the provided parameters.
Exceptions:
Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
Note:
- If completion_response is provided, the function extracts token information and the model name from it.
- If completion_response is not provided, the function calculates token counts based on the model and input text.
- The cost is calculated based on the model, prompt tokens, and completion tokens.
- For certain models containing "togethercomputer" in the name, prices are based on the model size.
- For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
@ -1549,14 +1555,20 @@ def completion_cost(
)
requested_model: Final = model
potential_model_names: Final = [
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
raw_candidates: Final = (
(
selected_model,
base_model_provider if base_model_provider is not None else request_pricing_provider,
),
(_get_response_model(completion_response), request_pricing_provider),
*(((model, request_pricing_provider),) if model is not None else ()),
)
potential_model_names: Final = (
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, model in enumerate(potential_model_names):
for idx, (model, pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
@ -1739,6 +1751,12 @@ def completion_cost(
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -1895,13 +1913,6 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
pricing_provider: str | None = (
base_model_provider
if base_model_provider is not None and idx == 0
else custom_llm_provider
if isinstance(custom_llm_provider, str)
else None
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1909,7 +1920,11 @@ def completion_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
@ -2008,7 +2023,11 @@ def completion_cost(
if cost_per_token_usage_object is not None and model:
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
usage=cost_per_token_usage_object,
service_tier=service_tier,
data_residency=data_residency,
@ -2048,7 +2067,7 @@ def completion_cost(
)
if idx == len(potential_model_names) - 1:
raise e
raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
except Exception as e:
raise e
@ -2063,13 +2082,20 @@ def get_response_cost_from_hidden_params(
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
if response_cost is None:
return None
return float(additional_headers["llm_provider-x-litellm-response-cost"])
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def _billable_header_cost(value: object) -> float | None:
if not isinstance(value, (str, int, float)):
return None
try:
cost: Final = float(value)
except ValueError:
return None
return cost if math.isfinite(cost) and cost >= 0 else None
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -2263,6 +2289,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
return None
_CostCandidate: TypeAlias = tuple[str | None, str | None]
def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
model, pricing_provider = candidate
if model is None:
return False
rule_info: Final = unmapped_model_generalization(model, pricing_provider)
return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
return (
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@ -3068,14 +3113,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
return response_ids[-1] if response_ids else None
@staticmethod
def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
def create_logging_object(
results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
) -> LiteLLMRealtimeStreamLoggingObject:
tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
return LiteLLMRealtimeStreamLoggingObject(
logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
results=cast(OpenAIRealtimeStreamList, list(results)), # cast-ok: raw WS frames, unvalidated field
service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
)
if litellm_model_name is not None:
logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
return logging_object
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

10
litellm/decisions/call.py
View file

@ -57,7 +57,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@ -458,7 +458,7 @@ def prepare_call(
) from error
try:
resolved_api_base, resolved_api_key = provider_config.resolve_connection(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or api_base, dynamic_api_key or api_key
)
except ValueError as error:
@ -469,6 +469,7 @@ def prepare_call(
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
@ -666,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)

31
litellm/decisions/main.py
View file

@ -1,5 +1,7 @@
import asyncio
import warnings
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -104,19 +106,22 @@ async def adecisions(
videos: DecisionsVideoClips | None = None,
**kwargs: object,
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_decisions_call,
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await asend(call)

6
litellm/exceptions.py
View file

@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
)
class MalformedRequestFieldError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
@ -1217,6 +1221,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
num_retries: int | None = None,
generated_content: str = "",
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
@ -1229,6 +1234,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.num_retries = num_retries
self.generated_content = generated_content
self.is_pre_first_chunk = is_pre_first_chunk
self.emitted_disqualifying_content = emitted_disqualifying_content
# Create a response if one wasn't provided
if response is None:

14
litellm/experimental_mcp_client/client.py
View file

@ -498,6 +498,7 @@ class MCPClient:
elicitation_callback: "ElicitationCallback | None" = None,
logging_callback: Callable | None = None,
protocol_version: MCPUpstreamProtocol = "auto",
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
MCPUpstreamProtocol
@ -524,6 +525,7 @@ class MCPClient:
self._sampling_callback: Callable | None = sampling_callback
self._elicitation_callback: ElicitationCallback | None = elicitation_callback
self._logging_callback: Callable | None = logging_callback
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
self._session_callbacks_override: _SessionCallbackMap | None = None
# handle the basic auth value if provided
if auth_value:
@ -610,7 +612,11 @@ class MCPClient:
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
return (
sse_client(
url=self.server_url,
@ -622,7 +628,11 @@ class MCPClient:
)
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
http_client = httpx_client_factory(
headers=headers,

6
litellm/files/main.py
View file

@ -259,6 +259,7 @@ def create_file(
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -388,6 +389,7 @@ def file_retrieve(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -569,6 +571,7 @@ def file_delete(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -775,6 +778,7 @@ def file_list(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -972,6 +976,7 @@ def file_content(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -1093,6 +1098,7 @@ def file_content_streaming(
organization=openai_creds.organization,
chunk_size=chunk_size,
client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
if not _is_async:

4
litellm/fine_tuning/main.py
View file

@ -239,6 +239,7 @@ def create_fine_tuning_job(
client=kwargs.get(
"client", None
), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":

3
litellm/google_genai/main.py
View file

@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
BaseGoogleGenAIGenerateContentConfig,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes
@ -194,12 +195,14 @@ class GenerateContentHelper:
if litellm_logging_obj is None:
raise ValueError("litellm_logging_obj is required, but got None")
vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
optional_params=dict(generate_content_config_dict),
litellm_params={
"litellm_call_id": litellm_call_id,
**({"vertex_location": vertex_location} if vertex_location else {}),
},
custom_llm_provider=custom_llm_provider,
)

9
litellm/images/main.py
View file

@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -505,6 +506,7 @@ def image_generation(
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
if model is None:
@ -681,10 +683,13 @@ def image_variation(
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
if api_key is None and (
resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
is None
):
raise ValueError("API key is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,

30
litellm/integrations/custom_guardrail.py
View file

@ -695,9 +695,7 @@ class CustomGuardrail(CustomLogger):
supported_event_hooks: list[GuardrailEventHooks],
) -> None:
allowed_hooks: Final = frozenset(supported_event_hooks) | (
frozenset((GuardrailEventHooks.logging_only,))
if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
else frozenset()
frozenset((GuardrailEventHooks.logging_only,)) if self.accepts_logging_only_mode() else frozenset()
)
def _validate_event_hook_list_is_in_supported_event_hooks(
@ -869,8 +867,13 @@ class CustomGuardrail(CustomLogger):
return True
return False
def uses_apply_guardrail_interface(self) -> bool:
return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def uses_apply_guardrail_interface(cls) -> bool:
return cls.apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def accepts_logging_only_mode(cls) -> bool:
return cls.uses_apply_guardrail_interface() and not cls.use_native_lifecycle_hooks
@classmethod
def supports_logging_only_scope(cls) -> bool:
@ -1029,7 +1032,7 @@ class CustomGuardrail(CustomLogger):
)
return kwargs, result
litellm_params: Final = kwargs.get("litellm_params") or {}
scratch_metadata: Final = {
scratch_metadata: Final[dict[str, object]] = { # mutable-ok: apply_guardrail records its verdict here
key: value
for key, value in (litellm_params.get("metadata") or {}).items()
if key != "standard_logging_guardrail_information"
@ -1083,11 +1086,11 @@ class CustomGuardrail(CustomLogger):
async def _scan_logged_call(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
kwargs: dict[str, object], # mutable-ok: CustomLogger.async_logging_hook contract
response: object | None,
translation: "BaseTranslation",
output_translation: "BaseTranslation",
scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata
scratch_metadata: dict[str, object], # mutable-ok: apply_guardrail records its verdict into request metadata
) -> None:
scratch_fields: Final = self._copy_scratch_request_fields(kwargs)
scratch_input, scratch_tools = scratch_fields or (None, None)
@ -1136,6 +1139,17 @@ class CustomGuardrail(CustomLogger):
"""
return False
def supports_structured_messages(self) -> bool:
"""Whether this guardrail scans every row of ``structured_messages`` and returns texts that agree with them.
Guardrails that scan only part of the rows (the latest user turn, the
last row, the turns after the last assistant turn, or no tool results),
derive their returned texts from the rows, or cannot rewrite rows whose
content is a list of parts, override this to return False. Translation handlers that can rewrite the request from ``texts``
alone, like Bedrock Converse, then hand them only ``texts``.
"""
return True
def _suppressed_by_auto_router_compression(self) -> bool:
"""True when an auto router's own compression policy suppresses this guardrail.

43
litellm/integrations/custom_logger.py
View file

@ -912,9 +912,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
This is useful for logging payloads that contain sensitive information.
"""
import litellm
from litellm import Choices, Message, ModelResponse
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
from litellm.litellm_core_utils.redact_messages import (
redacted_litellm_params,
redacted_standard_logging_payload,
)
turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@ -940,41 +942,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
}
# Handle turn_off_message_logging - redact messages and responses (if not already excluded)
if turn_off_message_logging and not self.redacts_messages_itself():
redacted_str: Final = "redacted-by-litellm"
if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None:
standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()]
if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None:
response: Final = standard_logging_object_copy["response"]
# Check if this is a ResponsesAPIResponse (has "output" field)
if isinstance(response, dict) and "output" in response:
# Make a copy to avoid modifying the original
from copy import deepcopy
response_copy: Final = deepcopy(response)
# Redact content in output array
if isinstance(response_copy.get("output"), list):
for output_item in response_copy["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
# Redact text in content items
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
content_item["text"] = redacted_str
standard_logging_object_copy["response"] = response_copy
else:
# Standard ModelResponse format
model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])
model_response_dict: Final = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
return {
**model_call_details,
**redacted_params,
"standard_logging_object": standard_logging_object_copy,
"standard_logging_object": (
redacted_standard_logging_payload(standard_logging_object_copy)
if turn_off_message_logging and not self.redacts_messages_itself()
else standard_logging_object_copy
),
}
async def get_proxy_server_request_from_cold_storage_with_object_key(

2
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -219,7 +219,7 @@ class GitLabTemplateManager:
elif value.replace(".", "").isdigit():
try:
result[key] = float(value)
except Exception:
except ValueError:
result[key] = value
else:
result[key] = value.strip("\"'")

71
litellm/integrations/langfuse/langfuse.py
View file

@ -103,6 +103,43 @@ def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_
)
def langfuse_trace_metadata(metadata: object, proxy_server_request: object) -> Mapping[str, object]:
entries: Final = _object_mapping(metadata) or _NO_METADATA
request: Final = _object_mapping(proxy_server_request)
headers: Final = _object_mapping(request.get("headers")) if request is not None else None
if headers is None:
return entries
header_entries: Final = {
key.removeprefix("langfuse_"): value
for key, value in _widened_items(headers)
if isinstance(key, str) and key.startswith("langfuse_")
}
return MappingProxyType({**entries, **header_entries})
def langfuse_trace_id(
metadata: Mapping[str, object],
standard_trace_id: object,
litellm_trace_id: object,
litellm_call_id: object,
proxy_server_request: object,
) -> object:
existing_trace_id: Final = metadata.get("existing_trace_id")
if existing_trace_id is not None:
return existing_trace_id
metadata_trace_id: Final = metadata.get("trace_id")
trace_id: Final = (
metadata_trace_id
if metadata_trace_id is not None
else standard_trace_id
if standard_trace_id is not None
else litellm_trace_id or litellm_call_id
)
if _is_session_header_trace(trace_id, metadata.get("session_id"), proxy_server_request):
return litellm_call_id or trace_id
return trace_id
class _UsageObject(Protocol):
"""Token-count surface the Langfuse logger reads off a response usage payload."""
@ -680,33 +717,19 @@ class LangFuseLogger:
# Add default langfuse tags
tags = self.add_default_langfuse_tags(tags=tags, kwargs=kwargs, metadata=metadata)
call_trace_id: Final = langfuse_trace_id(
metadata=clean_metadata,
standard_trace_id=standard_logging_object.get("trace_id")
if standard_logging_object is not None
else None,
litellm_trace_id=kwargs.get("litellm_trace_id"),
litellm_call_id=litellm_call_id,
proxy_server_request=litellm_params.get("proxy_server_request"),
)
session_id: Final = clean_metadata.pop("session_id", None)
trace_name = cast(str | None, clean_metadata.pop("trace_name", None))
trace_id = clean_metadata.pop("trace_id", None)
# Use standard_logging_object.trace_id if available (when trace_id from metadata is None)
# This allows standard trace_id to be used when provided in standard_logging_object
if trace_id is None and standard_logging_object is not None:
trace_id = cast(str | None, standard_logging_object.get("trace_id"))
# Fallback to litellm_call_id if no trace_id found
if trace_id is None:
trace_id = kwargs.get("litellm_trace_id") or litellm_call_id
clean_metadata.pop("trace_id", None)
existing_trace_id: Final = clean_metadata.pop("existing_trace_id", None)
# If existing_trace_id is provided, use it as the trace_id to return
# This allows continuing an existing trace while still returning the correct trace_id
if existing_trace_id is not None:
trace_id = existing_trace_id
call_trace_id: Final = (
litellm_call_id or trace_id
if existing_trace_id is None
and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request"))
else trace_id
)
if call_trace_id != trace_id:
verbose_logger.debug(
"Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace",
trace_id,
call_trace_id,
)
requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
update_trace_keys: Final = (
requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()

18
litellm/integrations/opentelemetry.py
View file

@ -6,9 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -106,6 +107,14 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@ -2434,6 +2443,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
litellm_params=litellm_params,
)
# Cost breakdown tracking
cache_view: Final = _CACHE_HIT_VIEW.validate_python(standard_logging_payload)
cache_hit: Final = cache_view.get("cache_hit") is True
cost_breakdown: Final[CostBreakdown | None] = standard_logging_payload.get("cost_breakdown")
if cost_breakdown:
for key, value in cost_breakdown.items():
@ -2443,6 +2454,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key=f"gen_ai.cost.{key}",
value=value,
)
self.safe_set_attribute(span=span, key=LiteLLM.CACHE_HIT, value=cache_hit)
if cache_hit:
saved_cache_cost: Final = cache_view.get("saved_cache_cost")
if isinstance(saved_cache_cost, (int, float)):
self.safe_set_attribute(span=span, key=LiteLLM.SAVED_CACHE_COST, value=saved_cache_cost)
#############################################
########## LLM Request Attributes ###########
#############################################

32
litellm/integrations/otel/emitter.py
View file

@ -100,23 +100,19 @@ def _carried_keys(
return frozenset(key for key, _value in attributes)
def attribute_budget(span: Span, reserved: int, overwrites: frozenset[str] = frozenset()) -> int | None:
"""How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more.
def carried_keys(span: Span) -> frozenset[str]:
"""The attribute keys ``span`` already holds, empty for a span that exposes none."""
if not isinstance(span, ReadableSpan):
return frozenset()
return _carried_keys(span.attributes or ())
``overwrites`` are the mapped keys already present on ``span``: setting one
replaces the value in place and consumes no slot against the limit, so only
the genuinely new pre-existing keys reduce the budget. Counting the
overwritten ones too reserves slots the fit can never spend and sheds
indexed message attributes for nothing.
"""
def attribute_budget(span: Span, reserved: int) -> int | None:
"""How many new attribute keys fit on ``span`` next to what it already carries and ``reserved`` more."""
limit: Final = span_attribute_limit(span)
if limit is None:
return None
if not isinstance(span, ReadableSpan):
return limit - reserved
existing_keys: Final = _carried_keys(span.attributes or ())
fresh: Final = len(existing_keys - overwrites) if overwrites else len(existing_keys)
return limit - fresh - reserved
return limit - len(carried_keys(span)) - reserved
def stamp_error(
@ -303,8 +299,14 @@ class SpanEmitter:
)
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
span.set_attribute(key, value)
overwrites: Final = carried_keys(span) & mapped.keys()
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
)
for key, value in mapped.items():
if key in overwrites or key in fitted:
span.set_attribute(key, value)
if error:
stamped: Final = stamp_error(span, error)
if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:

8
litellm/integrations/otel/logger.py
View file

@ -538,8 +538,12 @@ class OpenTelemetryV2(CustomLogger):
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)

2
litellm/integrations/otel/mappers/genai.py
View file

@ -91,6 +91,8 @@ class GenAIMapper:
f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.CACHE_HIT: lambda d: d.cache_hit,
LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose,

5
litellm/integrations/otel/model/metadata.py
View file

@ -249,6 +249,10 @@ class LLMCallEvent:
# ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the
# caller's own provider attempt.
purpose: str | None
# True for a response served from the litellm response cache. A cache hit
# never hands off to a provider, so ``upstream_started`` stays False even
# though the call completed and is worth a span.
cache_hit: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -273,6 +277,7 @@ class LLMCallEvent:
upstream_started=kwargs.get("api_call_start_time") is not None,
upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
purpose=internal_call_origin(payload, kwargs),
cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
trace=trace,

7
litellm/integrations/otel/model/payloads.py
View file

@ -441,6 +441,8 @@ class LLMCallSpanData:
embedding_output: EmbeddingOutput | None = None
promoted_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA)
routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
cache_hit: bool | None = None
saved_cache_cost: float | None = None
@classmethod
def from_standard_logging_payload(
@ -456,6 +458,7 @@ class LLMCallSpanData:
metadata_keys: tuple[str, ...] = (),
) -> LLMCallSpanData:
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
cache_hit: Final = as_bool(payload.get("cache_hit")) is True # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
# The single parse of the request's metadata — the request-vs-provider
# model split, the response model, api base, and identity all come from
# here rather than being re-derived from the raw payload dicts.
@ -490,7 +493,7 @@ class LLMCallSpanData:
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
server=ServerInfo.from_api_base(context.api_base),
server=None if cache_hit else ServerInfo.from_api_base(context.api_base),
identity=context.identity,
promoted_metadata=allowlisted_metadata(context.identity.metadata, metadata_keys),
is_streaming=as_bool(payload.get("stream")),
@ -507,6 +510,8 @@ class LLMCallSpanData:
session_id=session_id or None,
embedding_output=embedding_output if capture_content else None,
routing_attributes=routing_decision_attributes(decision),
cache_hit=cache_hit,
saved_cache_cost=as_float(payload.get("saved_cache_cost")) if cache_hit else None, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
)

2
litellm/integrations/otel/model/semconv.py
View file

@ -340,6 +340,8 @@ class LiteLLM:
# semconv key for an MCP server's *name* (the convention uses ``server.address``
# for its network location), so it lives under the vendor namespace.
MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
CACHE_HIT: Final = "litellm.cache_hit"
SAVED_CACHE_COST: Final = "litellm.saved_cache_cost"
class Metric:

32
litellm/integrations/prometheus.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
PROXY_REJECTED_BEFORE_ROUTING_KEY,
)
from litellm.exceptions import (
MalformedRequestFieldError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -258,6 +259,7 @@ _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field
_POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)])
_SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
_DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
_ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@ -269,6 +271,15 @@ def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _Series
return None
def _gauge_reading_or_none(value: object) -> float | None:
if not isinstance(value, str | int | float):
return None
try:
return float(value)
except (ValueError, OverflowError):
return None
def _positive_or_ignored(setting: str, value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
if value is None:
return None
@ -3104,7 +3115,11 @@ class PrometheusLogger(CustomLogger):
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
deployment_selected: Final = (
bool(model_id)
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
and not isinstance(exception, MalformedRequestFieldError)
)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id
@ -3345,12 +3360,11 @@ class PrometheusLogger(CustomLogger):
model_group=enum_values.model_group,
)
remaining_requests: int | None = None
remaining_tokens: int | None = None
if additional_headers := standard_logging_payload["hidden_params"]["additional_headers"]:
# OpenAI / OpenAI Compatible headers
remaining_requests = additional_headers.get("x_ratelimit_remaining_requests", None)
remaining_tokens = additional_headers.get("x_ratelimit_remaining_tokens", None)
additional_headers: Final = _ADDITIONAL_HEADERS.validate_python(
standard_logging_payload["hidden_params"]["additional_headers"] or {}
)
remaining_requests: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_requests"))
remaining_tokens: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_tokens"))
if litellm_overhead_time_ms := standard_logging_payload["hidden_params"].get("litellm_overhead_time_ms"):
_labels = prometheus_label_factory(
@ -3368,7 +3382,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
if remaining_requests:
if remaining_requests is not None:
"""
"model_group",
"api_provider",
@ -3382,7 +3396,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
if remaining_tokens:
if remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
enum_values=enum_values,

289
litellm/interactions/background_cost_polling.py
View file

@ -32,19 +32,24 @@ task to settle. A ``BackgroundSettlementStore`` makes the pending settlement
durable across processes: the create registers the request context that
billing needs (never provider credentials), the settlement is claimed
exactly once through the store, and a delete on any replica rebuilds the
billing context from the store when the poll task is not local. Rows left
unclaimed by a process that died are resumed at startup. The default store is
in-memory, which keeps the SDK and single-process behavior unchanged; the
proxy installs a database-backed one.
billing context from the store when the poll task is not local. Every row
still unsettled when a process starts is resumed there, including one whose
settler died between its claim and its outcome: the database-backed store
treats a claim as a lease that its settler renews until it records the
outcome, so such a row is claimable again once the lease runs out. The
default store is in-memory, which keeps the SDK and single-process behavior
unchanged; the proxy installs a database-backed one.
"""
import asyncio
import itertools
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from openai import APIStatusError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
from pydantic_core import PydanticSerializationError, to_jsonable_python
@ -54,10 +59,12 @@ from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import CredentialLiteLLMParams
from litellm.types.utils import CustomPricingLiteLLMParams
if TYPE_CHECKING:
@ -101,6 +108,7 @@ class PendingBackgroundInteraction:
custom_llm_provider: str
create_context: BackgroundInteractionCreateContext
created_at: datetime
claim_expires_at: datetime | None = None
class BackgroundSettlementStore(Protocol):
@ -112,9 +120,11 @@ class BackgroundSettlementStore(Protocol):
async def claim(self, interaction_id: str) -> bool: ...
async def renew_claim(self, interaction_id: str) -> None: ...
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None: ...
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]: ...
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]: ...
@dataclass(frozen=True, slots=True)
@ -145,10 +155,13 @@ class InMemoryBackgroundSettlementStore:
self._rows[interaction_id] = None
return True
async def renew_claim(self, interaction_id: str) -> None:
return None
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
return None
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return tuple(row for row in self._rows.values() if row is not None)
@ -169,11 +182,15 @@ class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
credentials: CredentialLiteLLMParams = field(default_factory=CredentialLiteLLMParams)
deployment_id: str | None = None
model_group: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
start_delay_seconds: float = 0.0
outcome_write_timeout_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 2
claim_renewal_interval_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 3
store: BackgroundSettlementStore = field(default_factory=InMemoryBackgroundSettlementStore)
resumed: bool = False
@ -184,15 +201,41 @@ FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Await
async def fetch_background_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
credentials: Final = context.credentials
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
api_key=credentials.api_key,
api_base=credentials.api_base,
vertex_project=credentials.vertex_project,
vertex_location=credentials.vertex_location,
vertex_credentials=credentials.vertex_credentials,
**{"no-log": True},
)
_FETCH_CREDENTIAL_FIELDS: Final = ("api_key", "api_base", "vertex_project", "vertex_location", "vertex_credentials")
def _fetch_credentials(request_kwargs: Mapping[str, object]) -> CredentialLiteLLMParams:
try:
return CredentialLiteLLMParams.model_validate(
{field_name: request_kwargs.get(field_name) for field_name in _FETCH_CREDENTIAL_FIELDS}
)
except ValidationError:
verbose_logger.debug(
"Fetching a background interaction with environment credentials: the request's are malformed"
)
return CredentialLiteLLMParams()
_PERMANENT_FETCH_STATUS_CODES: Final = frozenset({400, 401, 403, 404})
def _is_permanent_fetch_error(error: Exception) -> bool:
return isinstance(error, APIStatusError) and error.status_code in _PERMANENT_FETCH_STATUS_CODES
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0 # rebind-ok: the schedule accumulates the time it has already yielded
interval = initial # rebind-ok: the schedule doubles the interval up to the cap
@ -231,6 +274,7 @@ _UNCARRIED_METADATA_KEY: Final = "user_api_key_auth"
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_STRING: Final = TypeAdapter(str)
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECT_DICT: Final = TypeAdapter(dict[str, object])
def _carries(key: str) -> bool:
@ -256,10 +300,36 @@ def _as_datetime(start_time: datetime | float) -> datetime:
return start_time if isinstance(start_time, datetime) else datetime.fromtimestamp(start_time, tz=timezone.utc)
def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]:
"""
The metadata the creating worker's own spend row is built from, with the
request tags computed once, header-derived ones included, because a
rebuilt logging object has no request headers to derive them from.
"""
litellm_params: Final = _OBJECT_DICT.validate_python(logging_obj.model_call_details.get("litellm_params") or {})
return {
**_OBJECT_MAPPING.validate_python(get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)),
**_request_tags(litellm_params),
}
def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
try:
return {
"tags": StandardLoggingPayloadSetup.get_request_tags(
litellm_params=_OBJECT_DICT.validate_python(litellm_params),
proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
)
}
except AttributeError:
verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
return {}
def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext:
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
metadata: Final = _spend_metadata(logging_obj)
litellm_params: Final = _OBJECT_MAPPING.validate_python(logging_obj.litellm_params)
model: Final = logging_obj.model_call_details.get("model")
return BackgroundInteractionCreateContext(
@ -313,6 +383,16 @@ async def _settled_elsewhere(context: BackgroundInteractionPollContext) -> bool:
return False
async def _settled_elsewhere_before_the_window_ends(
context: BackgroundInteractionPollContext, intervals: Iterator[float]
) -> bool:
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return True
return False
async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
"""
Exactly-once gate between every settler of one interaction, on every
@ -326,30 +406,118 @@ async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
return None
async def _record(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> SettlementOutcome:
async def _wrote_outcome(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
try:
await context.store.record_outcome(context.interaction_id, outcome)
except Exception: # noqa: BLE001 # the outcome is an audit trail; the claim already made the settlement exclusive
verbose_logger.exception("Could not record the settlement of background interaction %s", context.interaction_id)
except Exception as e: # noqa: BLE001 # a failed write is retried while the claim still shuts out takeovers
verbose_logger.warning(
"Could not record the settlement of background interaction %s: %s", context.interaction_id, e
)
return False
return True
async def _recorded(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
"""
Retries a failed write for half the claim lease, while no other settler
can take the row over, so a database blip does not leave the row to a
takeover or hold the budget reservation that only a recorded release frees.
"""
retries: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.outcome_write_timeout_seconds,
)
for delay in itertools.chain((0.0,), retries):
await asyncio.sleep(delay)
if await _wrote_outcome(context, outcome):
return True
verbose_logger.error(
"Could not record the settlement of background interaction %s within %ss; the next process to take "
"over its lapsed claim settles it",
context.interaction_id,
context.outcome_write_timeout_seconds,
)
return False
async def _bill(context: BackgroundInteractionPollContext, response: InteractionsAPIResponse) -> SettlementOutcome:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
await _recorded(context, "billed")
return "billed"
async def _release(
context: BackgroundInteractionPollContext, outcome: Literal["released", "unsettled"]
) -> SettlementOutcome:
"""
A release writes no spend row, so a takeover cannot tell whether the
settler before it released. The outcome is recorded first and the
reservation released only once it is, which leaves a takeover with
nothing to release twice.
"""
if await _recorded(context, outcome):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return outcome
async def _renew_claim_until_cancelled(context: BackgroundInteractionPollContext) -> None:
while True:
await asyncio.sleep(context.claim_renewal_interval_seconds)
try:
await context.store.renew_claim(context.interaction_id)
except Exception as e: # noqa: BLE001 # a missed renewal is retried next interval, well inside the lease
verbose_logger.warning(
"Could not renew the settlement claim of background interaction %s: %s", context.interaction_id, e
)
async def _settle(
context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None
) -> SettlementOutcome:
"""
Billing runs every success callback in turn, and a slow one can outlast
the claim lease, so the claim is renewed until the outcome is recorded.
A settler that stops renewing has died, and only then can another take
the row over.
"""
renewal: Final = asyncio.create_task(_renew_claim_until_cancelled(context))
try:
if response is not None and response.status in _TERMINAL_STATUSES and response.usage is not None:
return await _bill(context, response)
return await _release(context, "released")
finally:
renewal.cancel()
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = fetch_background_interaction,
) -> SettlementOutcome | None:
last_response: InteractionsAPIResponse | None = None # rebind-ok: the give-up path settles from the last poll
for interval in _poll_intervals(
await asyncio.sleep(context.start_delay_seconds)
intervals: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
)
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return None
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
if context.resumed and _is_permanent_fetch_error(e):
verbose_logger.warning(
"Stopped fetching resumed background interaction %s: the provider answered %s, so this "
"process cannot read it and leaves it to the replica that created it until the poll window ends",
context.interaction_id,
e,
)
if await _settled_elsewhere_before_the_window_ends(context, intervals):
return None
break
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
@ -363,11 +531,11 @@ async def poll_and_log_background_interaction_cost(
continue
if not claimed:
return None
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
if not await _claim(context):
return None
if last_response is not None and last_response.status in _TERMINAL_STATUSES:
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=last_response))
return await _settle(context, last_response)
if last_response is not None and last_response.status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
@ -379,20 +547,10 @@ async def poll_and_log_background_interaction_cost(
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
"Gave up cost polling for background interaction %s; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "unsettled")
async def _settle_terminal(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> SettlementOutcome:
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=logging_obj, response=response)
return "billed"
await _release_open_budget_reservation(logging_obj=logging_obj)
return "released"
return await _release(context, "unsettled")
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
@ -405,7 +563,9 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") ->
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata: Final = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
budget_reservation: Final = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
@ -518,13 +678,18 @@ class _UnverifiedRegistrationStore:
return False
return await self.local.claim(interaction_id)
async def renew_claim(self, interaction_id: str) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.renew_claim(interaction_id)
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.record_outcome(interaction_id, outcome)
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unclaimed()
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unsettled()
async def _registered_store(
@ -559,8 +724,6 @@ async def maybe_schedule_background_interaction_cost_polling(
logging_obj: Final = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
api_key: Final = create_kwargs.get("api_key")
api_base: Final = create_kwargs.get("api_base")
pending: Final = PendingBackgroundInteraction(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
@ -571,8 +734,7 @@ async def maybe_schedule_background_interaction_cost_polling(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(create_kwargs),
store=await _registered_store(store or _STORE.store, pending),
)
return _track_poll(context, fetch_interaction)
@ -603,10 +765,7 @@ async def _settle_before_delete(
) -> SettlementOutcome | None:
if not await _claim(context):
return None
if response is None:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "released")
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
async def maybe_settle_background_interaction_before_delete(
@ -622,14 +781,11 @@ async def maybe_settle_background_interaction_before_delete(
pending: Final = await _pending(settlement_store, interaction_id)
if pending is None:
return None
api_key: Final = delete_kwargs.get("api_key")
api_base: Final = delete_kwargs.get("api_base")
context: Final = BackgroundInteractionPollContext(
interaction_id=interaction_id,
custom_llm_provider=pending.custom_llm_provider,
logging_obj=_rebuild_logging_obj(pending.create_context),
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(delete_kwargs),
store=settlement_store,
)
try:
@ -644,9 +800,9 @@ async def maybe_settle_background_interaction_before_delete(
return await _settle_before_delete(context, response)
async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
async def _unsettled(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
try:
return await store.unclaimed()
return await store.unsettled()
except Exception: # noqa: BLE001 # an unreadable store at startup leaves its rows for the next boot
verbose_logger.exception("Could not list the unsettled background interactions")
return ()
@ -662,17 +818,36 @@ class PollSchedule:
DEFAULT_POLL_SCHEDULE: Final = PollSchedule()
def _deployment_id(create_context: BackgroundInteractionCreateContext) -> str | None:
model_info: Final = create_context.metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
return deployment_id if isinstance(deployment_id, str) else None
def _model_group(create_context: BackgroundInteractionCreateContext) -> str | None:
model_group: Final = create_context.metadata.get("model_group")
return model_group if isinstance(model_group, str) else None
def _seconds_until(moment: datetime | None) -> float:
if moment is None:
return 0.0
return max((moment - datetime.now(timezone.utc)).total_seconds(), 0.0)
def _resumed_context(
row: PendingBackgroundInteraction, store: BackgroundSettlementStore, schedule: PollSchedule
) -> BackgroundInteractionPollContext:
age_seconds: Final = (datetime.now(timezone.utc) - row.created_at).total_seconds()
return BackgroundInteractionPollContext(
interaction_id=row.interaction_id,
custom_llm_provider=row.custom_llm_provider,
logging_obj=_rebuild_logging_obj(row.create_context),
deployment_id=_deployment_id(row.create_context),
model_group=_model_group(row.create_context),
initial_interval_seconds=schedule.initial_interval_seconds,
max_interval_seconds=schedule.max_interval_seconds,
timeout_seconds=max(schedule.timeout_seconds - age_seconds, schedule.initial_interval_seconds),
timeout_seconds=schedule.timeout_seconds,
start_delay_seconds=_seconds_until(row.claim_expires_at),
store=store,
resumed=True,
)
@ -684,13 +859,15 @@ async def resume_unsettled_background_interactions(
schedule: PollSchedule = DEFAULT_POLL_SCHEDULE,
) -> tuple["asyncio.Task[SettlementOutcome | None]", ...]:
"""
Pick up every settlement no process has claimed, which is what a replica
that died mid-poll leaves behind. Each resumed poll keeps the remaining
share of the original timeout and gets at least one fetch, so a completed
interaction is still billed however late the resume comes.
Pick up every settlement no process has finished, which is what a replica
that died mid-poll or mid-settlement leaves behind. A row whose claim is
still within its lease waits until the lease runs out, then polls like any
other. Each resumed poll gets the full poll window however old its row is,
so a fetch that fails right after a late restart is retried rather than
ending the row as unsettled.
"""
return tuple(
_track_poll(_resumed_context(row, store, schedule), fetch_interaction)
for row in await _unclaimed(store)
for row in await _unsettled(store)
if row.interaction_id not in _ACTIVE_POLLS
)

2
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_json,
write_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
@dataclass(frozen=True, slots=True)

124
litellm/litellm_core_utils/completion_timeout.py
View file

@ -2,15 +2,25 @@
from __future__ import annotations
from collections.abc import Callable
from collections.abc import Callable, Mapping
from enum import Enum
from functools import partial
from types import MappingProxyType
from typing import Final
import httpx
from openai import Timeout as SDKTimeout
from typing_extensions import assert_never
from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
class StreamKind(Enum):
NOT_STREAMED = "not_streamed"
LIVE = "live"
FAKED = "faked"
class CompletionTimeout:
"""Resolves HTTP timeout for ``completion()`` from model vs global settings."""
@ -36,17 +46,91 @@ class CompletionTimeout:
return COMPLETION_HTTP_FALLBACK_SECONDS
return float(global_timeout)
@staticmethod
def _longer_limit(first: float | None, second: float | None) -> float | None:
return None if first is None or second is None else max(first, second)
@staticmethod
def _longer(first: float | httpx.Timeout, second: float | httpx.Timeout) -> float | httpx.Timeout:
if not isinstance(first, httpx.Timeout) and not isinstance(second, httpx.Timeout):
return max(first, second)
first_limits: Final = httpx.Timeout(first)
second_limits: Final = httpx.Timeout(second)
longer: Final = CompletionTimeout._longer_limit
return httpx.Timeout(
connect=longer(first_limits.connect, second_limits.connect),
read=longer(first_limits.read, second_limits.read),
write=longer(first_limits.write, second_limits.write),
pool=longer(first_limits.pool, second_limits.pool),
)
@staticmethod
def for_stream(
stream_kind: StreamKind, request_timeout: float | httpx.Timeout, stream_timeout: float | httpx.Timeout | None
) -> float | httpx.Timeout:
"""
``stream_timeout`` bounds each read of a live stream, so it replaces the request timeout there. A fake
stream is one non-streaming request whose first byte arrives when generation ends, so it takes the longer
of each limit and never fails sooner than the same call would with either one alone.
"""
if stream_timeout is None:
return request_timeout
match stream_kind:
case StreamKind.NOT_STREAMED:
return request_timeout
case StreamKind.LIVE:
return stream_timeout
case StreamKind.FAKED:
return CompletionTimeout._longer(request_timeout, stream_timeout)
case _:
assert_never(stream_kind)
@staticmethod
def as_limit(timeout: object) -> float | httpx.Timeout:
if isinstance(timeout, (httpx.Timeout, SDKTimeout)):
return CompletionTimeout.normalize(timeout)
if isinstance(timeout, (float, int, str)):
return float(timeout)
raise TypeError(f"timeout must be a number or an httpx.Timeout, got {type(timeout).__name__}")
@staticmethod
def stream_request_kwargs(non_stream_timeout: object, stream_timeout: object) -> Mapping[str, object]:
if stream_timeout is None:
return MappingProxyType({"timeout": non_stream_timeout})
return MappingProxyType(
{
"timeout": None
if non_stream_timeout is None
else CompletionTimeout.for_stream(
StreamKind.FAKED,
CompletionTimeout.as_limit(non_stream_timeout),
CompletionTimeout.as_limit(stream_timeout),
),
"stream_timeout": stream_timeout,
}
)
@staticmethod
def _coerce(
timeout: object, custom_llm_provider: str, supports_httpx_timeout: Callable[[str], bool]
) -> float | httpx.Timeout:
limit: Final = CompletionTimeout.as_limit(timeout)
if not isinstance(limit, httpx.Timeout) or supports_httpx_timeout(custom_llm_provider):
return limit
return float(limit.read) if limit.read is not None else COMPLETION_HTTP_FALLBACK_SECONDS
@staticmethod
def resolve(
model_timeout: float | str | httpx.Timeout | SDKTimeout | None,
kwargs: dict,
kwargs: Mapping[str, object],
custom_llm_provider: str,
*,
global_timeout: float | str | None,
supports_httpx_timeout: Callable[[str], bool],
stream_kind: StreamKind,
) -> float | httpx.Timeout:
"""
Resolution order (first non-None wins):
Request timeout resolution order (first non-None wins):
1. ``model_timeout`` (call argument / merged ``litellm_params``)
2. ``kwargs["timeout"]``
@ -54,24 +138,20 @@ class CompletionTimeout:
4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
or 600 when nothing was configured.
A stream then applies ``kwargs["stream_timeout"]`` per :meth:`for_stream`.
Coerce :class:`httpx.Timeout` when the provider does not support it.
"""
resolved: float | str | httpx.Timeout | SDKTimeout
if model_timeout is not None:
resolved = model_timeout
elif kwargs.get("timeout") is not None:
resolved = kwargs["timeout"]
elif kwargs.get("request_timeout") is not None:
resolved = kwargs["request_timeout"]
else:
resolved = CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
if isinstance(resolved, (httpx.Timeout, SDKTimeout)):
normalized: Final = CompletionTimeout.normalize(resolved)
if supports_httpx_timeout(custom_llm_provider):
return normalized
read_timeout: Final = normalized.read
return (
float(read_timeout) if read_timeout is not None else COMPLETION_HTTP_FALLBACK_SECONDS
) # default 10 min timeout
return float(resolved)
coerce: Final = partial(
CompletionTimeout._coerce,
custom_llm_provider=custom_llm_provider,
supports_httpx_timeout=supports_httpx_timeout,
)
non_stream_candidates: Final = (model_timeout, kwargs.get("timeout"), kwargs.get("request_timeout"))
explicit: Final = next((candidate for candidate in non_stream_candidates if candidate is not None), None)
request_timeout: Final = coerce(
explicit if explicit is not None else CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
)
stream_timeout: Final = None if stream_kind is StreamKind.NOT_STREAMED else kwargs.get("stream_timeout")
return CompletionTimeout.for_stream(
stream_kind, request_timeout, None if stream_timeout is None else coerce(stream_timeout)
)

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
items.pop(index)
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
"""
Helper to get litellm metadata for spend tracking
@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
and user_api_key values are in 'metadata'.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in metadata.items():
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
litellm_metadata[key] = value
return litellm_metadata

52
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -1196,6 +1196,9 @@ def _map_sagemaker_exception(
)
_VERTEX_403_STATUS_CODE_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])403(?![0-9A-Za-z])")
def _map_vertex_exception(
*,
model: str,
@ -1206,6 +1209,8 @@ def _map_vertex_exception(
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final[object] = getattr(original_exception, "status_code", None)
body_code: Final[int | None] = _get_body_error_code(error_str)
if "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str:
raise BadRequestError(
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
@ -1252,7 +1257,11 @@ def _map_vertex_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
elif "403" in error_str:
elif (
(not isinstance(status_code, int) or status_code in (403, 500))
and body_code in (None, 403)
and _VERTEX_403_STATUS_CODE_PATTERN.search(error_str)
):
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
@ -1306,7 +1315,7 @@ def _map_vertex_exception(
elif (
isinstance(getattr(original_exception, "status_code", None), int)
and 500 <= original_exception.status_code < 600
and _get_body_error_code(error_str) == 429
and body_code == 429
):
# upstream gateway wraps a 429 inside a 5xx envelope
# e.g. HTTP 500/503 with {"error":{"code":429,...}}.
@ -2413,6 +2422,35 @@ def _map_exception_by_status(
)
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_URL_SCHEME: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*")
_HOST_WITH_OPTIONAL_PORT: Final = re.compile(
r"(?:(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+|localhost|\[[0-9A-Fa-f:.]+\])(?::\d+)?"
)
def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]:
params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS
metadata: Final = as_str_mapping(params.get("metadata"))
deployment_api_base: Final = None if metadata is None else metadata.get("api_base")
if not isinstance(deployment_api_base, str) or not deployment_api_base:
return params
return MappingProxyType({**params, "api_base": deployment_api_base})
def _api_base_without_credentials(api_base: str) -> str:
scheme, separator, _ = api_base.partition("://")
scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else ""
authority_and_rest: Final = api_base[len(scheme_prefix) :]
authority: Final = re.split(r"[/?#]", authority_and_rest, maxsplit=1)[0]
after_authority: Final = authority_and_rest[len(authority) :]
host: Final = authority.rpartition("@")[2]
if "@" in after_authority and _HOST_WITH_OPTIONAL_PORT.fullmatch(host) is None:
return ""
path: Final = re.split(r"[?#]", after_authority, maxsplit=1)[0]
return f"{scheme_prefix}{host}{path}"
def _is_guardrail_block(original_exception: Exception) -> bool:
from litellm.integrations.custom_guardrail import is_guardrail_intervention
@ -2483,8 +2521,9 @@ def exception_type(
if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0:
exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
if _api_base:
extra_information += f"\nAPI Base: `{_api_base}`"
shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else ""
if shown_api_base:
extra_information += f"\nAPI Base: `{shown_api_base}`"
if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False:
extra_information += f"\nMessages: `{messages}`"
@ -2512,7 +2551,8 @@ def exception_type(
################################################################################
if (
"Request Timeout Error" in error_str
isinstance(original_exception, httpx.TimeoutException)
or "Request Timeout Error" in error_str
or "Request timed out" in error_str
or "Timed out generating response" in error_str
or "The read operation timed out" in error_str
@ -2520,7 +2560,7 @@ def exception_type(
exception_mapping_worked = True
raise Timeout(
message=f"APITimeoutError - Request timed out. Error_str: {error_str}",
message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,

15
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_auth_type",
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)
@ -141,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
if metadata is None:
return None
model_info: Final = metadata.get("model_info")
if model_info:
model_info_mapping: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
Mapping[str, object], model_info
)
return cast( # cast-ok: model metadata values are caller-provided
str | None, model_info_mapping.get("base_model")
)
return None
if not model_info:
return None
base_model: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
Mapping[str, object], model_info
).get("base_model")
return base_model if isinstance(base_model, str) else None
_get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata

125
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -17,6 +17,7 @@ import random
import sys
import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
@ -24,11 +25,11 @@ from importlib.resources import files
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm import verbose_logger
from litellm.constants import (
@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
class _S3Body(Protocol):
def read(self) -> bytes: ...
class _S3GetObjectResponse(TypedDict):
Body: ReadOnly[_S3Body]
ETag: NotRequired[ReadOnly[str]]
class _S3Client(Protocol):
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
def _default_s3_client(timeout: int) -> _S3Client:
import boto3
from botocore.config import Config
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
_S3Client,
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
"s3",
region_name=os.getenv("AWS_REGION_NAME") or None,
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
),
)
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@ -283,7 +311,49 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
dict[str, object], parsed
)
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
try:
return urllib.parse.urlsplit(url)
except ValueError as error:
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
try:
client: Final = _default_s3_client(timeout)
response: Final = client.get_object(Bucket=bucket, Key=key)
content: Final = response["Body"].read()
except (BotoCoreError, ClientError) as error:
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
return _classify_fetch_content(content, url, response.get("ETag"))
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -292,6 +362,11 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return _attempt_fetch_s3(parts, url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -307,15 +382,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
try:
parsed: Final = response.json()
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
return ModelCostMapReloaded(
model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
)
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
def _next_retry_wait(
@ -490,22 +557,39 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
return _cost_map_source_info.loaded_at
def _expand_model_aliases(model_cost: dict) -> dict:
class AliasedCostEntry(dict[str, object]):
__slots__ = ("names",)
def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None:
super().__init__(entry)
self.names: tuple[str, ...] = names
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
return MappingProxyType(
{
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
for owner in dict.fromkeys(alias_owners.values())
}
)
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Each alias gets a reference to the **same** dict object as the canonical
entry (zero memory overhead). The ``aliases`` key is removed from the
entry so downstream code never sees it.
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
"""
aliases_to_add: Final[dict[str, dict]] = {}
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: list | None = model_info.get("aliases")
aliases: object = model_info.get("aliases")
if aliases is None:
continue
keys_with_aliases.append(model_name)
@ -527,7 +611,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
if alias in aliases_to_add:
if alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"was already claimed by another entry — skipping.",
@ -535,13 +619,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
aliases_to_add[alias] = model_info # same dict reference
alias_owners[alias] = model_name
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
model_cost.update(aliases_to_add)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
return model_cost

10
litellm/litellm_core_utils/get_provider_specific_headers.py
View file

@ -9,6 +9,7 @@ class ProviderSpecificHeaderUtils:
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
custom_llm_provider: str | None,
native_anthropic_messages: bool = False,
) -> Mapping[str, str]:
"""
Get the provider specific headers for the given custom llm provider.
@ -17,7 +18,8 @@ class ProviderSpecificHeaderUtils:
carries its own comma-separated provider list, so headers that are safe for several
providers and headers that are safe for exactly one can travel on the same request
without sharing a scope. Entries whose provider list does not contain
`custom_llm_provider` contribute nothing.
`custom_llm_provider` contribute nothing, and entries marked
`native_anthropic_messages_only` contribute only to a provider's native Messages route.
Returns:
Dict: The provider specific headers for the given custom llm provider
@ -31,6 +33,12 @@ class ProviderSpecificHeaderUtils:
matched_headers: Final = {}
for scoped_header in scoped_headers:
if (
"native_anthropic_messages_only" in scoped_header
and scoped_header["native_anthropic_messages_only"]
and not native_anthropic_messages
):
continue
stored_providers = scoped_header.get("custom_llm_provider", "")
provider_list = [p.strip() for p in stored_providers.split(",")]
if custom_llm_provider in provider_list:

5
litellm/litellm_core_utils/hidden_params.py
View file

@ -77,6 +77,11 @@ def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
return _as_hidden_params_mapping(hidden_params)
def served_from_cache(obj: object) -> bool:
hidden_params: Final = get_hidden_params(obj)
return hidden_params is not None and hidden_params.get("cache_hit") is True
def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
if isinstance(obj, dict):
obj[_HIDDEN_PARAMS_ATTR] = hidden_params

607
litellm/litellm_core_utils/litellm_logging.py
View file

File diff suppressed because it is too large Load diff

175
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -2,12 +2,13 @@
Helper utilities for tracking the cost of built-in tools.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final, Literal, cast
from pydantic import ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
@ -30,20 +31,36 @@ from litellm.types.utils import (
)
def _output_item_field(output_item: object, field: str) -> object:
if isinstance(output_item, dict):
fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) # cast-ok: narrowed by isinstance
return fields.get(field)
return getattr(output_item, field, None)
def _output_item_type(output_item: object) -> str | None:
item_type: Final = output_item.get("type") if isinstance(output_item, dict) else getattr(output_item, "type", None)
item_type: Final = _output_item_field(output_item, "type")
return item_type if isinstance(item_type, str) else None
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = getattr(response_object, "tool_usage", None)
if tool_usage is None:
def _responses_tool_usage(response_object: ResponsesAPIResponse) -> ResponsesToolUsage | None:
top_level: Final = getattr(response_object, "tool_usage", None)
raw: Final = (
top_level if top_level is not None else getattr(getattr(response_object, "usage", None), "tool_usage", None)
)
if raw is None:
return None
try:
web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
return ResponsesToolUsage.model_validate(raw)
except ValidationError:
return None
return None if web_search is None else web_search.num_requests
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None:
return None
return None if tool_usage.web_search is None else tool_usage.web_search.num_requests
def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
@ -87,31 +104,48 @@ class StandardBuiltInToolCostTracking:
usage=usage,
)
image_generation_cost: Final = StandardBuiltInToolCostTracking._handle_image_generation_cost(
response_object=response_object,
custom_llm_provider=custom_llm_provider,
)
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
)
)
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost(
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_file_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
)
# Handle Azure assistant features
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
@staticmethod
@ -210,6 +244,87 @@ class StandardBuiltInToolCostTracking:
)
return max(count, 1)
@staticmethod
def _image_generation_tool_model(response_object: ResponsesAPIResponse) -> str | None:
tools: Final[Sequence[object]] = cast(Sequence[object], response_object.tools or ()) # cast-ok: dict tool union
for tool in tools:
if _output_item_field(tool, "type") != "image_generation":
continue
if isinstance(model := _output_item_field(tool, "model"), str) and model:
return model
return None
@staticmethod
def _image_generation_call_cost(output_item: object, tool_model: str, custom_llm_provider: str | None) -> float:
from litellm.cost_calculator import (
default_image_cost_calculator, # pyright: ignore[reportUnknownVariableType] # optional_params param is untyped
)
quality: Final = _output_item_field(output_item, "quality")
size: Final = _output_item_field(output_item, "size")
try:
return default_image_cost_calculator(
model=tool_model,
custom_llm_provider=custom_llm_provider,
quality=quality if isinstance(quality, str) and quality != "auto" else None,
n=1,
size=size if isinstance(size, str) and size != "auto" else None,
)
except Exception as e: # noqa: BLE001 # pricing helpers raise bare Exception for unmapped models; bill 0.0
verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e)
return 0.0
@staticmethod
def _image_generation_token_cost(
response_object: ResponsesAPIResponse, tool_model: str, custom_llm_provider: str | None
) -> float | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0:
return None
try:
model_info: Final = litellm.get_model_info(model=tool_model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back
verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e)
return None
image_gen: Final = tool_usage.image_gen
input_details: Final = image_gen.input_tokens_details
output_details: Final = image_gen.output_tokens_details
token_cost: Final = (
(input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0)
+ (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0)
+ (output_details.image_tokens if output_details else 0)
* (model_info.get("output_cost_per_image_token") or 0)
+ (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0)
)
return token_cost if token_cost > 0 else None
@staticmethod
def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float:
if not isinstance(response_object, ResponsesAPIResponse):
return 0.0
output: Final[Sequence[object]] = cast(Sequence[object], response_object.output) # cast-ok: isinstance-narrowed
completed_items: Final = tuple(
output_item
for output_item in output
if _output_item_type(output_item) == "image_generation_call"
and _output_item_field(output_item, "status") == "completed"
)
if not completed_items:
return 0.0
tool_model: Final = StandardBuiltInToolCostTracking._image_generation_tool_model(response_object)
if tool_model is None:
verbose_logger.debug("Responses API image_generation_call has no tools[].model to price, billing 0.0")
return 0.0
token_cost: Final = StandardBuiltInToolCostTracking._image_generation_token_cost(
response_object, tool_model, custom_llm_provider
)
if token_cost is not None:
return token_cost
return sum(
StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, tool_model, custom_llm_provider)
for output_item in completed_items
)
@staticmethod
def _handle_file_search_cost(
model: str,
@ -445,9 +560,12 @@ class StandardBuiltInToolCostTracking:
return False
elif isinstance(response_object, ResponsesAPIResponse):
# response api explicitly includes web_search_call in the output
return StandardBuiltInToolCostTracking.response_includes_output_type(
if StandardBuiltInToolCostTracking.response_includes_output_type(
response_object=response_object, output_type="web_search_call"
)
):
return True
reported: Final = _reported_web_search_requests(response_object)
return reported is not None and reported > 0
elif usage is not None:
if get_web_search_requests_from_usage(usage) is not None or (
hasattr(usage, "prompt_tokens_details")
@ -740,9 +858,9 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
if "web_search_options" in kwargs:
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
return WebSearchOptions(**web_search_options)
web_search_options: Final = kwargs.get("web_search_options")
if isinstance(web_search_options, dict):
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search_preview"
@ -758,10 +876,9 @@ class StandardBuiltInToolCostTracking:
_get_web_search_options = get_web_search_options
@staticmethod
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
if "tools" in kwargs:
return cast(list[object], kwargs.get("tools", []))
return None
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
tools: Final = kwargs.get("tools")
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
@staticmethod
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

72
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -251,14 +251,7 @@ async def convert_to_streaming_response_async(
setattr(
model_response_object,
"usage",
Usage(
completion_tokens=response_object["usage"].get("completion_tokens", 0),
prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
total_tokens=response_object["usage"].get("total_tokens", 0),
completion_tokens_details=_cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
),
),
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -345,12 +338,10 @@ def convert_to_streaming_response(
model_response_object.choices = choice_list
if "usage" in response_object and response_object["usage"] is not None:
setattr(model_response_object, "usage", Usage())
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage.completion_tokens_details = _cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
setattr(
model_response_object,
"usage",
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -414,13 +405,28 @@ class _CachedMessageProviderFields(TypedDict, total=False):
class _CachedUsageDetails(TypedDict, total=False):
completion_tokens: ReadOnly[object]
prompt_tokens: ReadOnly[object]
total_tokens: ReadOnly[object]
completion_tokens_details: ReadOnly[object]
prompt_tokens_details: ReadOnly[object]
cache_read_input_tokens: ReadOnly[object]
cache_creation_input_tokens: ReadOnly[object]
class _PromptCacheTokenCounts(TypedDict, total=False):
cache_read_input_tokens: ReadOnly[int]
cache_creation_input_tokens: ReadOnly[int]
_CACHED_MESSAGE_PROVIDER_FIELDS: Final = TypeAdapter(_CachedMessageProviderFields)
_CACHED_USAGE_DETAILS: Final = TypeAdapter(_CachedUsageDetails)
_FIELD_MAP: Final = TypeAdapter(dict[str, object])
_COMPLETION_TOKENS_DETAILS: Final = TypeAdapter(CompletionTokensDetailsWrapper)
_PROMPT_TOKENS_DETAILS: Final = TypeAdapter(PromptTokensDetailsWrapper)
_PROMPT_CACHE_TOKEN_COUNTS: Final = TypeAdapter(_PromptCacheTokenCounts)
_TOKEN_COUNT: Final = TypeAdapter(int)
_PROMPT_CACHE_TOKEN_FIELDS: Final = ("cache_read_input_tokens", "cache_creation_input_tokens")
def _cached_provider_specific_fields(message: _CachedMessageProviderFields) -> Mapping[str, object] | None:
@ -438,6 +444,44 @@ def _cached_completion_tokens_details(usage: _CachedUsageDetails) -> CompletionT
return None
def _cached_prompt_tokens_details(usage: _CachedUsageDetails) -> PromptTokensDetailsWrapper | None:
details: Final = usage.get("prompt_tokens_details")
if not isinstance(details, dict):
return None
try:
return _PROMPT_TOKENS_DETAILS.validate_python(details)
except ValidationError:
return None
def _cached_token_count(count: object) -> int | None:
try:
return _TOKEN_COUNT.validate_python(count)
except ValidationError:
return None
def _cached_prompt_cache_token_counts(usage: _CachedUsageDetails) -> _PromptCacheTokenCounts:
return _PROMPT_CACHE_TOKEN_COUNTS.validate_python(
{
field: count
for field in _PROMPT_CACHE_TOKEN_FIELDS
if (count := _cached_token_count(usage.get(field))) is not None
}
)
def _replayed_usage(usage: _CachedUsageDetails) -> Usage:
return Usage(
completion_tokens=_cached_token_count(usage.get("completion_tokens")) or 0,
prompt_tokens=_cached_token_count(usage.get("prompt_tokens")) or 0,
total_tokens=_cached_token_count(usage.get("total_tokens")) or 0,
completion_tokens_details=_cached_completion_tokens_details(usage),
prompt_tokens_details=_cached_prompt_tokens_details(usage),
**_cached_prompt_cache_token_counts(usage),
)
def handle_invalid_parallel_tool_calls(
tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall],
) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None:

13
litellm/litellm_core_utils/llm_response_utils/get_headers.py
View file

@ -1,8 +1,11 @@
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import PROVIDER_REQUEST_ID_HEADERS
_HEADERS: Final = TypeAdapter(Mapping[str, str])
_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
@ -16,6 +19,16 @@ def get_provider_request_id(headers: object) -> str | None:
return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None)
def provider_request_id_headers(headers: Mapping[str, object] | None) -> httpx.Headers | None:
"""The provider request id headers in `headers` whose values are ASCII strings, or None when there are none."""
request_id_headers: Final = tuple(
(name, value)
for name, value in (headers or {}).items()
if name.lower() in PROVIDER_REQUEST_ID_HEADERS and isinstance(value, str) and value.isascii()
)
return httpx.Headers(request_id_headers) if request_id_headers else None
def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
"""

19
litellm/litellm_core_utils/model_param_helper.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import ClassVar, Final
@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.rerank import RerankRequest
def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None:
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
"""
if "reasoningSummary" in optional_params:
return optional_params["reasoningSummary"]
if "reasoning_summary" in optional_params:
return optional_params["reasoning_summary"]
extra_body: Final = optional_params.get("extra_body")
if isinstance(extra_body, dict):
if "reasoningSummary" in extra_body:
return extra_body["reasoningSummary"]
if "reasoning_summary" in extra_body:
return extra_body["reasoning_summary"]
return None
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

125
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,7 +9,8 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby, islice
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
from pathlib import Path
from types import MappingProxyType
@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str,
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
@ -1894,6 +2012,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
"""Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
"""
Filters a value from a dictionary

221
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -9,10 +9,11 @@ import xml.etree.ElementTree as ET
from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, TypeAlias, TypedDict, cast, overload
from typing import Annotated, Any, Final, TypeAlias, TypedDict, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, JsonValue, TypeAdapter
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
from pydantic_core import ErrorDetails, PydanticCustomError
import litellm
import litellm.types
@ -2582,6 +2583,48 @@ def _with_native_redacted_data(
_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool:
return (
message_field(message, "role") in ("tool", "function")
and message_field(message, "tool_call_id") not in replaced_server_tool_answers
)
def _tool_result_later_in_turn(
messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str]
) -> tuple[bool, ...]:
from_the_end: Final = itertools.accumulate(
reversed(messages),
lambda later, message: (
message_field(message, "role") in ("user", "tool", "function")
and (later or _client_tool_result(message, replaced_server_tool_answers))
),
initial=False,
)
return tuple(from_the_end)[::-1][1:]
def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam:
placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER)
add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message)
return placeholder
def _cache_breakpoint_placeholder(
message: AllMessageValues, tool_result_follows: bool
) -> tuple[AnthropicMessagesTextParam, ...]:
if tool_result_follows or not message_field(message, "cache_control"):
return ()
return (_empty_text_placeholder(message),)
def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None:
empty_user_message: Final = next(
(message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None
)
return None if empty_user_message is None else _empty_text_placeholder(empty_user_message)
def anthropic_messages_pt(
messages: list[AllMessageValues],
model: str,
@ -2636,6 +2679,7 @@ def anthropic_messages_pt(
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers)
msg_i = 0
while msg_i < len(messages):
user_content: list[AnthropicMessagesUserMessageValues] = []
@ -2648,8 +2692,12 @@ def anthropic_messages_pt(
ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage
) = messages[msg_i]
if user_message_types_block["role"] == "user":
if isinstance(user_message_types_block["content"], list):
for m in user_message_types_block["content"]:
if not (user_block_content := user_message_types_block.get("content")):
user_content.extend(
_cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i])
)
elif isinstance(user_block_content, list):
for m in user_block_content:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
@ -2720,10 +2768,10 @@ def anthropic_messages_pt(
_file_content_element,
)
)
elif isinstance(user_message_types_block["content"], str):
elif isinstance(user_block_content, str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",
"text": user_message_types_block["content"],
"text": user_block_content,
}
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_text_element,
@ -2747,6 +2795,8 @@ def anthropic_messages_pt(
if user_content:
new_messages.append({"role": "user", "content": user_content})
elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None:
new_messages.append({"role": "user", "content": [placeholder]})
# Track unique tool IDs in this merge block to avoid duplication
unique_tool_ids: set[str] = set()
@ -4253,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
"""Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
if message["role"] != "assistant":
return message
blocks: Final = message["content"]
last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
return BedrockMessageBlock(
role="assistant",
content=[
_rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
],
)
def _ensure_document_messages_have_text(
contents: list[BedrockMessageBlock],
) -> list[BedrockMessageBlock]:
@ -4844,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(
_rename_duplicate_bedrock_document_names(contents)
)
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
@staticmethod
def translate_thinking_blocks_to_reasoning_content_blocks(
@ -5253,7 +5326,10 @@ def _bedrock_converse_messages_pt(
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@ -5428,14 +5504,14 @@ def _bedrock_tools_pt(
description = name
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
# Expand $ref references in parameters using the definitions
# Note: We don't pre-flatten defs as that causes exponential memory growth
# with circular references (see issue #19098). unpack_defs handles nested
# refs recursively and correctly detects/skips circular references.
unpack_defs(parameters, defs_copy)
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types):
parameters["type"] = "object"
tool_block = cast(
BedrockToolBlock,
@ -5457,6 +5533,11 @@ def _bedrock_tools_pt(
return tool_block_list
def bedrock_tool_blocks(tools: Sequence[Mapping[str, object]], model: str | None) -> tuple[BedrockToolBlock, ...]:
"""Bedrock Converse ``toolConfig.tools`` entries for OpenAI or Anthropic tool definitions."""
return tuple(_bedrock_tools_pt([dict(tool) for tool in tools], model=model))
def _is_system_message(message: Mapping[str, object]) -> bool:
role: Final = message.get("role")
return isinstance(role, str) and "system" in role
@ -5516,7 +5597,7 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict) -> str:
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
@ -5532,11 +5613,8 @@ def response_schema_prompt(model: str, response_schema: dict) -> str:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return custom_prompt(
role_dict=custom_prompt_details["roles"],
initial_prompt_value=custom_prompt_details["initial_prompt_value"],
final_prompt_value=custom_prompt_details["final_prompt_value"],
messages=response_schema_as_message,
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
return default_response_schema_prompt(response_schema=response_schema)
@ -5556,8 +5634,95 @@ def default_response_schema_prompt(response_schema: dict) -> str:
# Custom prompt template
def _mapping_or_none(value: object) -> object:
if value is None or issubclass(type(value), Mapping):
return value
verbose_logger.warning(
"Ignoring a custom prompt template roles value of type %s because it is not a mapping, "
"so those messages render without role markers",
type(value).__name__,
)
return None
class CustomPromptRole(BaseModel):
model_config = ConfigDict(frozen=True)
pre_message: str | None = None
post_message: str | None = None
class CustomPromptRoles(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Mapping[str, Annotated[CustomPromptRole | None, BeforeValidator(_mapping_or_none)]]
class CustomPromptTemplate(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Annotated[Mapping[object, object] | None, BeforeValidator(_mapping_or_none)] = None
initial_prompt_value: str | None = None
final_prompt_value: str | None = None
@model_validator(mode="after")
def _sets_a_rendered_field(self) -> "CustomPromptTemplate":
if self.roles is None and self.initial_prompt_value is None and self.final_prompt_value is None:
raise PydanticCustomError(
"empty_custom_prompt_template",
"it sets none of roles, initial_prompt_value, or final_prompt_value",
)
return self
def render_custom_prompt_template(
template: Mapping[str, object],
messages: Sequence[Mapping[str, object]],
model: str,
llm_provider: str,
) -> str:
try:
return _render_custom_prompt_template(template=template, messages=messages)
except ValidationError as error:
raise litellm.APIConnectionError(
message=f"The custom prompt template for {model} is invalid: {_describe_template_errors(error)}",
model=model,
llm_provider=llm_provider,
) from error
def _describe_template_errors(error: ValidationError) -> str:
return "; ".join(
_describe_template_error(detail) for detail in error.errors(include_url=False, include_input=False)
)
def _describe_template_error(detail: ErrorDetails) -> str:
location: Final = ".".join(str(part) for part in detail["loc"])
return f"{location}: {detail['msg']}" if location else detail["msg"]
def _render_custom_prompt_template(template: Mapping[str, object], messages: Sequence[Mapping[str, object]]) -> str:
validated: Final = CustomPromptTemplate.model_validate(template)
message_roles: Final = frozenset(message.get("role") for message in messages)
template_roles: Final = validated.roles or {}
used_roles: Final = CustomPromptRoles.model_validate(
{"roles": {role: entry for role, entry in template_roles.items() if role in message_roles}}
)
return custom_prompt(
role_dict={
role: {"pre_message": entry.pre_message or "", "post_message": entry.post_message or ""}
for role, entry in used_roles.roles.items()
if entry is not None
},
initial_prompt_value=validated.initial_prompt_value or "",
final_prompt_value=validated.final_prompt_value or "",
messages=list(messages),
)
def custom_prompt(
role_dict: dict,
role_dict: Mapping[str, Mapping[str, str]],
messages: list,
initial_prompt_value: str = "",
final_prompt_value: str = "",
@ -6006,3 +6171,25 @@ def resolve_structured_messages(
if structured:
return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
return None
def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
if is_flat_responses_tool(tool):
chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in a flat Responses tool shape
)
return tuple(chat_tools)
anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in an Anthropic or chat tool shape
)
return tuple(anthropic_tools)

15
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool:
def _renders(message: object, answered: frozenset[str]) -> bool:
"""Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
A tool message always becomes a ``tool_result`` and a user message with string
content always becomes a text block (empty text gets a placeholder). A user list
renders only through parts of a type the converter emits; ``None``, an empty list,
and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
A tool message always becomes a ``tool_result``, and a user message with string or
missing content (``None``, an empty list) always puts its turn on the wire, an empty
turn getting a placeholder. A user list renders only through parts of a type the
converter emits, so a list of only other parts vanishes. Assistant messages follow
``_assistant_renders``.
"""
role: Final = message_field(message, "role")
if role in _TOOL_ROLES:
@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool:
if role == "assistant":
return _assistant_renders(message, answered)
content: Final = message_field(message, "content")
return isinstance(content, str) or any(
message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
return (
isinstance(content, str)
or not content
or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content))
)

247
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,17 +1,20 @@
import asyncio
import json
import traceback
from collections.abc import Coroutine, Mapping, Sequence
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from typing_extensions import ReadOnly
import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@ -98,6 +101,16 @@ class ScopedWebSocket(Protocol):
def scope(self) -> _ASGIScope: ...
@runtime_checkable
class _ReplayScopeWebSocket(Protocol):
@property
def scope(self) -> MutableMapping[str, object]: ...
def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
class _ClientWebSocket(ScopedWebSocket, Protocol):
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
@ -116,10 +129,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
return json.loads(payload)
_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
{"type": "server_vad", "create_response": False}
)
_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
def _json_object(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
return _json_object(container.get(key))
def _guardrail_event_id() -> str:
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
def _error_event_id(event: Mapping[str, object]) -> str | None:
error: Final = _mapping_field(event, "error")
caused_by: Final = None if error is None else error.get("event_id")
return caused_by if isinstance(caused_by, str) else None
_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
client_turn_detection: Final = _json_object(turn_detection)
if client_turn_detection is None:
return dict(_GUARDRAIL_TURN_DETECTION_GATE)
return {**client_turn_detection, "create_response": False}
def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
audio: Final = _mapping_field(session, "audio")
if audio is None:
return {}
audio_input: Final = _mapping_field(audio, "input")
if audio_input is None or "turn_detection" not in audio_input:
return {}
gated_input: Final = {
**audio_input,
"turn_detection": gate(audio_input["turn_detection"]),
}
return {"audio": {**audio, "input": gated_input}}
def _with_create_response_disabled(
session: Mapping[str, object], gate: _TurnDetectionGate
) -> Mapping[str, object] | None:
"""``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
nested_part: Final = _nested_turn_detection_part(session, gate)
if not flat_part and not nested_part:
return None
return {**session, **flat_part, **nested_part}
def _gate_session_update(
session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
) -> Mapping[str, object] | None:
"""The session to forward under a transcript guardrail, or None when the frame needs no change."""
client_session: Final = _json_object(session)
if client_session is None:
return None
gated: Final = _with_create_response_disabled(client_session, gate)
if gated is not None or not inject_gate_when_absent:
return gated
return {**client_session, "turn_detection": gate(None)}
class RealtimeEventNormalizer(Protocol):
def should_drop(self, event: object) -> bool: ...
def normalize(self, event: dict) -> dict: ...
def patch_outgoing_session(self, session: dict) -> dict: ...
def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
@property
def deletes_conversation_items(self) -> bool: ...
class RealtimeUsageMeter(Protocol):
@ -154,6 +244,7 @@ class RealTimeStreaming:
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -197,6 +288,8 @@ class RealTimeStreaming:
# Track whether we have already sent the guardrail turn-detection update
# that disables provider auto-response for transcription guardrails.
self._guardrail_turn_detection_update_sent: bool = False
self._session_gate_sent: bool = False
self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
# Deferred Gemini Live setup: Pipecat may stream audio before session.update.
# Buffer client audio until the backend acknowledges setup (setupComplete).
self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@ -213,6 +306,12 @@ class RealTimeStreaming:
self._is_transcription_session: bool = force_transcription_model is not None
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
self._event_normalizer = event_normalizer
self._gate_turn_detection: Final[_TurnDetectionGate] = (
_turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
)
self._deletes_conversation_items: Final = (
event_normalizer is None or event_normalizer.deletes_conversation_items
)
self._usage_meter = usage_meter
@property
@ -734,20 +833,37 @@ class RealTimeStreaming:
except (json.JSONDecodeError, TypeError):
return
def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
session: Final[dict[str, object]] = (
{"turn_detection": turn_detection}
if self._backend_uses_beta_protocol
else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
)
return {"type": "session.update", "session": session}
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
turn_detection: Final[dict[str, str | bool]] = {
"type": "server_vad",
"create_response": False,
}
if self._backend_uses_beta_protocol:
session: dict[str, object] = {"turn_detection": turn_detection}
else:
session = {
"type": "realtime",
"audio": {"input": {"turn_detection": turn_detection}},
}
return json.dumps({"type": "session.update", "session": session})
return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
async def _send_guardrail_cancel(self) -> None:
event_id: Final = _guardrail_event_id()
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
return _error_event_id(event) in self._guardrail_cancel_event_ids
async def _send_session_gate_once(self) -> None:
"""Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
if self._session_gate_sent or not self._should_disable_vad_auto_response():
return
self._session_gate_sent = True
await self._send_to_backend(self._make_disable_auto_response_message())
async def _maybe_send_guardrail_turn_detection_update(self) -> None:
"""Disable provider auto-response once when transcription guardrails are enabled."""
@ -921,7 +1037,7 @@ class RealTimeStreaming:
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._send_to_backend(json.dumps({"type": "response.cancel"}))
await self._withdraw_blocked_turn(item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1051,10 +1167,7 @@ class RealTimeStreaming:
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
item_id=cast(str | None, event.get("item_id")),
)
blocked = await self.run_realtime_guardrails(cast(str, transcript))
if not blocked and self._should_disable_vad_auto_response():
await self._send_to_backend(json.dumps({"type": "response.create"}))
continue
@ -1086,7 +1199,7 @@ class RealTimeStreaming:
if event_type == "session.created" and self._should_disable_vad_auto_response():
self.store_message(event_obj)
await self.websocket.send_text(self._event_to_client_json(event_obj))
await self._send_to_backend(self._make_disable_auto_response_message())
await self._send_session_gate_once()
return True
if event_type == "conversation.item.input_audio_transcription.completed":
@ -1140,6 +1253,10 @@ class RealTimeStreaming:
if self._usage_meter is not None:
self._usage_meter.observe_backend_event(event)
if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
@ -1346,6 +1463,13 @@ class RealTimeStreaming:
return item
async def _receive_client_message(self) -> str | None:
refused_opening_message: Final = (
None
if self._replay_scope is None
else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
)
if isinstance(refused_opening_message, str):
return refused_opening_message
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone
@ -1472,19 +1596,21 @@ class RealTimeStreaming:
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
):
session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
if isinstance(session, dict):
existing_td = session.get("turn_detection")
if not isinstance(existing_td, dict):
existing_td = {}
existing_td["create_response"] = False
session["turn_detection"] = existing_td
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
and (
gated_first := _gate_session_update(
client_event.get("session", _EMPTY_SESSION),
self._gate_turn_detection,
inject_gate_when_absent=True,
)
)
is not None
):
msg_obj["session"] = gated_first
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
## GUARDRAIL: Force ``create_response`` to False in any
# client-provided ``turn_detection`` so a later
@ -1492,52 +1618,25 @@ class RealTimeStreaming:
# and bypass the transcription guardrail after the
# initial disable. Covers both the flat beta key and the
# nested GA ``audio.input.turn_detection`` shape, since
# the GA remap below also accepts either form. Skipped
# when the injection block above already ran for this
# message, to avoid redundant double-serialization.
# the GA remap below also accepts either form. A later
# update with no turn_detection is forwarded as is: OpenAI
# keeps the earlier disable for an unset field, and xAI
# falls back to its manual default, which never auto-responds.
if (
msg_type == "session.update"
and not guardrail_turn_detection_injected
and self._should_disable_vad_auto_response()
and (
gated_later := _gate_session_update(
client_event.get("session"),
self._gate_turn_detection,
inject_gate_when_absent=False,
)
)
is not None
):
session = client_event.get("session")
if isinstance(session, dict):
td_overridden = False
flat_td = session.get("turn_detection")
flat_td_present = flat_td is not None
if flat_td_present:
if not isinstance(flat_td, dict):
flat_td = {}
if flat_td.get("create_response") is not False:
flat_td["create_response"] = False
session["turn_detection"] = flat_td
td_overridden = True
nested_td_present = False
audio = session.get("audio")
if isinstance(audio, dict):
audio_input = audio.get("input")
if isinstance(audio_input, dict):
nested_td = audio_input.get("turn_detection")
if nested_td is not None:
nested_td_present = True
if not isinstance(nested_td, dict):
nested_td = {}
if nested_td.get("create_response") is not False:
nested_td["create_response"] = False
audio_input["turn_detection"] = nested_td
td_overridden = True
# Symmetric with the first-update injection block:
# if the client omitted turn_detection entirely on
# a subsequent session.update, still inject the
# ``create_response: False`` override so the
# transcription guardrail cannot be re-enabled by
# any downstream merge that drops the original
# disable.
if not flat_td_present and not nested_td_present:
session["turn_detection"] = {"create_response": False}
td_overridden = True
if td_overridden:
message = json.dumps(msg_obj)
msg_obj["session"] = gated_later
message = json.dumps(msg_obj)
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
@ -1590,6 +1689,8 @@ class RealTimeStreaming:
# would permanently disable the injection if ``_send_to_backend``
# raised — neither this loop nor
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
sent = await self._send_to_backend(message)
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True

4
litellm/litellm_core_utils/served_output_texts.py
View file

@ -126,13 +126,13 @@ def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int)
content
for chunk in chunks
for choice in chunk.choices
if choice.index == index and isinstance(content := choice.delta.content, str)
if (choice.index or 0) == index and isinstance(content := choice.delta.content, str)
)
return "".join(contents) if contents else None
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0)
choice_count: Final = max(((choice.index or 0) + 1 for chunk in chunks for choice in chunk.choices), default=0)
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
return texts if any(text is not None for text in texts) else None

140
litellm/litellm_core_utils/streaming_handler.py
View file

@ -45,9 +45,9 @@ from litellm.types.utils import (
)
from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError
from ..exceptions import OpenAIError, Timeout
from .core_helpers import map_finish_reason, process_response_headers
from .exception_mapping_utils import exception_type
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -88,6 +88,16 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
setattr(response, key, value)
def _is_transport_timeout(error: BaseException) -> bool:
return isinstance(error, httpx.TimeoutException) or (
isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
)
def _skips_stream_fallback(status_code: int | None) -> bool:
return status_code is not None and 400 <= status_code < 500 and status_code != 429
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -242,6 +252,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._failure_logging: asyncio.Task[None] | None = None
self._closed = False
self._stream_created_time: float = time.time()
@ -269,6 +280,7 @@ class CustomStreamWrapper:
self.holding_chunk = ""
self.complete_response = ""
self.response_uptil_now = ""
self._emitted_disqualifying_content = False
_model_info: Final[dict] = litellm_params.model_info or {}
_api_base: Final = get_api_base(
@ -1126,31 +1138,24 @@ class CustomStreamWrapper:
"""
if self.merge_reasoning_content_in_choices is True:
reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
if reasoning_content:
if self.sent_first_thinking_block is False:
# Ensure content is not None before concatenation
if model_response.choices[0].delta.content is None:
model_response.choices[0].delta.content = ""
model_response.choices[0].delta.content += "<think>" + reasoning_content
self.sent_first_thinking_block = True
elif (
self.sent_first_thinking_block is True
and hasattr(model_response.choices[0].delta, "reasoning_content")
and model_response.choices[0].delta.reasoning_content
):
model_response.choices[0].delta.content = reasoning_content
elif (
self.sent_first_thinking_block is True
and not self.sent_last_thinking_block
and model_response.choices[0].delta.content
):
model_response.choices[0].delta.content = "</think>" + (model_response.choices[0].delta.content or "")
self.sent_last_thinking_block = True
if self.merge_reasoning_content_in_choices is not True:
return
delta: Final = model_response.choices[0].delta
reasoning_content: Final = getattr(delta, "reasoning_content", None)
content: Final = delta.content or ""
if reasoning_content:
self._emitted_disqualifying_content = True
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
opening: Final = "" if self.sent_first_thinking_block else "<think>"
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
self.sent_first_thinking_block = True
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
delta.content = "</think>" + content
self.sent_last_thinking_block = True
if hasattr(model_response.choices[0].delta, "reasoning_content"):
del model_response.choices[0].delta.reasoning_content
if hasattr(delta, "reasoning_content"):
del delta.reasoning_content
def _dispatch_provider_chunk(
self,
@ -1584,11 +1589,7 @@ class CustomStreamWrapper:
except Exception as e:
traceback.format_exc()
setattr(e, "message", str(e))
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
)
raise self._map_stream_exception(e)
def set_logging_event_loop(self, loop):
"""
@ -1865,9 +1866,7 @@ class CustomStreamWrapper:
if response.choices:
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# HANDLE STREAM OPTIONS
self.chunks.append(response)
@ -2084,9 +2083,7 @@ class CustomStreamWrapper:
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk and processed_chunk.choices:
@ -2156,9 +2153,7 @@ class CustomStreamWrapper:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
@ -2172,7 +2167,7 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
await self.aclose()
@ -2305,11 +2300,17 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._handle_stream_fallback_error(e)
async def wait_for_failure_logging(self, timeout: float) -> None:
"""Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
if self._failure_logging is None:
return
await asyncio.wait((self._failure_logging,), timeout=timeout)
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2346,6 +2347,43 @@ class CustomStreamWrapper:
recover_error,
)
# Delta fields a text-only prefill continuation cannot carry, so a stream
# that emitted any of them is not eligible for mid-stream continuation.
_CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
"tool_calls",
"function_call",
"reasoning_content",
"thinking_blocks",
"reasoning_items",
"audio",
"images",
"annotations",
)
@classmethod
def _delta_disqualifies_continuation(cls, delta: object) -> bool:
get: Final = getattr(delta, "get", None)
if not callable(get):
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.
get: Final = getattr(delta, "get", None)
content: Final = get("content") if callable(get) else None
self.response_uptil_now += content if isinstance(content, str) else ""
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _map_stream_exception(self, original_exception: Exception) -> Exception:
return exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=original_exception,
extra_kwargs=with_deployment_api_base(self.logging_obj.model_call_details.get("litellm_params")),
)
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2356,6 +2394,8 @@ class CustomStreamWrapper:
429 (rate-limit) is explicitly exempted from the 4xx filter because
it is transient and the Router should switch to another model group.
So is a transport read timeout, which maps to a 408 Timeout, including
one an inner Responses stream already mapped for the chat bridge.
An error an inner stream already wrapped (the chat-to-Responses bridge
consumes a Responses stream) is rebuilt around the provider exception
@ -2377,6 +2417,7 @@ class CustomStreamWrapper:
original_exception=e.original_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
)
# Map to OpenAI exception format. Some providers' mappers (e.g.
@ -2389,13 +2430,7 @@ class CustomStreamWrapper:
mapped_exception: Exception = e
else:
try:
mapped_exception = exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
mapped_exception = self._map_stream_exception(e)
except Exception as mapping_error:
mapped_exception = mapping_error
self._restore_consumer_correlation_context()
@ -2425,9 +2460,9 @@ class CustomStreamWrapper:
# Raise non-retriable client errors directly (skip fallback).
# Exception: 429 (rate-limit) IS retriable/transient — allow it
# through so the Router can switch to a different model group.
if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429:
raise mapped_exception
if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429:
if not _is_transport_timeout(e) and (
_skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
):
raise mapped_exception
raise MidStreamFallbackError(
@ -2437,6 +2472,7 @@ class CustomStreamWrapper:
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content,
)
@staticmethod

164
litellm/litellm_core_utils/tool_fields.py Normal file
View file

@ -0,0 +1,164 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
"acompletion",
"aresponses",
"acompact_responses",
"anthropic_messages",
"agenerate_content",
"agenerate_content_stream",
"acreate_interaction",
}
)
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
_TOOLS: Final = TypeAdapter(list[object])
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
def _schema_as_read(value: object) -> object:
if not value:
return None
if not isinstance(value, str):
return value
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return value
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
class _ToolFunction(TypedDict, total=False):
parameters: ReadOnly[_Schema]
class _ToolEntry(TypedDict, total=False):
function: ReadOnly[_ToolFunction | None]
parameters: ReadOnly[_Schema]
input_schema: ReadOnly[_Schema]
vector_store_ids: ReadOnly[list[str] | None]
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
@dataclass(frozen=True, slots=True)
class DropPolicy:
drop_params: bool
additional_drop_params: frozenset[str]
def drops(self, field: str) -> bool:
return self.drop_params or field in self.additional_drop_params
@dataclass(frozen=True, slots=True)
class MalformedField:
field: str
message: str
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
def _field_names(value: object) -> frozenset[str]:
try:
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
except ValidationError:
return frozenset()
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
return DropPolicy(
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
additional_drop_params=_field_names(params.get("additional_drop_params")),
)
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
error: Final = e.errors(include_url=False)[0]
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
def _tool_entry_error(index: int, entry: object) -> str | None:
try:
_TOOL_ENTRY.validate_python(entry, strict=True)
except ValidationError as e:
return first_validation_error(e, prefix=("tools", index))
return None
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
return None
try:
entries: Final = _TOOLS.validate_python(tools)
except ValidationError as e:
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
first_error: Final = next((error for error in entry_errors if error is not None), None)
if first_error is None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
)
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name != "web_search_options"}
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
try:
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
except ValidationError as e:
return MalformedField(
"web_search_options",
first_validation_error(e, prefix=("web_search_options",)),
_without_web_search_options,
)
return None
def resolve_tool_fields(
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
) -> Mapping[str, object] | MalformedField:
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
malformed: Final = tuple(field for field in checked if field is not None)
if not malformed:
return data
policies: Final = candidates()
rejected: Final = next(
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
)
if rejected is not None:
return rejected
return reduce(
lambda resolved, field: field.dropped(resolved),
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
data,
)
def resolve_deployment_tool_fields(
kwargs: Mapping[str, object], route: object
) -> Mapping[str, object] | MalformedField:
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

67
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -30,6 +30,7 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
)
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import (
build_complete_streaming_response,
build_usage_only_response_from_chunks,
@ -45,6 +46,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
scoped_structured_message_indices,
@ -924,10 +926,10 @@ class AnthropicMessagesHandler(BaseTranslation):
images=(),
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(
@ -1455,6 +1457,9 @@ class AnthropicMessagesHandler(BaseTranslation):
return _SSEFieldRewrite("delta", "text", next(replacements))
cls._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
cls._redact_ended_stream_message_delta(
responses_so_far, kept_tool_use_ids=frozenset(cls._streamed_tool_use_ids(responses_so_far).values())
)
@classmethod
def _write_ended_stream_tool_call_rewrites(
@ -1522,6 +1527,64 @@ class AnthropicMessagesHandler(BaseTranslation):
return None
cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use)
streamed_tool_use_ids: Final = cls._streamed_tool_use_ids(responses_so_far)
cls._redact_ended_stream_message_delta(
responses_so_far,
kept_tool_use_ids=frozenset(
tool_use_id for index, tool_use_id in streamed_tool_use_ids.items() if index not in rewrites_by_block
),
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
tool_use_blocks: Final = (
(event.get("index"), block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and block.get("type") == "tool_use"
)
return MappingProxyType(
{
index: tool_use_id
for index, block in tool_use_blocks
if isinstance(index, int) and isinstance(tool_use_id := block.get("id"), str)
}
)
@classmethod
def _redact_ended_stream_message_delta(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
*,
kept_tool_use_ids: frozenset[str],
) -> None:
"""Hold the buffered ``message_delta`` to the rules a re-emitted rewrite follows: a verdict survives
only for a tool call delivered unchanged, and no free-text ``explanation`` describes the text the
guardrail rewrote."""
def rewrite_stop_details(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "stop_details" not in delta:
return None
return _SSEFieldRewrite(
"delta", "stop_details", redacted_stop_details(delta["stop_details"], keep_explanations=False)
)
def rewrite_safeguard_results(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "safeguard_results" not in delta:
return None
return _SSEFieldRewrite(
"delta",
"safeguard_results",
filtered_safeguard_results(delta["safeguard_results"], kept_tool_use_ids, keep_explanations=False),
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_stop_details)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_safeguard_results)
@staticmethod
def _rewrite_ended_stream_events(

Some files were not shown because too many files have changed in this diff Show more