scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 99

chore: merge main into litellm_registry_audit_2026_09_23

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-24 01:19:28 +00:00
commit efc36b96ac
248 changed files with 13375 additions and 1454 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/scripts/path_filter.sh
View file

@ -11,7 +11,7 @@ run_full() {
[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
candidate_bases="main"
candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}"
merge_base=""
for base in $candidate_bases; do
git fetch --quiet origin "$base" 2>/dev/null || continue

292
.circleci/tests.yml Normal file
View file

@ -0,0 +1,292 @@
version: 2.1
commands:
wait_for_service:
parameters:
url:
type: string
timeout:
type: string
default: "60"
steps:
- run:
name: "Wait for << parameters.url >>"
command: |
TIMEOUT=<< parameters.timeout >>
URL="<< parameters.url >>"
ELAPSED=0
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
if echo "$URL" | grep -q '^tcp://'; then
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
sleep 2; ELAPSED=$((ELAPSED+2))
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
done
else
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
sleep 2; ELAPSED=$((ELAPSED+2))
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
done
fi
echo "Service ready after ${ELAPSED}s"
install_uv:
steps:
- run:
name: Install uv (pinned 0.10.9)
command: |
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
install_rust:
steps:
- run:
name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
command: |
case "$(uname -m)" in
x86_64)
RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
;;
aarch64)
RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
;;
*)
echo "install_rust: unsupported architecture $(uname -m)" >&2
exit 1
;;
esac
curl -sSLf -o /tmp/rustup-init \
"https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c -
chmod +x /tmp/rustup-init
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
rm -f /tmp/rustup-init
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.cargo/bin:$PATH"
rustc --version
cargo --version
install_codecov_cli:
steps:
- run:
name: Install Codecov CLI (pinned v11.3.1)
command: |
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
(cd /tmp && sha256sum -c codecov.SHA256SUM)
chmod +x /tmp/codecov
mkdir -p "$HOME/.local/bin"
mv /tmp/codecov "$HOME/.local/bin/codecov"
setup_litellm_enterprise_pip:
steps:
- run:
name: "Install local version of litellm-enterprise"
command: |
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
setup_test_deps:
steps:
- checkout
- install_uv
- install_rust
- restore_cache:
keys:
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ~/.cache/uv
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
skip_unless_relevant:
parameters:
category:
type: string
default: backend
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
steps:
- run:
name: "Skip job when no << parameters.category >>-relevant files changed"
command: |
export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}"
export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>"
[ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH
bash .circleci/scripts/path_filter.sh << parameters.category >>
start_postgres:
parameters:
db_name:
type: string
default: circle_test
image:
type: string
default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
steps:
- run:
name: Start PostgreSQL
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=<< parameters.db_name >> \
-p 5432:5432 \
<< parameters.image >>
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
start_redis:
steps:
- run:
name: Start Redis
command: |
docker run -d \
--name redis-cache \
-p 6379:6379 \
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
- wait_for_service:
url: tcp://localhost:6379
timeout: "60"
jobs:
unit:
parameters:
tests_path:
type: string
default: tests/unit
flag:
type: string
default: unit
shards:
type: integer
default: 6
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
parallelism: << parameters.shards >>
environment:
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- setup_test_deps
- skip_unless_relevant:
base_ref: << parameters.base_ref >>
pull_request_url: << parameters.pull_request_url >>
- run:
name: "Run << parameters.tests_path >> shard"
no_output_timeout: 20m
command: |
mkdir -p test-results/<< parameters.flag >>
mapfile -t files < <(find << parameters.tests_path >> -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)
if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.tests_path >> files; nothing to run"; exit 0; fi
set +e
uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
status=$?
set -e
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
exit "$status"
- install_codecov_cli
- run:
name: Upload coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; }
codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage.xml
documentation:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- setup_test_deps
- run:
name: Checkout litellm-docs
command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website
- run:
name: Run documentation validation
command: |
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
integration:
parameters:
suite:
type: string
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- setup_test_deps
- skip_unless_relevant:
base_ref: << parameters.base_ref >>
pull_request_url: << parameters.pull_request_url >>
- start_postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
- start_redis
- run:
name: Run owned integration contracts
command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
no_output_timeout: 15m
- run:
name: Stop owned database and Redis
when: always
command: |
mkdir -p test-results/integration-<< parameters.suite >>
docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
docker rm -f postgres-db redis-cache
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
workflows:
tests:
when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_"))
jobs:
- unit:
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- documentation
- integration:
name: integration-<< matrix.suite >>
matrix:
parameters:
suite: [sdk]
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>

13
.github/ci-coverage-allowlist.yml vendored
View file

@ -10,12 +10,13 @@ test_paths:
paths:
- tests/rust-python-harness
- reason: >-
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
test_caching.py by node ID. It does not run that file's other tests.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live

13
.github/pull_request_template.md vendored
View file

@ -1,6 +1,8 @@
<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
everyday engineering language, extremely parsable and readable at a glance. This goes double for
the TLDR, User Flow, and Caveats sections -->
the TLDR, User Flow, and Caveats sections
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
"## Affected release" with nothing under it must not appear in the final description -->
## TLDR
@ -21,6 +23,7 @@ How it solves it:
<!-- Two ordered lists, Before and After, walking the same end user through the same task, written strictly from that user's seat
Read the linked issue, ticket, or customer thread first so the flow reflects the real application and the routes its users actually hit; don't invent a generic scenario
Lead each list with one plain sentence saying where the flow fails (Before) or succeeds (After), then number the steps
Keep it tight: aim for 3 to 5 steps per list, one line each, roughly 20 words max, and never pad a shorter flow with filler steps to hit the count. Cover the one path the PR changes and fold variants (case, other field, second endpoint) into a clause on the step they belong to rather than their own steps. The example below is the target length
Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
@ -45,15 +48,15 @@ After: the same request comes back with real token counts, so the dashboard show
## Relevant issues
<!-- e.g., "Fixes #000" -->
<!-- e.g., "Fixes #000". Drop the section if there is none -->
## Affected release
<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Leave the section blank otherwise -->
<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Drop the section otherwise -->
## Linear ticket
<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, drop the section rather than guessing -->
## Pre-Submission checklist
@ -134,7 +137,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
human reader
If you assumed something instead of testing it, e.g. "only reproduces with X on" or "no
user-observable behavior difference", list it here too with what breaks if it is wrong
Leave this section empty if there are none -->
Drop this section if there are none -->
## QA runbook

29
.github/workflows/test-redis-compat.yml vendored
View file

@ -9,6 +9,7 @@ on:
- "litellm/_redis.py"
- "litellm/_redis_credential_provider.py"
- "tests/test_litellm/test_redis.py"
- "tests/local_testing/test_caching.py"
- "tests/test_litellm/caching/test_redis_connection_pool.py"
- ".github/workflows/test-redis-compat.yml"
- "pyproject.toml"
@ -26,6 +27,9 @@ jobs:
name: "redis-py ${{ matrix.redis-version }}"
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
@ -55,7 +59,7 @@ jobs:
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router
- name: Pin redis-py to the matrix version
env:
@ -64,12 +68,33 @@ jobs:
uv pip install "redis==${REDIS_VERSION:?}"
uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
- name: Build Redis for cluster authentication tests
run: |
curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz"
echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP"
make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server
echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH"
- name: Run redis unit tests
run: |
redis-server --version
uv run --no-sync pytest \
tests/test_litellm/test_redis.py \
tests/test_litellm/caching/test_redis_connection_pool.py \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
--tb=short -vv \
--reruns 2 \
--reruns-delay 1 \
--durations=20
--durations=20 \
--cov=./litellm --cov-report=xml:coverage-redis.xml
- name: Upload Redis coverage
if: matrix.redis-version == '5.3.1'
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
files: coverage-redis.xml
flags: redis-compat
fail_ci_if_error: false

1
.github/workflows/test-unit.yml vendored
View file

@ -111,6 +111,7 @@ jobs:
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/messages
tests/test_litellm/embeddings
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag

4
AGENTS.md
View file

@ -33,11 +33,11 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it

63
litellm-rust/crates/python-bridge/src/cache/binding.rs vendored
View file

@ -29,6 +29,7 @@ pub(super) enum CacheBinding {
#[pyclass(frozen, name = "_ResponseCacheRuntime")]
pub(crate) struct ResolvedCache {
binding: CacheBinding,
guard: Option<super::facade::FacadeGuard>,
pid: u32,
}
@ -36,10 +37,24 @@ impl ResolvedCache {
pub(super) fn new(binding: CacheBinding) -> Self {
Self {
binding,
guard: None,
pid: std::process::id(),
}
}
pub(super) fn with_guard(mut self, guard: super::facade::FacadeGuard) -> Self {
self.guard = Some(guard);
self
}
pub(super) fn native_service(&self) -> PyResult<Option<NativeResponseCache>> {
self.check_process()?;
Ok(match &self.binding {
CacheBinding::Native(service) => Some(service.clone()),
_ => None,
})
}
fn check_process(&self) -> PyResult<()> {
if matches!(self.binding, CacheBinding::Native(_)) && self.pid != std::process::id() {
return Err(PyRuntimeError::new_err(
@ -70,6 +85,43 @@ impl ResolvedCache {
#[pymethods]
impl ResolvedCache {
#[staticmethod]
pub(crate) fn from_selected(cache: &Bound<'_, PyAny>) -> PyResult<Self> {
let py = cache.py();
let binding = if cache.is_none() {
CacheBinding::Disabled
} else if let Ok(handle) = cache.extract::<PyRef<'_, super::handle::CacheTestHandle>>() {
CacheBinding::Native(handle.service()?)
} else if let Some(service) = super::facade::resolve(py, cache)? {
CacheBinding::Native(service)
} else if let Some(runtime) = cache
.getattr_opt("_native_cache")?
.filter(|value| !value.is_none())
{
let resolved = runtime
.getattr("native")?
.extract::<PyRef<'_, ResolvedCache>>()?;
match resolved.native_service()? {
Some(service) => {
if !resolved
.guard
.as_ref()
.is_some_and(|guard| guard.matches(py, cache).unwrap_or(false))
{
return Err(RustBridgeDeclined::new_err(
"native cache runtime no longer matches its facade",
));
}
CacheBinding::Native(service)
}
None => CacheBinding::PythonCallback(PythonCallback::new(cache.clone().unbind())),
}
} else {
CacheBinding::PythonCallback(PythonCallback::new(cache.clone().unbind()))
};
Ok(Self::new(binding))
}
#[staticmethod]
fn from_cache(cache: &Bound<'_, PyAny>) -> PyResult<Self> {
let config = match NativeCacheConfig::project(cache)? {
@ -80,7 +132,13 @@ impl ResolvedCache {
};
let backend = cache.getattr("cache")?;
let service = activate(cache.py(), &backend, config)?;
Ok(Self::new(CacheBinding::Native(service)))
let resolved = Self::new(CacheBinding::Native(service.clone()));
Ok(
match super::facade::FacadeGuard::capture(cache.py(), cache, &service) {
Ok(guard) => resolved.with_guard(guard),
Err(_) => resolved,
},
)
}
#[getter]
@ -323,6 +381,9 @@ impl ResolvedCache {
if let CacheBinding::PythonCallback(callback) = &self.binding {
callback.traverse(&visit)?;
}
if let Some(guard) = &self.guard {
guard.traverse(visit)?;
}
Ok(())
}
}

2
litellm-rust/crates/python-bridge/src/cache/facade.rs vendored
View file

@ -472,7 +472,7 @@ impl FacadeGuard {
})
}
fn matches(&self, py: Python<'_>, facade: &Bound<'_, PyAny>) -> PyResult<bool> {
pub(super) fn matches(&self, py: Python<'_>, facade: &Bound<'_, PyAny>) -> PyResult<bool> {
if !self.outer.matches(py, facade)? {
return Ok(false);
}

4
litellm-rust/crates/python-bridge/src/cache/mod.rs vendored
View file

@ -20,9 +20,7 @@ use pyo3::{
types::PyDict,
};
pub(crate) use self::{
binding::ResolvedCache, handle::CacheTestHandle, resolver::CacheTestResolver,
};
pub(crate) use self::{binding::ResolvedCache, handle::CacheTestHandle, resolver::CacheResolver};
fn cache_error(error: Error) -> PyErr {
match error {

24
litellm-rust/crates/python-bridge/src/cache/resolver.rs vendored
View file

@ -1,19 +1,14 @@
use pyo3::{PyTraverseError, PyVisit, prelude::*};
use super::{
binding::{CacheBinding, ResolvedCache},
callback::PythonCallback,
facade,
handle::CacheTestHandle,
};
use super::binding::ResolvedCache;
#[pyclass(frozen, name = "_CacheTestResolver")]
pub(crate) struct CacheTestResolver {
#[pyclass(frozen, name = "_CacheResolver")]
pub(crate) struct CacheResolver {
namespace: Py<PyAny>,
}
#[pymethods]
impl CacheTestResolver {
impl CacheResolver {
#[new]
fn new(namespace: Py<PyAny>) -> Self {
Self { namespace }
@ -21,16 +16,7 @@ impl CacheTestResolver {
pub(crate) fn resolve(&self, py: Python<'_>) -> PyResult<ResolvedCache> {
let object = self.namespace.bind(py).getattr("cache")?;
let binding = if object.is_none() {
CacheBinding::Disabled
} else if let Ok(handle) = object.extract::<PyRef<'_, CacheTestHandle>>() {
CacheBinding::Native(handle.service()?)
} else if let Some(service) = facade::resolve(py, &object)? {
CacheBinding::Native(service)
} else {
CacheBinding::PythonCallback(PythonCallback::new(object.unbind()))
};
Ok(ResolvedCache::new(binding))
ResolvedCache::from_selected(&object)
}
fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {

17
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -13,7 +13,7 @@ mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
use crate::cache::{CacheTestHandle, CacheTestResolver, ResolvedCache};
use crate::cache::{CacheResolver, CacheTestHandle, ResolvedCache};
#[cfg(feature = "panic-test")]
#[pymodule_export]
use crate::diagnostics::_panic_for_test;
@ -27,14 +27,16 @@ mod _native {
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{
achat_completions, chat_completions, chat_completions_decline,
achat_completions, acompletion, chat_completions, chat_completions_decline, completion,
};
#[pymodule_export]
use crate::routes::embeddings::{aembedding, embedding};
#[pymodule_export]
use crate::routes::messages::{amessages, messages};
#[pymodule_export]
use crate::routes::ocr::{aocr, ocr};
#[pymodule_export]
use crate::routes::responses::ResponsesWebSocketConnection;
use crate::routes::responses::{ResponsesWebSocketConnection, aresponses, responses};
#[pymodule_export]
use crate::routes::token_counter::TokenCounter;
#[cfg(feature = "huggingface")]
@ -51,7 +53,8 @@ mod _native {
let py = module.py();
let dict = module.dict();
dict.set_item("_CacheTestHandle", py.get_type::<CacheTestHandle>())?;
dict.set_item("_CacheTestResolver", py.get_type::<CacheTestResolver>())?;
dict.set_item("_CacheResolver", py.get_type::<CacheResolver>())?;
dict.set_item("_CacheTestResolver", py.get_type::<CacheResolver>())?;
dict.set_item("_ResponseCacheRuntime", py.get_type::<ResolvedCache>())?;
dict.set_item(
"_SecretManagerRuntime",
@ -82,6 +85,8 @@ mod tests {
"ProcessReservedForForking",
"ocr",
"aocr",
"embedding",
"aembedding",
"transcription",
"atranscription",
"messages",
@ -89,6 +94,10 @@ mod tests {
"chat_completions_decline",
"chat_completions",
"achat_completions",
"completion",
"acompletion",
"responses",
"aresponses",
"ResponsesWebSocketConnection",
"NativeDiagnosticProcessor",
"TokenCounter",

54
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -1,3 +1,6 @@
use pyo3::types::{PyDict, PyTuple};
use crate::errors::RustBridgeDeclined;
use crate::logger::{run_async, run_sync};
use litellm_core::chat_completions::{
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
@ -123,9 +126,58 @@ pub(crate) fn achat_completions<'py>(
)
}
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn completion(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native chat completions route is not implemented",
))
}
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn acompletion(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native chat completions route is not implemented",
))
}
#[cfg(test)]
mod tests {
use pyo3::{prelude::*, types::PyList};
use pyo3::{
prelude::*,
types::{PyDict, PyList, PyTuple},
};
use crate::errors::RustBridgeDeclined;
#[test]
fn both_entrypoints_decline_before_provider_execution() {
Python::initialize();
Python::attach(|py| {
let request = PyDict::new(py);
let args = PyTuple::empty(py);
let kwargs = PyDict::new(py);
for entrypoint in [super::completion, super::acompletion] {
let error = entrypoint(request.clone().into_any(), args.clone(), kwargs.clone())
.expect_err(
"native chat completions must decline until a route machine exists",
);
assert!(error.is_instance_of::<RustBridgeDeclined>(py));
}
});
}
#[test]
fn chat_completions_decline_keeps_existing_reasons() {

58
litellm-rust/crates/python-bridge/src/routes/embeddings.rs Normal file
View file

@ -0,0 +1,58 @@
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use crate::errors::RustBridgeDeclined;
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn embedding(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native embeddings route is not implemented",
))
}
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn aembedding(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native embeddings route is not implemented",
))
}
#[cfg(test)]
mod tests {
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use crate::errors::RustBridgeDeclined;
#[test]
fn both_entrypoints_decline_before_provider_execution() {
Python::initialize();
Python::attach(|py| {
let request = PyDict::new(py);
let args = PyTuple::empty(py);
let kwargs = PyDict::new(py);
for entrypoint in [super::embedding, super::aembedding] {
let error = entrypoint(request.clone().into_any(), args.clone(), kwargs.clone())
.expect_err("native embeddings must decline until a route machine exists");
assert!(error.is_instance_of::<RustBridgeDeclined>(py));
}
});
}
}

1
litellm-rust/crates/python-bridge/src/routes/mod.rs
View file

@ -1,5 +1,6 @@
pub(crate) mod audio_transcription;
pub(crate) mod chat_completions;
pub(crate) mod embeddings;
pub(crate) mod messages;
pub(crate) mod ocr;
pub(crate) mod responses;

56
litellm-rust/crates/python-bridge/src/routes/responses.rs
View file

@ -1,12 +1,41 @@
use litellm_core::responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
use pyo3::prelude::*;
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use serde_json::Value;
use crate::{
errors::responses_error_to_pyerr,
errors::{RustBridgeDeclined, responses_error_to_pyerr},
marshal::{marshal_headers, optional_timeout},
};
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn responses(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native responses route is not implemented",
))
}
#[pyfunction]
#[pyo3(signature = (request, args, kwargs))]
pub(crate) fn aresponses(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
) -> PyResult<Py<PyAny>> {
drop((request, args, kwargs));
Err(RustBridgeDeclined::new_err(
"native responses route is not implemented",
))
}
#[pyclass]
pub(crate) struct ResponsesWebSocketConnection {
inner: RustResponsesWebSocketConnection,
@ -63,7 +92,28 @@ mod tests {
use std::{ffi::CString, time::Duration};
use futures_util::{SinkExt, StreamExt};
use pyo3::{prelude::*, types::PyDict};
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use crate::errors::RustBridgeDeclined;
#[test]
fn both_entrypoints_decline_before_provider_execution() {
Python::initialize();
Python::attach(|py| {
let request = PyDict::new(py);
let args = PyTuple::empty(py);
let kwargs = PyDict::new(py);
for entrypoint in [super::responses, super::aresponses] {
let error = entrypoint(request.clone().into_any(), args.clone(), kwargs.clone())
.expect_err("native responses must decline until a route machine exists");
assert!(error.is_instance_of::<RustBridgeDeclined>(py));
}
});
}
use tokio::net::TcpListener;
use tokio_tungstenite::{accept_async, tungstenite::Message};

4
litellm/__init__.py
View file

@ -1458,6 +1458,7 @@ from .skills.main import (
from .containers.main import *
from .ocr.dispatch import *
from .chat_completions.dispatch import *
from .embeddings.dispatch import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
@ -1871,6 +1872,9 @@ if TYPE_CHECKING:
from .llms.openrouter.responses.transformation import (
OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig,
)
from .llms.bedrock.responses.transformation import (
BedrockOpenAIResponsesConfig as BedrockOpenAIResponsesConfig,
)
from .llms.bedrock_mantle.responses.transformation import (
BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig,
)

5
litellm/_lazy_imports_registry.py
View file

@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
"PerplexityResponsesConfig",
"DatabricksResponsesAPIConfig",
"OpenRouterResponsesAPIConfig",
"BedrockOpenAIResponsesConfig",
"BedrockMantleResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"VertexAIInteractionsConfig",
@ -921,6 +922,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenAITextCompletionConfig",
),
"GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"),
"BedrockOpenAIResponsesConfig": (
".llms.bedrock.responses.transformation",
"BedrockOpenAIResponsesConfig",
),
"BedrockMantleChatConfig": (
".llms.bedrock_mantle.chat.transformation",
"BedrockMantleChatConfig",

4
litellm/_logging.py
View file

@ -631,9 +631,9 @@ class LevelRoutingStreamHandler(logging.StreamHandler):
)
preferred: Final = sys.stdout if is_stdout_record else sys.stderr
if preferred is None or getattr(preferred, "closed", False):
self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record
self.stream = sys.stderr
else:
self.stream = preferred # rebind-ok: StreamHandler.emit writes self.stream under the handler lock
self.stream = preferred
super().emit(record)

30
litellm/_redis.py
View file

@ -689,11 +689,12 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
verbose_logger.debug("init_redis_cluster: startup nodes are being initialized.")
from redis.cluster import ClusterNode
auth_kwargs: Final = _credential_provider_auth_kwargs(redis_kwargs)
args: Final = _get_redis_cluster_kwargs()
cluster_kwargs: Final = {}
for arg in redis_kwargs:
for arg in auth_kwargs:
if arg in args:
cluster_kwargs[arg] = redis_kwargs[arg]
cluster_kwargs[arg] = auth_kwargs[arg]
new_startup_nodes: Final[list[ClusterNode]] = []
@ -771,13 +772,13 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
return sentinel.master_for(service_name, **connection_kwargs)
def _async_credential_provider(redis_connect_func: object | None) -> CredentialProvider | None:
"""The Azure AD and GCP IAM connect funcs run their AUTH exchange with the blocking client
API, so on an async connection their ``send_command``/``read_response`` calls return
coroutines nobody awaits and every connect fails. Async paths authenticate through a
``CredentialProvider`` instead, which redis-py consults per connection so the token stays
fresh. Any other ``redis_connect_func`` is left where it is, since redis-py awaits it
itself when it is a coroutine function."""
def _credential_provider_from_connect_func(redis_connect_func: object | None) -> CredentialProvider | None:
"""Translate IAM callbacks for paths that need credentials during the standard handshake.
Async connections cannot run blocking AUTH callbacks. Sync clusters authenticate before
invoking the callback, so they also need the provider during the initial handshake.
redis-py consults the provider for each connection, keeping token refresh intact.
"""
gcp_service_account: Final = getattr(redis_connect_func, "_gcp_service_account", None)
if gcp_service_account is not None:
return GCPIAMCredentialProvider(gcp_service_account)
@ -789,14 +790,13 @@ def _async_credential_provider(redis_connect_func: object | None) -> CredentialP
return None
def _async_auth_kwargs(redis_kwargs: dict) -> dict:
"""Swaps a connect func an async path cannot run for the equivalent credential provider,
which supersedes any static username or password redis-py would otherwise reject it with."""
def _credential_provider_auth_kwargs(redis_kwargs: dict) -> dict:
"""Use a credential provider instead of an IAM callback and conflicting static credentials."""
explicit_provider: Final = redis_kwargs.get("credential_provider")
credential_provider: Final = (
explicit_provider
if explicit_provider is not None
else _async_credential_provider(redis_kwargs.get("redis_connect_func"))
else _credential_provider_from_connect_func(redis_kwargs.get("redis_connect_func"))
)
if credential_provider is None:
return redis_kwargs
@ -834,7 +834,7 @@ def get_redis_async_client(
connection_pool: async_redis.BlockingConnectionPool | None = None,
**env_overrides,
) -> async_redis.Redis | async_redis.RedisCluster:
redis_kwargs: Final = _async_auth_kwargs(_get_redis_client_logic(**env_overrides))
redis_kwargs: Final = _credential_provider_auth_kwargs(_get_redis_client_logic(**env_overrides))
if "startup_nodes" in redis_kwargs:
from redis.cluster import ClusterNode
@ -906,7 +906,7 @@ def get_redis_async_client(
def get_redis_connection_pool(
**env_overrides,
) -> async_redis.BlockingConnectionPool | None:
redis_kwargs: Final = _async_auth_kwargs(_get_redis_client_logic(**env_overrides))
redis_kwargs: Final = _credential_provider_auth_kwargs(_get_redis_client_logic(**env_overrides))
verbose_logger.debug("get_redis_connection_pool: redis_kwargs", redis_kwargs)
if "startup_nodes" in redis_kwargs:

4
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -191,8 +191,6 @@ def _as_chat_reasoning_items(
) -> list[ChatCompletionReasoningItem] | None:
if not reasoning_items:
return None
# cast-ok: _BuiltReasoningItem is the structural shape ChatCompletionReasoningItem
# describes, and TypedDict invariance is what stops the two from unifying here.
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
@ -1370,7 +1368,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
if tool_call_index_map is None:
return output_index
if output_index not in tool_call_index_map:
tool_call_index_map[output_index] = len(tool_call_index_map) # mutable-ok: per-stream accumulator state
tool_call_index_map[output_index] = len(tool_call_index_map)
return tool_call_index_map[output_index]
@staticmethod

0
litellm/embeddings/__init__.py Normal file
View file

95
litellm/embeddings/dispatch.py Normal file
View file

@ -0,0 +1,95 @@
import inspect
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.embeddings.entrypoints import (
NATIVE_AEMBEDDING,
NATIVE_EMBEDDING,
LiteLLMEmbeddingRequest,
)
from litellm.rust_bridge.public_call import bind, optional_mapping, optional_str, signature
from litellm.types.utils import EmbeddingResponse
__all__ = ("aembedding", "embedding")
PythonEmbedding: TypeAlias = Callable[..., EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]]
PythonAembedding: TypeAlias = Callable[..., Awaitable[EmbeddingResponse]]
_PYTHON_EMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy public callable contract
PythonEmbedding, main.embedding
)
_PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy public callable contract
PythonAembedding, main.aembedding
)
_EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
def _public_request(
legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> LiteLLMEmbeddingRequest | None:
fields: Final = bind(legacy, args, kwargs)
if fields is None:
return None
model: Final = fields.get("model")
if not isinstance(model, str):
return None
extra: Final = optional_mapping(fields.get("kwargs")) or MappingProxyType({})
return LiteLLMEmbeddingRequest(
model=model,
input=fields.get("input"),
api_key=optional_str(fields.get("api_key")),
api_base=optional_str(fields.get("api_base")),
custom_llm_provider=optional_str(fields.get("custom_llm_provider")),
kwargs=extra,
)
def _context(request: LiteLLMEmbeddingRequest) -> RouteContext:
return RouteContext(Route.EMBEDDINGS, provider=request.custom_llm_provider, model=request.model)
_DISPATCH: Final = PublicDispatch(
route=Route.EMBEDDINGS,
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
context=_context,
bypass=lambda request: request.kwargs.get("aembedding") is True,
)
_ADISPATCH: Final = PublicDispatch(
route=Route.EMBEDDINGS,
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
context=_context,
)
def embedding(
*args: object,
**kwargs: object, # kwargs-ok: preserve the public embedding call shape
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
return _DISPATCH.run(
args,
kwargs,
python=_PYTHON_EMBEDDING,
binding=NATIVE_EMBEDDING,
native=call_hook,
)
async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape
return await _ADISPATCH.arun(
args,
kwargs,
python=_PYTHON_AEMBEDDING,
binding=NATIVE_AEMBEDDING,
native=call_hook,
)
embedding.__doc__ = _PYTHON_EMBEDDING.__doc__
embedding.__wrapped__ = _PYTHON_EMBEDDING # pyright: ignore[reportFunctionMemberAccess] # preserve the legacy signature
aembedding.__doc__ = _PYTHON_AEMBEDDING.__doc__
aembedding.__wrapped__ = _PYTHON_AEMBEDDING # pyright: ignore[reportFunctionMemberAccess] # preserve the legacy signature

6
litellm/experimental_mcp_client/client.py
View file

@ -764,7 +764,7 @@ class MCPClient:
follow_redirects=True,
event_hooks=MappingProxyType(
{"response": [capture_upstream_error_response], "request": [guard] if guard else []}
), # mutable-ok: httpx types require lists of hooks
),
)
return factory
@ -921,9 +921,7 @@ class MCPClient:
with anyio.fail_after(max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)):
for page_index in range(MCP_TOOL_LISTING_MAX_PAGES):
try:
page = await fetch_page( # rebind-ok: each SDK page replaces the previous one
None if cursor is None else PaginatedRequestParams(cursor=cursor)
)
page = await fetch_page(None if cursor is None else PaginatedRequestParams(cursor=cursor))
except MCPError as error:
if page_index > 0 and error.error.code == METHOD_NOT_FOUND:
raise RuntimeError("MCP list operation became unavailable during pagination") from error

2
litellm/integrations/custom_guardrail.py
View file

@ -1641,5 +1641,5 @@ def log_guardrail_information(func):
return async_wrapper(*args, **kwargs)
return sync_wrapper(*args, **kwargs)
vars(wrapper)[LOGS_GUARDRAIL_INFORMATION_MARKER] = True # rebind-ok: stamps the wrapper this call just built
vars(wrapper)[LOGS_GUARDRAIL_INFORMATION_MARKER] = True
return wrapper

4
litellm/integrations/newrelic/newrelic_metrics.py
View file

@ -366,9 +366,7 @@ class NewRelicMetricsLogger(CustomBatchLogger):
error to keep the client-error path (drop) distinct from 5xx (retry)."""
payload: Final = build_metric_payload(records=batch, window_start=window_start, now=time.time())
try:
status = (
await self.async_send_compressed_data(payload)
).status_code # rebind-ok: reassigned from the raised HTTPStatusError below
status = (await self.async_send_compressed_data(payload)).status_code
except HTTPStatusError as e:
status = e.response.status_code
except Exception as e: # noqa: BLE001 # transport/network failure re-queues the batch

19
litellm/integrations/otel/README.md
View file

@ -63,7 +63,24 @@ Spans are named `"{service} {call_type}"` (e.g. `"redis set"`) so repeated calls
to one service stay distinguishable. Like every other span they parent to the
**ambient** context, falling back to the threaded `litellm_parent_otel_span` only
when ambient has no live span; a background job with neither starts its own root
trace. Caller-supplied `event_metadata` is **sanitized** before it reaches a span
trace.
**Post-response work is its own trace.** Spend tracking, the response cache write
and the spend-counter increment all run after the response is on the wire, so they
add nothing to the request's latency. Parenting them under the (already ended)
server span stretched the request trace past the request itself, which is what a
viewer shows as trace duration. `context.resolve_service_span_context` compares
the call's end time with the resolved parent's end time: a call that finished
after its parent ended starts a **new root trace** carrying a **span link** back
to the request span (the `FollowsFrom` relationship of OpenTracing; the default
`:link` propagation style of the OTel Ruby ActiveJob and Sidekiq
instrumentations). Identity Baggage still rides along, so the detached span keeps
its team / key / user attributes. Only an SDK span that has really ended detaches:
a sampled-out or remote `NonRecordingSpan` is never recording but is still the
right parent. A call that ended before the server span did stays a child even when
its `asyncio.create_task`-dispatched hook runs after the response.
Caller-supplied `event_metadata` is **sanitized** before it reaches a span
(primitives only, no live objects, no secrets/headers, bounded) — see
`payloads.sanitize_event_metadata`.

11
litellm/integrations/otel/logger.py
View file

@ -56,8 +56,8 @@ from litellm.integrations.otel.plumbing.context import (
request_root_http_route,
request_root_span,
resolve_mcp_span_context,
resolve_parent_context,
resolve_request_span_context,
resolve_service_span_context,
set_request_baggage,
set_request_root_span,
)
@ -671,14 +671,17 @@ class OpenTelemetryV2(CustomLogger):
# rides along and the call nests under whatever request phase is active —
# e.g. a DB lookup under the live ``auth`` span), falling back to the
# server span the proxy threaded as ``parent_otel_span``. A background
# service call has neither, so it starts its own root trace.
parent_context: Final = resolve_parent_context(threaded=parent_otel_span)
# service call has neither, so it starts its own root trace, as does one
# that finished after the request span ended (linked back to it).
end_time_ns: Final = to_ns(end_time)
parent_context, links = resolve_service_span_context(threaded=parent_otel_span, end_time_ns=end_time_ns)
return self._emitter.emit(
role,
data,
parent_context=parent_context,
start_time_ns=to_ns(start_time),
end_time_ns=to_ns(end_time),
end_time_ns=end_time_ns,
links=links,
)
# ====================================================================== #

23
litellm/integrations/otel/plumbing/context.py
View file

@ -9,6 +9,7 @@ from opentelemetry import baggage
from opentelemetry.context import Context, get_current
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import (
INVALID_SPAN,
Link,
NonRecordingSpan,
Span,
@ -225,6 +226,28 @@ def resolve_parent_context(threaded: Span | None = None) -> Context:
return ctx
def resolve_service_span_context(
threaded: Span | None = None, end_time_ns: int | None = None
) -> tuple[Context, tuple[Link, ...]]:
"""Parent context + links for a service/DB span that ended at ``end_time_ns``.
A call that finished after its parent ended (post-response spend tracking)
starts its own root trace with a span link back to the parent instead of
stretching the parent's trace. Baggage stays on the returned context.
"""
ctx: Final = resolve_parent_context(threaded)
parent: Final = get_current_span(ctx)
if not _ended_before(parent, end_time_ns):
return ctx, ()
return set_span_in_context(INVALID_SPAN, ctx), (Link(parent.get_span_context()),)
def _ended_before(span: Span, end_time_ns: int | None) -> bool:
if not isinstance(span, ReadableSpan) or span.end_time is None:
return False
return end_time_ns is None or end_time_ns > span.end_time
def resolve_request_span_context() -> Context:
"""The parent context for a request-level span (the LLM call, a guardrail).

4
litellm/integrations/otel/plumbing/providers.py
View file

@ -353,7 +353,7 @@ class _DrainPool:
def _drain_until_closed(self) -> None:
while True:
processor: SpanProcessor | None = self._pending.get() # rebind-ok: loop variable
processor: SpanProcessor | None = self._pending.get()
if processor is None:
return
_shutdown_quietly(processor)
@ -572,7 +572,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
span, destination.span_scope
):
continue
processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop
processor = self._acquire(destination)
if processor is None:
continue
try:

2
litellm/integrations/otel/presets/destinations.py
View file

@ -151,7 +151,7 @@ def destination_for(
endpoint, protocol = resolved
return OtelDestination(
endpoint=endpoint,
headers=MappingProxyType(dict(headers)), # mutable-ok: MappingProxyType needs a concrete mapping to wrap
headers=MappingProxyType(dict(headers)),
resource_attributes=MappingProxyType({"service.name": service_name}) if service_name else _NO_ATTRS,
callback_name=callback_name,
protocol=protocol,

2
litellm/integrations/prometheus.py
View file

@ -131,7 +131,7 @@ def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrisma
"""View a repository's prisma table through the pagination surface budget metrics need."""
return cast(
_PaginatedPrismaTable[_TableRowT],
repository.table, # cast-ok: prisma rows carry the budget columns the domain model declares
repository.table,
)

111
litellm/integrations/shadow_eval_logger.py
View file

@ -11,6 +11,7 @@ across pods or stop races; the hook reads active jobs through a short-TTL cache.
import asyncio
import hashlib
import json
import random
import traceback
from collections.abc import Awaitable, Callable, Mapping, Sequence
@ -28,7 +29,7 @@ from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.tools import is_web_search_tool_responses
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, independent_snapshot
from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
from litellm.litellm_core_utils.llm_judge import (
default_router_provider,
@ -281,10 +282,8 @@ class _SurfaceOps:
request (messages plus translated generation params) and how its response yields
the judgeable final text. Membership in this table IS the sampling allowlist;
unknown call types fail closed. ``wire_params`` marks the surfaces whose params
come from the proxy's wire-body snapshot, which is taken before the guardrail
pre-call hook: those rows must not sample a request a pre-call guardrail rewrote,
or the shadow call would replay content (tools, unmasked entities) the guardrail
removed."""
come from the proxy's native request snapshot. Requests rewritten by guardrails
require a post-hook snapshot whose guardrail history is still current."""
__slots__ = ("chat_request", "final_text", "wire_params")
@ -311,19 +310,85 @@ _NON_MUTATING_GUARDRAIL_MODES: Final = frozenset(
)
def _guardrail_is_non_mutating(entry: Mapping[str, object], allowed_modes: frozenset[str]) -> bool:
modes: Final = entry.get("guardrail_mode")
return all(
isinstance(mode, str) and mode in allowed_modes
for mode in (modes if isinstance(modes, list | tuple) else (modes,))
)
def _request_mutating_guardrail_ran(request_metadata: Mapping[str, object]) -> bool:
"""Whether a guardrail that can rewrite the outbound request ran on this one, read
from the same guardrail-information entries spend logging uses. str-enum modes
compare equal to their plain-string values, and an entry whose mode is missing or
unrecognized counts as mutating."""
raw: Final = request_metadata.get("standard_logging_guardrail_information")
entries: Final = raw if isinstance(raw, Sequence) else ()
modes_per_entry: Final = tuple(entry.get("guardrail_mode") for entry in entries if isinstance(entry, Mapping))
return any(
not all(
mode in _NON_MUTATING_GUARDRAIL_MODES for mode in (modes if isinstance(modes, list | tuple) else (modes,))
not _guardrail_is_non_mutating(entry, _NON_MUTATING_GUARDRAIL_MODES)
for entry in entries
if isinstance(entry, Mapping)
)
def request_guardrail_fingerprint(request_metadata: Mapping[str, object]) -> str | None:
raw: Final = request_metadata.get("standard_logging_guardrail_information")
entries: Final = raw if isinstance(raw, Sequence) else ()
replay_safe_modes: Final = _NON_MUTATING_GUARDRAIL_MODES - frozenset(("logging_only",))
relevant: Final = tuple(
entry
for entry in entries
if isinstance(entry, Mapping) and not _guardrail_is_non_mutating(entry, replay_safe_modes)
)
try:
serialized: Final = json.dumps(relevant, sort_keys=True, default=str)
except (TypeError, ValueError):
return None
return hashlib.sha256(serialized.encode()).hexdigest()
@dataclass(frozen=True, slots=True)
class GuardrailRequestSnapshot:
body: Mapping[str, object]
fingerprint: str
@staticmethod
def capture(body: Mapping[str, object], metadata: Mapping[str, object]) -> "GuardrailRequestSnapshot | None":
if not _request_mutating_guardrail_ran(metadata):
return None
fingerprint: Final = request_guardrail_fingerprint(metadata)
if fingerprint is None:
return None
return GuardrailRequestSnapshot(
body=MappingProxyType(
_CHAT_REQUEST_ADAPTER.validate_python(
independent_snapshot(dict(body)) # mutable-ok: snapshot helper requires a plain dictionary
)
),
fingerprint=fingerprint,
)
for modes in modes_per_entry
def _post_guardrail_kwargs(
kwargs: Mapping[str, object],
request_metadata: Mapping[str, object],
ops: _SurfaceOps,
guardrail_snapshot: GuardrailRequestSnapshot | None,
) -> Mapping[str, object] | None:
if guardrail_snapshot is None or guardrail_snapshot.fingerprint != request_guardrail_fingerprint(request_metadata):
return None
raw_params: Final = kwargs.get("litellm_params")
litellm_params: Final = raw_params if isinstance(raw_params, Mapping) else _EMPTY_METADATA
raw_request: Final = litellm_params.get("proxy_server_request")
request: Final = raw_request if isinstance(raw_request, Mapping) else _EMPTY_METADATA
body: Final = guardrail_snapshot.body
return MappingProxyType(
{
**kwargs,
"messages": body.get("input" if ops is _RESPONSES_OPS else "messages"),
"system": body.get("system"),
"instructions": body.get("instructions"),
"litellm_params": MappingProxyType(
{**litellm_params, "proxy_server_request": MappingProxyType({**request, "body": body})}
),
}
)
@ -808,7 +873,6 @@ class ShadowEvalLogger(CustomLogger):
await prisma.db.litellm_shadowevalattempt.group_by(
by=["job_id"],
count=True,
# mutable-ok: Prisma aggregate spec
sum={"judge_cost": True, "shadow_cost": True, "shadow_classifier_cost": True},
where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter
)
@ -836,7 +900,7 @@ class ShadowEvalLogger(CustomLogger):
{target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))}
)
await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs)
self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill
self._job_starts = {}
return jobs
except Exception as e: # noqa: BLE001 # a DB blip must never break request logging
verbose_logger.debug("shadow_eval: active-job read failed: %s", e)
@ -881,6 +945,8 @@ class ShadowEvalLogger(CustomLogger):
response_obj: object,
start_time: object,
end_time: object,
*,
guardrail_snapshot: GuardrailRequestSnapshot | None = None,
) -> None:
try:
payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") # pyright: ignore[reportAssignmentType] # untyped callback kwargs
@ -914,8 +980,13 @@ class ShadowEvalLogger(CustomLogger):
ops: Final = _SURFACE_OPS.get(str(payload.get("call_type") or ""))
if ops is None:
return # only surfaces this table can normalize are comparable; unknown types fail closed
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
sample_kwargs: Final = (
_post_guardrail_kwargs(kwargs, request_metadata, ops, guardrail_snapshot)
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata)
else kwargs
)
if sample_kwargs is None:
return
active_jobs: Final = await self._active_jobs()
eligible: Final = self._sampled_jobs(
tuple(job for target in targets for job in active_jobs.get(target, ())),
@ -927,7 +998,7 @@ class ShadowEvalLogger(CustomLogger):
return
sample: Final = _judgeable_sample(
ops,
kwargs,
sample_kwargs,
MappingProxyType(dict(payload.get("model_parameters") or {})), # mutable-ok: frozen snapshot
response_obj,
)
@ -961,7 +1032,7 @@ class ShadowEvalLogger(CustomLogger):
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=MappingProxyType(dict(request_metadata)), # mutable-ok: frozen snapshot
parent_metadata=MappingProxyType(dict(request_metadata)),
)
).add_done_callback(self._release_shadow_slot)
except Exception as e: # noqa: BLE001 # logging hooks must never fail the request
@ -1275,7 +1346,7 @@ class ShadowEvalLogger(CustomLogger):
{
"role": "user",
"content": _judge_user_prompt(conversation, response_a, response_b, _tool_definitions_text(tools)),
}, # mutable-ok: SDK message
},
]
try:
response: Final = await judge_acompletion(

4
litellm/interactions/background_cost_polling.py
View file

@ -79,9 +79,7 @@ async def _fetch_interaction(context: BackgroundInteractionPollContext) -> Inter
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
**{
"no-log": True
}, # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
**{"no-log": True},
)

2
litellm/litellm_core_utils/core_helpers.py
View file

@ -764,4 +764,4 @@ def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: flo
**(additional_headers if isinstance(additional_headers, Mapping) else _NO_HEADERS),
RESPONSE_COST_HEADER: cost,
}
hidden_params["additional_headers"] = merged # rebind-ok: the caller's record is the point
hidden_params["additional_headers"] = merged

4
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -32,9 +32,7 @@ def get_supported_openai_params(
- None if unmapped
"""
if not custom_llm_provider:
custom_llm_provider = declared_authenticating_provider(
model
) # rebind-ok: resolving would run the provider's OAuth flow
custom_llm_provider = declared_authenticating_provider(model)
if not custom_llm_provider:
try:
custom_llm_provider = litellm.get_llm_provider(model=model)[1]

32
litellm/litellm_core_utils/json_fragment_accumulator.py
View file

@ -21,20 +21,18 @@ class JSONFragmentAccumulator:
def __init__(self) -> None:
self._chunks: list[str] = [] # mutable-ok: O(1) append; string concat would copy the buffer each time
self._buffer: str = (
"" # mutable-ok: lazily materialized join of _chunks, rebuilt only when _chunks is non-empty
)
self._offset: int = 0 # mutable-ok: cursor past already-consumed values; avoids re-slicing on every pop
self._could_close: bool = False # mutable-ok: cached heuristic; rescanning past fragments was itself O(n^2)
self._buffer: str = ""
self._offset: int = 0
self._could_close: bool = False
def __bool__(self) -> bool:
return bool(self._chunks) or self._offset < len(self._buffer)
def append(self, fragment: str) -> None:
self._chunks.append(fragment) # mutable-ok: see __init__
self._chunks.append(fragment)
stripped: Final = fragment.rstrip()
if stripped:
self._could_close = stripped[-1] in ("}", "]") # mutable-ok: see __init__
self._could_close = stripped[-1] in ("}", "]")
def could_close_json(self) -> bool:
"""
@ -50,8 +48,8 @@ class JSONFragmentAccumulator:
if not self._chunks:
return
unconsumed: Final = self._buffer[self._offset :]
self._buffer = unconsumed + "".join(self._chunks) # mutable-ok: merge pending fragments, once per append batch
self._offset = 0 # mutable-ok: see __init__
self._buffer = unconsumed + "".join(self._chunks)
self._offset = 0
self._chunks = [] # mutable-ok: see __init__
def pop_next_value(self) -> tuple[bool, object]:
@ -69,7 +67,7 @@ class JSONFragmentAccumulator:
while start < length and self._buffer[start].isspace():
start += 1
if start >= length:
self._offset = start # mutable-ok: see __init__
self._offset = start
return False, None
decoder: Final = json.JSONDecoder()
try:
@ -77,11 +75,11 @@ class JSONFragmentAccumulator:
except json.JSONDecodeError:
return False, None
decoded, end_index = cast("tuple[object, int]", raw_value) # cast-ok: raw_decode returns tuple[Any, int]
self._offset = end_index # mutable-ok: see __init__
self._offset = end_index
if self._offset >= len(self._buffer):
self._buffer = "" # mutable-ok: see __init__
self._offset = 0 # mutable-ok: see __init__
self._could_close = False # mutable-ok: buffer is empty, nothing can close
self._buffer = ""
self._offset = 0
self._could_close = False
return True, decoded
def snapshot(self) -> str:
@ -91,7 +89,7 @@ class JSONFragmentAccumulator:
def set(self, value: str) -> None:
"""Replace the buffer's contents with a single fragment."""
self._chunks = [] # mutable-ok: see __init__
self._buffer = value # mutable-ok: see __init__
self._offset = 0 # mutable-ok: see __init__
self._buffer = value
self._offset = 0
stripped: Final = value.rstrip()
self._could_close = bool(stripped) and stripped[-1] in ("}", "]") # mutable-ok: see __init__
self._could_close = bool(stripped) and stripped[-1] in ("}", "]")

22
litellm/litellm_core_utils/litellm_logging.py
View file

@ -226,6 +226,7 @@ if TYPE_CHECKING:
from litellm.integrations.otel.logger import OpenTelemetryV2
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.shadow_eval_logger import GuardrailRequestSnapshot
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
@ -714,6 +715,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._defer_async_logging: bool = False
self._enqueue_deferred_logging: Callable[[], None] | None = None
self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
@ -2825,6 +2827,7 @@ class Logging(LiteLLMLoggingBaseClass):
):
continue
self.shadow_eval_request_snapshot = None
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
result=result,
@ -3391,6 +3394,7 @@ class Logging(LiteLLMLoggingBaseClass):
):
continue
self.shadow_eval_request_snapshot = None
self.model_call_details, result = await callback.async_logging_hook(
kwargs=self.model_call_details,
result=result,
@ -3450,6 +3454,8 @@ class Logging(LiteLLMLoggingBaseClass):
)
if isinstance(callback, CustomLogger): # custom logger class
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
model_call_details: dict = self.model_call_details
##################################
# call redaction hook for custom logger
@ -3460,7 +3466,19 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=model_call_details, custom_logger=callback
)
##################################
if self.stream is True:
if isinstance(callback, ShadowEvalLogger) and (
not self.stream or "async_complete_streaming_response" in model_call_details
):
await callback.async_log_success_event(
kwargs=model_call_details,
response_obj=model_call_details["async_complete_streaming_response"]
if self.stream
else result,
start_time=start_time,
end_time=end_time,
guardrail_snapshot=self.shadow_eval_request_snapshot,
)
elif self.stream is True:
if "async_complete_streaming_response" in model_call_details:
await callback.async_log_success_event(
kwargs=model_call_details,
@ -6649,7 +6667,7 @@ def get_standard_logging_object_payload(
"version": 3,
"status": "unknown",
"reason": "pending_projection",
} # mutable-ok: spend-log JSON serialization requires plain mappings
}
if captured_baseline is not None
else (
{ # mutable-ok: spend-log JSON serialization requires plain mappings

4
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -372,9 +372,7 @@ from collections import defaultdict
def _handle_invalid_parallel_tool_calls(
tool_calls: list[
ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall
], # mutable-ok: patched in place via slice assignment
tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall],
):
"""
Handle hallucinated parallel tool call from openai - https://community.openai.com/t/model-tries-to-call-unknown-function-multi-tool-use-parallel/490653

4
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -208,7 +208,7 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool:
for _ in range(_IMAGE_SCAN_MAX_DEPTH):
if any(isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_PART_TYPES for part in frontier):
return True
frontier = tuple( # rebind-ok: depth-bounded frontier walk
frontier = tuple(
nested
for part in frontier
if isinstance(part, Mapping)
@ -2020,7 +2020,7 @@ def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
def _strip_encrypted_reasoning_from_blocks(content: object) -> None:
blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance
kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block))
blocks[:] = kept # rebind-ok: shared with fallback snapshot
blocks[:] = kept
def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:

2
litellm/litellm_core_utils/provider_affinity.py
View file

@ -83,7 +83,7 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
return None
def add_provider_affinity_header( # mutable-ok: downstream handlers add auth and signing headers
def add_provider_affinity_header(
headers: Mapping[str, object], litellm_params: object | None
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
header_name: Final = _get_provider_affinity_header_name(litellm_params)

4
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -475,9 +475,7 @@ class ChunkProcessor:
def get_combined_tool_content(
self, tool_call_chunks: Sequence["_ToolCallChunk"]
) -> list[
ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall
]: # mutable-ok: assigned verbatim to Message.tool_calls, a list field
) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall]:
tool_calls_list: list[
ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall
] = [] # mutable-ok: see return type

44
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -199,9 +199,7 @@ def _write_back_system_block(system: object, block_idx: int, response: str) -> N
return
text_blocks: Final = tuple(block for block in system if isinstance(block, dict) and block.get("type") == "text")
if block_idx < len(text_blocks):
text_blocks[block_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
text_blocks[block_idx]["text"] = response
def _write_back_message_text(message: _WritableMessage, target: MessageTextTarget, response: str) -> None:
@ -211,22 +209,16 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
match target:
case MessageContentTarget():
if isinstance(content, str):
message["content"] = response # mutable-ok: guardrails rewrite the caller's request payload in place
message["content"] = response
case ContentBlockTextTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
content[content_idx]["text"] = response
case ToolResultStringTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["content"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
content[content_idx]["content"] = response
case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
if isinstance(content, list):
content[content_idx]["content"][block_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
content[content_idx]["content"][block_idx]["text"] = response
case _:
assert_never(target)
@ -248,9 +240,9 @@ def _write_back_tool_use(
block: Final = content[target.content_idx] if isinstance(content, list) else None
if not isinstance(block, dict):
return
block["input"] = rewritten_input # mutable-ok: guardrails rewrite the caller's request payload in place
block["input"] = rewritten_input
if shape.name is not None and shape.name != block.get("name"):
block["name"] = shape.name # mutable-ok: guardrails rewrite the caller's request payload in place
block["name"] = shape.name
@dataclass(frozen=True, slots=True)
@ -603,13 +595,9 @@ class AnthropicMessagesHandler(BaseTranslation):
*(item for one_message in extracted for item in one_message.scanned),
)
texts_to_check: Final = [item.text for item in scanned] # mutable-ok: GenericGuardrailAPIInputs takes list[str]
images_to_check: Final = [
image for one_message in extracted for image in one_message.images
] # mutable-ok: GenericGuardrailAPIInputs takes list[str]
images_to_check: Final = [image for one_message in extracted for image in one_message.images]
scanned_tool_calls: Final = tuple(item for one_message in extracted for item in one_message.tool_calls)
tool_calls_to_check: Final = [
item.tool_call for item in scanned_tool_calls
] # mutable-ok: GenericGuardrailAPIInputs takes list[ChatCompletionToolCallChunk]
tool_calls_to_check: Final = [item.tool_call for item in scanned_tool_calls]
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check)
# Step 2: Apply guardrail to all texts and tool calls in batch
@ -697,9 +685,7 @@ class AnthropicMessagesHandler(BaseTranslation):
return data
def _hoisted_top_level_system_message(
self, data: dict
) -> AllMessageValues | None: # mutable-ok: API message payload
def _hoisted_top_level_system_message(self, data: dict) -> AllMessageValues | None:
"""Return the system message produced by translating the top-level prompt."""
system: Final = data.get("system")
if not system:
@ -736,7 +722,7 @@ class AnthropicMessagesHandler(BaseTranslation):
if isinstance(content, str):
return (
{"role": "system", "content": content} if content else None # mutable-ok: API message payload
) # mutable-ok: API message payload
)
if not isinstance(content, list):
return None
blocks: Final[list[dict[str, object]]] = [] # mutable-ok: API message payload
@ -749,14 +735,14 @@ class AnthropicMessagesHandler(BaseTranslation):
anthropic_block: dict[str, object] = { # mutable-ok: API message payload
"type": "text",
"text": text,
} # mutable-ok: API message payload
}
cache_control = block.get("cache_control")
if cache_control:
anthropic_block["cache_control"] = deepcopy(cache_control)
blocks.append(anthropic_block)
return (
{"role": "system", "content": blocks} if blocks else None # mutable-ok: API message payload
) # mutable-ok: API message payload
)
@staticmethod
def _fold_leading_systems_into_top_level(
@ -1098,9 +1084,7 @@ class AnthropicMessagesHandler(BaseTranslation):
match item.target:
case SystemStringTarget():
if isinstance(data.get("system"), str):
data["system"] = (
guardrail_response # mutable-ok: guardrails rewrite the caller's request payload in place
)
data["system"] = guardrail_response
case SystemBlockTextTarget(block_idx=block_idx):
_write_back_system_block(data.get("system"), block_idx, guardrail_response)
case (

2
litellm/llms/anthropic/common_utils.py
View file

@ -1591,7 +1591,7 @@ def _flatten_web_search_results_in_message(message: object) -> object:
return {**message, "content": [b for b in rewritten if b is not None]} # mutable-ok: JSON wire format
def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: as sibling sanitizers
def flatten_unencrypted_web_search_results_in_anthropic_messages(
messages: list[Any],
) -> list[Any]:
"""

4
litellm/llms/anthropic/experimental_pass_through/messages/mcp_handler.py
View file

@ -50,14 +50,14 @@ def _build_tool_result_message(tool_results: Sequence[Mapping[str, object]]) ->
"""Turn executed tool results into the user message Anthropic expects."""
return AnthropicMessagesUserMessageParam(
role="user",
content=tuple(
content=[
AnthropicMessagesToolResultParam(
type="tool_result",
tool_use_id=str(result.get("tool_call_id") or ""),
content=str(result.get("result") or ""),
)
for result in tool_results
),
],
)

4
litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py
View file

@ -88,9 +88,7 @@ class AnthropicMessagesStreamCacheWriter:
try:
events: Final = _split_sse_events(collected_stream.decode("utf-8"))
cached_payload: Final = {
CACHED_STREAM_EVENTS_KEY: events
} # mutable-ok: cache backends serialize plain dicts
cached_payload: Final = {CACHED_STREAM_EVENTS_KEY: events}
await litellm.cache.async_add_cache(
cached_payload,
dynamic_cache_object=self.caching_handler.dual_cache,

2
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -186,7 +186,7 @@ def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
def _incomplete_stream_error_sse_event() -> bytes:
return _sse_event( # mutable-ok: one-shot JSON payload, never mutated after construction
return _sse_event(
"error",
{"type": "error", "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}},
)

6
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -148,13 +148,11 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(content, str):
return (
[{"type": "input_text", "text": content}] if content else [] # mutable-ok: API message payload
) # mutable-ok: API message payload
)
if not isinstance(content, list):
return [] # mutable-ok: API message payload
return [ # mutable-ok: API message payload
with_prompt_cache_breakpoint(
{"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint")
) # mutable-ok: API message payload
with_prompt_cache_breakpoint({"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint"))
for block in content
if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload
]

4
litellm/llms/azure/passthrough/transformation.py
View file

@ -59,9 +59,7 @@ def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) ->
terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks=all_chunks)
if terminal_event is None:
return None
logging_obj.call_type = (
RESPONSES_RELAY_SHAPE.call_type.value
) # rebind-ok: routes cost calculation to the relayed shape's pricing path
logging_obj.call_type = RESPONSES_RELAY_SHAPE.call_type.value
return terminal_event

4
litellm/llms/azure_ai/image_generation/flux_transformation.py
View file

@ -73,9 +73,7 @@ class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig):
normalized_model: Final = model.lower().replace(".", "-").replace("_", "-")
return "flux-2-flex" if "flux-2-flex" in normalized_model else "flux-2-pro"
def get_supported_openai_params( # mutable-ok: inherited config contract returns a list
self, model: str
) -> list[OpenAIImageGenerationOptionalParams]:
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
if not self.is_flux2_model(model):
return super().get_supported_openai_params(model)
return [ # mutable-ok: BaseImageGenerationConfig requires a list

4
litellm/llms/base_llm/passthrough/transformation.py
View file

@ -95,9 +95,7 @@ def logged_relay_shape(
parsed: Final = shape.parse(body)
except ValidationError:
return None
logging_obj.call_type = (
shape.call_type.value
) # rebind-ok: routes cost calculation to the relayed shape's pricing path
logging_obj.call_type = shape.call_type.value
return parsed

154
litellm/llms/base_llm/responses/codex_compat.py Normal file
View file

@ -0,0 +1,154 @@
"""Codex CLI wire-format quirks shared by the Responses API providers that need them.
Codex sends history item types that api.openai.com accepts but other Responses
backends reject with ``400 Invalid 'input': value did not match any expected
variant``. Both Amazon Bedrock endpoints reject them:
- ``bedrock-mantle.{region}.api.aws`` (verified against ``openai.gpt-5.6-sol``)
- ``bedrock-runtime.{region}.amazonaws.com/openai/v1`` (same, verified separately)
They are *history* items, so they only appear from the second turn of a session
onward -- a first-turn request succeeds and hides the problem entirely.
Codex also sends a ``web_search`` tool on every turn. api.openai.com runs that tool
itself; a backend with no server-side tools rejects the whole request over it, so
the same providers drop the tool types their backend does not accept.
Both helpers are pure transforms that report what they rewrote or dropped; callers
do their own logging, so each provider keeps its own wording.
"""
import json
from collections.abc import Mapping, Sequence
from typing import Final
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponseInputParam
AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
class _RewrittenOutputTextBlock(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
class _RewrittenAssistantMessageItem(TypedDict):
type: ReadOnly[str]
role: ReadOnly[str]
content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
class _RewrittenCompactionItem(TypedDict):
type: ReadOnly[str]
encrypted_content: ReadOnly[str]
class _RewrittenFunctionCallItem(TypedDict):
type: ReadOnly[str]
call_id: ReadOnly[str]
name: ReadOnly[str]
arguments: ReadOnly[str]
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
return "".join(
str(block.get("text") or block.get("encrypted_content") or "") for block in content if isinstance(block, dict)
)
def _normalize_agent_message_item(item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = _agent_message_text(item)
if not text:
return None
rewritten: Final[_RewrittenAssistantMessageItem] = {
"type": "message",
"role": "assistant",
"content": ({"type": "output_text", "text": text},),
}
return rewritten
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
return rewritten
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None
action: Final = item.get("action")
rewritten: Final[_RewrittenFunctionCallItem] = {
"type": "function_call",
"call_id": call_id,
"name": "local_shell",
"arguments": json.dumps(action) if isinstance(action, dict) else "{}",
}
return rewritten
def _normalize_input_item(item: object) -> "tuple[object, str | None]":
"""Returns (normalized item, or None to drop it; original type when rewritten)."""
if not isinstance(item, dict):
return item, None
item_type: Final = item.get("type")
if item_type == AGENT_MESSAGE_INPUT_ITEM_TYPE:
return _normalize_agent_message_item(item), item_type
if item_type == CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
return _normalize_context_compaction_item(item), item_type
if item_type == LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
return _normalize_local_shell_call_item(item), item_type
return item, None
def normalize_codex_input_items(
input: "str | ResponseInputParam",
) -> "tuple[str | ResponseInputParam, tuple[str, ...]]":
"""Rewrite the Codex history item types a Responses backend rejects.
``agent_message`` (Codex multi-agent traffic; its ``encrypted_content`` slot
carries the plaintext payload when the model never issued encrypted args)
becomes an assistant message, ``context_compaction`` becomes the ``compaction``
spelling these backends accept, and ``local_shell_call`` becomes the
``function_call`` its recorded ``function_call_output`` already pairs with.
Returns the normalized input and the sorted set of types that were rewritten,
so the caller can log in its own words. Non-list input is returned untouched.
"""
if not isinstance(input, list):
return input, ()
normalized: Final = tuple(_normalize_input_item(item) for item in input)
rewritten_types: Final = tuple(sorted(frozenset(item_type for _, item_type in normalized if item_type is not None)))
kept: Final = [i for i, _ in normalized if i is not None] # mutable-ok: downstream narrows on isinstance(list)
# Codex passthrough items sit outside the OpenAI input union.
return kept, rewritten_types # pyright: ignore[reportReturnType] # see above
def drop_unsupported_tools(
tools: "Sequence[object]", supported_types: "frozenset[str]"
) -> "tuple[tuple[object, ...], tuple[str, ...]]":
"""Keep the tools whose ``type`` the backend accepts; non-dict tools pass through.
Returns the kept tools and the sorted set of dropped types.
"""
kept: Final = tuple(tool for tool in tools if not isinstance(tool, dict) or tool.get("type") in supported_types)
dropped_types: Final = tuple(
sorted(
frozenset(
str(tool.get("type"))
for tool in tools
if isinstance(tool, dict) and tool.get("type") not in supported_types
)
)
)
return kept, dropped_types

16
litellm/llms/base_llm/responses/transformation.py
View file

@ -130,6 +130,22 @@ class BaseResponsesAPIConfig(ABC):
) -> dict:
pass
async def async_transform_responses_api_request(
self,
model: str,
input: str | ResponseInputParam,
response_api_optional_request_params: dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
return self.transform_responses_api_request(
model=model,
input=input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
@abstractmethod
def transform_response_api_response(
self,

22
litellm/llms/bedrock/common_utils.py
View file

@ -827,6 +827,28 @@ def _mantle_api_base_from_env() -> str | None:
return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base)
def bedrock_supports_openai_responses(model: str | None, model_cost: Mapping[str, object]) -> bool:
"""Whether a Bedrock model is served by bedrock-runtime's OpenAI Responses surface.
Purely data-driven from the model's price-map capability signal -- ``/v1/responses``
in ``supported_endpoints`` -- and overridable via ``register_model`` and proxy
``model_info``, so onboarding a model is a JSON change, never a code change.
There is deliberately no model-name match: AWS exposes this surface per model,
not per family, and the two Bedrock endpoints do not agree with each other
(bedrock-runtime accepts Codex's ``additional_tools`` items where
bedrock-mantle rejects them), so a name-shaped gate would be wrong.
A model absent from ``model_cost`` has no signal and returns False, leaving the
chat-completions bridge in place exactly as before.
"""
if not model:
return False
candidates: Final = (model_cost.get(key) for key in (model, f"bedrock/{model}"))
return any(
isinstance(entry, Mapping) and "/v1/responses" in (entry.get("supported_endpoints") or ())
for entry in candidates
)
def build_mantle_messages_url(
api_base: str | None,
aws_bedrock_runtime_endpoint: str | None,

2
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -45,7 +45,7 @@ def _move_betas_into_header(request: Mapping[str, object], headers: dict[str, st
if betas:
headers["anthropic-beta"] = ",".join(betas) # rebind-ok: the handler signs and sends this same dict
return
headers.pop("anthropic-beta", None) # rebind-ok: a caller header Mantle rejects in full must not reach it
headers.pop("anthropic-beta", None)
class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):

4
litellm/llms/bedrock/realtime/handler.py
View file

@ -489,9 +489,7 @@ class BedrockRealtime(BaseAWSLLM):
parsed_client_message = _parse_client_message(message)
is_session_update = _json_str(parsed_client_message.get("type")) == "session.update"
if is_session_update:
client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = (
message # rebind-ok: scope outlives the attempt
)
client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = message
transformed_messages = transformation_config.transform_realtime_request(
message=message,

338
litellm/llms/bedrock/responses/transformation.py Normal file
View file

@ -0,0 +1,338 @@
"""Amazon Bedrock Runtime - native OpenAI Responses API.
AWS serves the OpenAI models on ``bedrock-runtime`` through an OpenAI-compatible
surface at ``https://bedrock-runtime.{region}.{dns_suffix}/openai/v1/responses``,
alongside Converse. Without this config the ``bedrock`` provider has no Responses
config at all, so ``/v1/responses`` falls back to the Chat Completions bridge and
the request is translated into Converse, which rejects Responses-only parameters
such as ``prompt_cache_key`` with a 400 and never sees reasoning items.
Payloads and SSE follow the OpenAI Responses spec, so this inherits
OpenAIResponsesAPIConfig and overrides only the endpoint URL, authentication, the
Codex history-item normalization the endpoint requires, and the tool filter below.
Tools: bedrock-runtime runs no server-side tools, so it rejects Codex's default
``web_search`` tool with "web search is not supported for this request". The
Converse bridge dropped that tool silently (Converse has no web search either),
so this config drops every tool type the endpoint rejects the same way. The
supported set is the one bedrock-runtime's own validation error names.
Parity with the Converse bridge on what it used to accept: ``background`` never
reached Converse (the bridge answered synchronously), while bedrock-runtime rejects
it with "The background parameter is not supported.", so it is dropped here. The
bridge also downloaded ``input_image`` http(s) URLs for Converse, while
bedrock-runtime only accepts ``data:`` and ``s3://`` image URLs, so remote image
URLs are fetched and inlined as data URIs before the request is signed.
Auth: Bearer token (litellm_params.api_key or the standard AWS_BEARER_TOKEN_BEDROCK)
when present; otherwise AWS SigV4 (service "bedrock") over the standard credential
chain, signed via BaseAWSLLM._sign_request once the body is final.
Model IDs: bedrock-runtime serves these models only through a cross-Region
inference profile, so the model is named ``us.openai.gpt-5.6-sol`` or
``global.openai.gpt-5.6-sol``; there is no in-Region form.
"""
import asyncio
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import Final
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import (
BedrockError,
bedrock_supports_openai_responses,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH: Final = "/openai/v1/responses"
BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES: Final = (
"/openai/v1/responses",
"/v1/responses",
"/responses",
"/openai/v1",
"/v1",
)
BEDROCK_RUNTIME_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
{"function", "mcp", "custom", "apply_patch", "namespace", "tool_search", "computer"}
)
BEDROCK_RUNTIME_UNSUPPORTED_RESPONSE_PARAMS: Final = frozenset({"background"})
REMOTE_IMAGE_URL_SCHEMES: Final = ("http://", "https://")
IMAGE_BLOCK_KEYS: Final = ("content", "output")
IMAGE_BLOCK_TYPES: Final = frozenset({"input_image", "computer_screenshot"})
def resolve_bedrock_bearer_token(api_key: str | None) -> str | None:
return api_key or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
def _remote_image_url(block: object) -> str | None:
if not isinstance(block, dict) or block.get("type") not in IMAGE_BLOCK_TYPES:
return None
image_url: Final = block.get("image_url")
if not isinstance(image_url, str) or not image_url.startswith(REMOTE_IMAGE_URL_SCHEMES):
return None
return image_url
def _blocks_under(value: object) -> "tuple[object, ...]":
if isinstance(value, list):
return tuple(value)
if isinstance(value, dict):
return (value,)
return ()
def _image_blocks(item: object) -> "tuple[object, ...]":
"""The blocks of ``item`` that can carry an image: its content and tool output lists, or a screenshot output dict."""
if not isinstance(item, dict):
return ()
return tuple(block for key in IMAGE_BLOCK_KEYS for block in _blocks_under(item.get(key)))
def collect_remote_image_urls(input: "str | ResponseInputParam") -> "tuple[str, ...]":
"""The distinct http(s) image URLs in message content, tool output lists, and computer screenshots, in first-seen order."""
if not isinstance(input, list):
return ()
return tuple(
dict.fromkeys(
url for item in input for block in _image_blocks(item) if (url := _remote_image_url(block)) is not None
)
)
def _inline_block(block: object, inlined: "Mapping[str, str]") -> object:
url: Final = _remote_image_url(block)
if url is None or not isinstance(block, dict):
return block
return {**block, "image_url": inlined[url]} # mutable-ok: outgoing JSON request item
def _inline_value(value: object, inlined: "Mapping[str, str]") -> object:
if isinstance(value, list):
return [_inline_block(block, inlined) for block in value] # mutable-ok: outgoing JSON request item
return _inline_block(value, inlined)
def _inline_item(item: object, inlined: "Mapping[str, str]") -> object:
if not isinstance(item, dict):
return item
inlined_fields: Final = { # mutable-ok: outgoing JSON request item
key: _inline_value(item[key], inlined) for key in IMAGE_BLOCK_KEYS if isinstance(item.get(key), (list, dict))
}
if not inlined_fields:
return item
return {**item, **inlined_fields} # mutable-ok: same
def inline_remote_image_urls(
input: "str | ResponseInputParam", inlined: "Mapping[str, str]"
) -> "str | ResponseInputParam":
"""``input`` with every http(s) image URL replaced by its entry in ``inlined``."""
if not isinstance(input, list) or not inlined:
return input
items: Final = [_inline_item(item, inlined) for item in input] # mutable-ok: downstream narrows on isinstance(list)
return items # pyright: ignore[reportReturnType] # items keep the caller's input union
class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
"""Responses API config for the OpenAI models on the bedrock-runtime endpoint."""
def __init__(
self,
fetch_image: "Callable[[str], str]" = convert_url_to_base64,
async_fetch_image: "Callable[[str], Awaitable[str]]" = async_convert_url_to_base64,
) -> None:
super().__init__()
self.fetch_image = fetch_image
self.async_fetch_image = async_fetch_image
@classmethod
def for_model(cls, model: str | None) -> "BedrockOpenAIResponsesConfig | None":
"""This config when ``model`` is served on the OpenAI Responses surface, else ``None``.
The capability decision lives here rather than in the shared dispatch so that
onboarding a model, or changing how the signal is read, stays inside the
Bedrock adapter. ``None`` leaves the caller's existing behaviour untouched --
chat-only Bedrock models keep the Chat Completions bridge.
"""
if not bedrock_supports_openai_responses(model, litellm.model_cost):
return None
return cls()
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK
def get_error_class(
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
) -> BaseLLMException:
# The OpenAI base builds a blank response, dropping x-amzn-RequestId.
return BedrockError(status_code=status_code, message=error_message, headers=headers)
def get_complete_url(
self,
api_base: str | None,
litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
) -> str:
region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None)
override: Final = (
api_base
or litellm_params.get("aws_bedrock_runtime_endpoint")
or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
)
# Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other
# suffixes in GovCloud/ISO, so defer to the shared endpoint builder.
host: Final = (
override or self._select_default_endpoint_url(endpoint_type="runtime", aws_region_name=region)
).rstrip("/")
base: Final = next(
(host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)),
host,
)
return f"{base}{BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH}"
def supports_native_file_search(self) -> bool:
return False
def validate_environment(
self,
headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
model: str,
litellm_params: GenericLiteLLMParams | None,
) -> dict: # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
api_key: Final = litellm_params.api_key if litellm_params is not None else None
bearer: Final = resolve_bedrock_bearer_token(api_key)
if not bearer:
return headers
return {**headers, "Authorization": f"Bearer {bearer}"} # mutable-ok: dict return per the contract
def sign_request(
self,
headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
optional_params: dict, # mutable-ok: same
request_data: dict, # mutable-ok: same
api_base: str,
api_key: str | None = None,
model: str | None = None,
stream: bool | None = None,
fake_stream: bool | None = None,
) -> "tuple[dict, bytes | None]": # mutable-ok: signature fixed by the override contract
if resolve_bedrock_bearer_token(api_key):
# Bedrock API keys are Bearer credentials; SigV4 on top would be wrong.
return headers, None
return self._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
model=model,
stream=stream,
fake_stream=fake_stream,
)
def map_openai_params(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict: # mutable-ok: signature fixed by the override contract
mapped: Final = super().map_openai_params(
response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params
)
unsupported: Final = tuple(sorted(BEDROCK_RUNTIME_UNSUPPORTED_RESPONSE_PARAMS & mapped.keys()))
if unsupported:
verbose_logger.warning(
"Bedrock Runtime Responses API: dropping unsupported parameter(s) %s that the endpoint rejects.",
unsupported,
)
params: Final = { # mutable-ok: outgoing JSON request params
key: value for key, value in mapped.items() if key not in unsupported
}
tools: Final = params.get("tools")
if not isinstance(tools, list):
return params
kept, dropped_types = drop_unsupported_tools(tools, BEDROCK_RUNTIME_SUPPORTED_RESPONSE_TOOL_TYPES)
if not dropped_types:
return params
verbose_logger.warning(
"Bedrock Runtime Responses API: dropping unsupported tool type(s) %s (supported: %s).",
list(dropped_types),
sorted(BEDROCK_RUNTIME_SUPPORTED_RESPONSE_TOOL_TYPES),
)
without_tools: Final = {key: value for key, value in params.items() if key != "tools"}
if not kept:
return without_tools
return {**without_tools, "tools": list(kept)}
def transform_responses_api_request(
self,
model: str,
input: "str | ResponseInputParam",
response_api_optional_request_params: dict, # mutable-ok: signature fixed by the override contract
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: same
) -> dict: # mutable-ok: same
inlined: Final = MappingProxyType({url: self.fetch_image(url) for url in collect_remote_image_urls(input)})
return self._transform_inlined_request(
model=model,
input=inline_remote_image_urls(input, inlined),
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
async def async_transform_responses_api_request(
self,
model: str,
input: "str | ResponseInputParam",
response_api_optional_request_params: dict, # mutable-ok: signature fixed by the override contract
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: same
) -> dict: # mutable-ok: same
remote_urls: Final = collect_remote_image_urls(input)
data_uris: Final = await asyncio.gather(*(self.async_fetch_image(url) for url in remote_urls))
return self._transform_inlined_request(
model=model,
input=inline_remote_image_urls(input, MappingProxyType(dict(zip(remote_urls, data_uris, strict=True)))),
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
def _transform_inlined_request(
self,
model: str,
input: "str | ResponseInputParam",
response_api_optional_request_params: dict, # mutable-ok: signature fixed by the override contract
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: same
) -> dict: # mutable-ok: same
normalized_input, rewritten_types = normalize_codex_input_items(input)
if rewritten_types:
verbose_logger.warning(
"Bedrock Runtime Responses API: rewrote Codex input item type(s) %s that the endpoint rejects.",
rewritten_types,
)
return super().transform_responses_api_request(
model=model,
input=normalized_input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)

140
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -15,16 +15,15 @@ role / access key / profile / web identity), signed via the shared
BaseAWSLLM._sign_request after the request body is finalized.
"""
import json
from collections.abc import Mapping, Sequence
from typing import Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock_mantle.common_utils import (
@ -59,33 +58,6 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
_BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
_BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
class _RewrittenOutputTextBlock(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
class _RewrittenAssistantMessageItem(TypedDict):
type: ReadOnly[str]
role: ReadOnly[str]
content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
class _RewrittenCompactionItem(TypedDict):
type: ReadOnly[str]
encrypted_content: ReadOnly[str]
class _RewrittenFunctionCallItem(TypedDict):
type: ReadOnly[str]
call_id: ReadOnly[str]
name: ReadOnly[str]
arguments: ReadOnly[str]
class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
def __init__(
@ -144,26 +116,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
@staticmethod
def _filter_unsupported_tools(tools: "Sequence[object]") -> "list[object]":
"""Keep only tool types Mantle's Responses API accepts."""
kept: Final[list[object]] = []
dropped_types: Final[list[str]] = []
for tool in tools:
if not isinstance(tool, dict):
kept.append(tool)
continue
tool_type = tool.get("type")
if tool_type in _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES:
kept.append(tool)
else:
dropped_types.append(str(tool_type))
kept, dropped_types = drop_unsupported_tools(tools, _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES)
if dropped_types:
verbose_logger.warning(
"Bedrock Mantle Responses API: dropping unsupported tool type(s) %s (supported: %s).",
sorted(set(dropped_types)),
list(dropped_types),
sorted(_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES),
)
return kept
return list(kept)
@staticmethod
def _handle_unsupported_service_tier(params: dict, drop_params: bool) -> dict:
@ -236,7 +196,12 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
if rewritten_types:
verbose_logger.warning(
"Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
list(rewritten_types),
)
request_params: Final = (
self._params_with_hoisted_tools(params, hoisted)
if hoisted.hoisted
@ -259,91 +224,6 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return {**params, "tools": supported_tools}
return {key: value for key, value in params.items() if key != "tools"}
@staticmethod
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
return "".join(
str(block.get("text") or block.get("encrypted_content") or "")
for block in content
if isinstance(block, dict)
)
@classmethod
def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = cls._agent_message_text(item)
if not text:
return None
rewritten: Final[_RewrittenAssistantMessageItem] = {
"type": "message",
"role": "assistant",
"content": ({"type": "output_text", "text": text},),
}
return rewritten
@staticmethod
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
return rewritten
@staticmethod
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None
action: Final = item.get("action")
rewritten: Final[_RewrittenFunctionCallItem] = {
"type": "function_call",
"call_id": call_id,
"name": "local_shell",
"arguments": json.dumps(action) if isinstance(action, dict) else "{}",
}
return rewritten
@classmethod
def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
"""Returns (normalized item or None to drop it, original type when rewritten)."""
if not isinstance(item, dict):
return item, None
item_type: Final = item.get("type")
if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
return cls._normalize_agent_message_item(item), item_type
if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
return cls._normalize_context_compaction_item(item), item_type
if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
return cls._normalize_local_shell_call_item(item), item_type
return item, None
@classmethod
def _normalize_codex_input_items(
cls,
input: "str | ResponseInputParam",
) -> "str | ResponseInputParam":
"""Rewrite Codex history item types Mantle rejects with 400 "Invalid
'input': value did not match any expected variant" into supported
equivalents. `agent_message` (Codex multi-agent traffic; its
encrypted_content slot carries the plaintext payload when the model
never issued encrypted args) becomes an assistant message,
`context_compaction` becomes the `compaction` spelling Mantle accepts,
and `local_shell_call` becomes the function_call its recorded
function_call_output already pairs with.
"""
if not isinstance(input, list):
return input
normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
if rewritten_types:
verbose_logger.warning(
"Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
rewritten_types,
)
kept: Final = [item for item, _ in normalized if item is not None] # mutable-ok: ResponseInputParam is a list
return kept # pyright: ignore[reportReturnType] # Codex passthrough items sit outside the OpenAI input union
@staticmethod
def _model_map_lookup_name(model: str) -> str:
return model.split("/")[-1].removeprefix("openai.")

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2881,7 +2881,7 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
data = responses_api_provider_config.transform_responses_api_request(
data = await responses_api_provider_config.async_transform_responses_api_request(
model=model,
input=input,
response_api_optional_request_params=response_api_optional_request_params,

6
litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py
View file

@ -27,7 +27,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a list
return list(SUPPORTED_OPENAI_PARAMS) # mutable-ok: base class contract returns a list
def map_openai_params( # mutable-ok: base class contract returns a dict
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
@ -63,9 +63,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
if len(images) > 1:
raise ValueError(f"{FLUX_LORA_DEPTH_ENDPOINT} accepts exactly one control image")
provider_params: Final[Mapping[str, object]] = MappingProxyType(
{
key: value for key, value in image_edit_optional_request_params.items() if key != "mask"
} # mutable-ok: frozen by MappingProxyType
{key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
)
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
"prompt": prompt,

6
litellm/llms/fal_ai/image_edit/transformation.py
View file

@ -84,7 +84,7 @@ class FalAIImageEditConfig(BaseImageEditConfig):
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a list
return list(SUPPORTED_OPENAI_PARAMS) # mutable-ok: base class contract returns a list
def map_openai_params( # mutable-ok: base class contract returns a dict
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
@ -146,9 +146,7 @@ class FalAIImageEditConfig(BaseImageEditConfig):
MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
)
provider_params: Final[Mapping[str, object]] = MappingProxyType(
{
key: value for key, value in image_edit_optional_request_params.items() if key != "mask"
} # mutable-ok: frozen by MappingProxyType
{key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
)
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
"prompt": prompt,

8
litellm/llms/fal_ai/image_generation/gpt_image_2_transformation.py
View file

@ -101,12 +101,10 @@ class FalAIGPTImage2Config(FalAIBaseConfig):
endpoint: Final[str] = model if model.startswith(self.MODEL_PREFIX) else f"{self.MODEL_PREFIX}{model}"
return f"{base_url}/{endpoint}"
def get_supported_openai_params( # mutable-ok: base class contract returns a list
self, model: str
) -> list[OpenAIImageGenerationOptionalParams]:
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
return list(SUPPORTED_OPENAI_PARAMS) # mutable-ok: base class contract returns a list
def map_openai_params( # mutable-ok: base class contract returns a dict
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
@ -138,7 +136,7 @@ class FalAIGPTImage2Config(FalAIBaseConfig):
return map_gpt_image_quality(value, model)
return value
def transform_image_generation_request( # mutable-ok: base class contract returns a dict
def transform_image_generation_request(
self,
model: str,
prompt: str,

2
litellm/llms/gigachat/authenticator.py
View file

@ -243,7 +243,7 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
)
# expires_at is in milliseconds
expires_at: int # rebind-ok: conditionally assigned from str or int
expires_at: int
if isinstance(expires_at_raw, str):
expires_at = int(expires_at_raw) # rebind-ok: conditionally assigned from str or int
else:

8
litellm/llms/gigachat/chat/streaming.py
View file

@ -30,7 +30,7 @@ class GigaChatModelResponseIterator:
def chunk_parser(self, chunk: Mapping[str, object]) -> GenericStreamingChunk:
"""Parse a single streaming chunk from GigaChat."""
choices: Sequence = chunk.get("choices") or () # mutable-ok: tuple literal as default
choices: Sequence = chunk.get("choices") or ()
if not choices:
return GenericStreamingChunk(
text="",
@ -56,7 +56,7 @@ class GigaChatModelResponseIterator:
if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call:
func_call: Final[Mapping[str, object]] = raw_function_call
args_raw: Final[object] = func_call.get("arguments") or {}
args_str: str # rebind-ok: conditionally assigned from dict or str
args_str: str
if isinstance(args_raw, dict):
args_str = json.dumps(args_raw, ensure_ascii=False) # rebind-ok: build from dict
else:
@ -80,10 +80,10 @@ class GigaChatModelResponseIterator:
usage = convert_usage(validated_usage)
_prompt_details: dict | None = (
usage.prompt_tokens_details.model_dump() if usage.prompt_tokens_details else None
) # rebind-ok: conditional
)
_completion_details: dict | None = (
usage.completion_tokens_details.model_dump() if usage.completion_tokens_details else None
) # rebind-ok: conditional
)
usage_block = ChatCompletionUsageBlock( # pyright: ignore[reportCallIssue] # TypedDict kwarg constructor
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,

2
litellm/llms/mistral/batches/transformation.py
View file

@ -33,7 +33,7 @@ OpenAIBatchStatus: TypeAlias = Literal[
"validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"
]
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) # mutable-ok: frozen at module scope
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
_STATUS_MAP: Final[MappingProxyType[MistralBatchStatus, OpenAIBatchStatus]] = MappingProxyType(
{
"QUEUED": "validating",

2
litellm/llms/mongodb/vector_stores/transformation.py
View file

@ -197,7 +197,7 @@ class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
**headers,
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
} # mutable-ok: writable HTTP headers
}
def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
if not api_base:

2
litellm/llms/nvidia_nim/passthrough/transformation.py
View file

@ -110,7 +110,7 @@ class NvidiaNimPassthroughConfig(BasePassthroughConfig):
return {
**headers,
"Authorization": f"Bearer {api_key}",
} # mutable-ok: base class contract returns dict for httpx
}
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:

2
litellm/llms/nvidia_nim/rerank/transformation.py
View file

@ -215,7 +215,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig):
elif isinstance(doc, dict):
# Preserve only the structured passage fields supported by the
# selected rerank route.
supported_fields: NvidiaNimPassageObject = {} # mutable-ok: assembling a request TypedDict
supported_fields: NvidiaNimPassageObject = {}
if "text" in self.SUPPORTED_PASSAGE_FIELDS and "text" in doc:
supported_fields["text"] = doc["text"]
if "image" in self.SUPPORTED_PASSAGE_FIELDS and "image" in doc:

4
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -596,9 +596,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
for choice in choices:
## HANDLE JSON MODE - anthropic returns single function call]
tool_calls = choice["message"].get("tool_calls", None)
new_tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None = (
None # mutable-ok: holds _handle_invalid_parallel_tool_calls' list; Message.__init__ expects list
)
new_tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None = None
message_content = choice["message"].get("content", None)
if tool_calls is not None:
_openai_tool_calls = []

8
litellm/llms/openai/openai.py
View file

@ -1427,9 +1427,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
},
)
request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict
{**data, "extra_headers": headers} if headers else data
)
request_data: Final = {**data, "extra_headers": headers} if headers else data
response = await openai_aclient.images.generate(**request_data, timeout=timeout)
stringified_response: Final = response.model_dump()
## LOGGING
@ -1513,9 +1511,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
## COMPLETION CALL
request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict
{**data, "extra_headers": headers} if headers else data
)
request_data: Final = {**data, "extra_headers": headers} if headers else data
_response: Final = openai_client.images.generate(**request_data, timeout=timeout)
response: Final = _response.model_dump()

34
litellm/llms/snowflake/chat/transformation.py
View file

@ -118,7 +118,7 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
anthropic_process_openai_file_message({"type": "file", "file": {"file_data": url}})
if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document"
else create_anthropic_image_param(
image_url if isinstance(image_url, dict) else url, # mutable-ok: caller's JSON block
image_url if isinstance(image_url, dict) else url,
format=_image_url_field(image_url, "format"),
is_bedrock_invoke=True,
)
@ -191,12 +191,8 @@ def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]: # mutable-
]
def _clean_input_schema(schema: object) -> object: # mutable-ok: JSON schema copy
return (
{key: value for key, value in schema.items() if key != "$schema"}
if isinstance(schema, Mapping)
else schema # mutable-ok: JSON schema copy
) # mutable-ok: JSON schema copy
def _clean_input_schema(schema: object) -> object:
return {key: value for key, value in schema.items() if key != "$schema"} if isinstance(schema, Mapping) else schema
class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
@ -299,9 +295,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
)
return anthropic_tools
def _extract_system_and_messages( # mutable-ok: JSON wire messages
self, messages: list[AllMessageValues]
) -> tuple[list[dict] | None, list[dict]]:
def _extract_system_and_messages(self, messages: list[AllMessageValues]) -> tuple[list[dict] | None, list[dict]]:
"""
Split messages into system prompt and conversation turns for Anthropic format.
@ -330,9 +324,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
{ # mutable-ok: JSON wire system block
"type": "text",
"text": block.get("text", ""),
**(
{"cache_control": block["cache_control"]} if "cache_control" in block else {}
), # mutable-ok: JSON wire block
**({"cache_control": block["cache_control"]} if "cache_control" in block else {}),
}
for block in content
if isinstance(block, Mapping) and block.get("type") == "text"
@ -372,7 +364,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
]
if isinstance(content, list)
else [*thinking_blocks, *([{"type": "text", "text": content}] if content else [])]
) # rebind-ok: loop-local normalized content
)
conversation.append({"role": "assistant", "content": thinking_content})
else:
conversation.append({"role": "assistant", "content": content})
@ -380,9 +372,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
tool_call_id_value = (
msg.get("tool_call_id", "") if isinstance(msg, dict) else getattr(msg, "tool_call_id", "")
)
tool_call_id = (
tool_call_id_value if isinstance(tool_call_id_value, str) else ""
) # rebind-ok: normalized loop value
tool_call_id = tool_call_id_value if isinstance(tool_call_id_value, str) else ""
tool_result_block = _convert_tool_result_to_anthropic(content, tool_call_id, msg_cache_control)
if (
conversation
@ -395,13 +385,13 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
else:
conversation.append(
{"role": "user", "content": [tool_result_block]} # mutable-ok: JSON wire message
) # mutable-ok: JSON wire message
)
else:
conversation.append( # mutable-ok: JSON wire message
conversation.append(
{ # mutable-ok: JSON wire message
"role": role,
"content": _convert_image_url_blocks_to_anthropic(content),
} # mutable-ok: JSON wire message
}
)
system: Final[list[dict] | None] = system_parts if system_parts else None # mutable-ok: JSON wire messages
@ -516,11 +506,11 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
"messages": conversation,
"stream": stream,
**optional_params,
**extra_body, # mutable-ok: JSON wire body
**extra_body,
}
)
if system is not None:
body["system"] = normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire payload
body["system"] = normalize_cache_control_in_anthropic_payload(
{"system": system} # mutable-ok: JSON wire payload
)["system"]

16
litellm/llms/vertex_ai/text_to_speech/transformation.py
View file

@ -43,9 +43,7 @@ else:
LiteLLMLoggingObj = Any
HttpxBinaryResponseContent = Any
_LyriaVoice: TypeAlias = (
str | dict | None
) # mutable-ok: inherited interface supports structured provider voice dictionaries
_LyriaVoice: TypeAlias = str | dict | None
class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
@ -664,21 +662,15 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig):
if model_info["vertex_ai_audio_api"] == "lyria_predict":
predictions: Final = response_json.get("predictions") or ()
if predictions:
audio_data = predictions[0].get("audioContent") or predictions[0].get(
"bytesBase64Encoded"
) # rebind-ok: predict response supplies the generated audio value
audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded")
mime_type = predictions[0].get("mimeType") # rebind-ok: predict response supplies its audio MIME type
else:
for step in response_json.get("steps") or response_json.get("outputs") or ():
content_items = step.get("content") or () if step.get("type") == "model_output" else (step,)
for content in content_items:
if content.get("type") == "audio" and content.get("data"):
audio_data = content[
"data"
] # rebind-ok: interactions response supplies the generated audio value
mime_type = content.get(
"mime_type"
) # rebind-ok: interactions response supplies its audio MIME type
audio_data = content["data"]
mime_type = content.get("mime_type")
if audio_data is None:
raise ValueError(f"No generated audio found in Vertex AI {base_model} response")
binary_data: Final = base64.b64decode(audio_data)

4
litellm/llms/xai/audio_transcription/transformation.py
View file

@ -168,9 +168,7 @@ class XAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
for word in payload.words
]
hidden_params: Final[dict[str, object]] = dict(
payload.model_dump(mode="json")
) # mutable-ok: TranscriptionResponse._hidden_params is a dict
hidden_params: Final[dict[str, object]] = dict(payload.model_dump(mode="json"))
if payload.duration is not None:
hidden_params["audio_transcription_duration"] = payload.duration
response._hidden_params = hidden_params # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter

646
litellm/model_prices_and_context_window_backup.json
View file

@ -27474,6 +27474,253 @@
"supports_vision": true,
"tpm": 10000000
},
"gemini/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_flex": 1e-06,
"input_cost_per_token_priority": 3.6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"rpm": 1000,
"tpm": 4000000,
"output_cost_per_token_batches": 6e-06,
"output_cost_per_token_flex": 6e-06,
"output_cost_per_token_priority": 2.16e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
"gemini/gemini-3.1-flash-image-preview": {
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "image_generation",
"output_cost_per_image": 0.045,
"output_cost_per_image_token": 6e-05,
"output_cost_per_token": 3e-06,
"output_cost_per_token_batches": 1.5e-06,
"rpm": 1000,
"tpm": 4000000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-preview": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"input_cost_per_audio_token": 5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 1.5e-06,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"output_cost_per_token_flex": 7.5e-07,
"output_cost_per_token_priority": 2.7e-06,
"rpm": 15,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
},
"gemini/gemini-embedding-2-preview": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
"mode": "embedding",
"output_cost_per_token": 0,
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supports_audio_input": true,
"supports_multimodal": true,
"supports_vision": true,
"tpm": 10000000
},
"gemini/deep-research-preview-04-2026": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-05,
"rpm": 1000,
"tpm": 4000000,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": false,
"supports_prompt_caching": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
}
},
"gemini/deep-research-max-preview-04-2026": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-05,
"rpm": 1000,
"tpm": 4000000,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": false,
"supports_prompt_caching": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
}
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_token_cost": 3e-08,
@ -40246,36 +40493,36 @@
"supports_web_search": false
},
"openrouter/deepseek/deepseek-v4-pro": {
"input_cost_per_token": 8.8044e-07,
"input_cost_per_token": 9.396e-07,
"input_cost_per_token_cache_hit": 4.4e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.76088e-06,
"output_cost_per_token": 1.8792e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 7.337e-08,
"cache_read_input_token_cost": 7.83e-08,
"supports_audio_input": false,
"supports_pdf_input": false,
"supports_vision": false,
"supports_web_search": false
},
"openrouter/deepseek/deepseek-v4.1-flash": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4.2e-07,
"cache_read_input_token_cost": 4.2e-09,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 943718,
"max_tokens": 943718,
"mode": "chat",
"off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":3e-7,"output_cost_per_token":0.0000012,"cache_read_input_token_cost":6e-9},
"off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":1.5e-7,"output_cost_per_token":6e-7,"cache_read_input_token_cost":3e-9},
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,
@ -40288,21 +40535,21 @@
"supports_web_search": false
},
"openrouter/deepseek/deepseek-v4-pro-0813": {
"input_cost_per_token": 1.32e-06,
"input_cost_per_token": 4.62e-07,
"input_cost_per_token_cache_hit": 1.9272e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"output_cost_per_token": 1.386e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 4.4e-08,
"cache_read_input_token_cost": 1.54e-08,
"off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":0.00000132,"output_cost_per_token":0.00000396,"cache_read_input_token_cost":4.4e-8},
"supports_audio_input": false,
"supports_pdf_input": false,
@ -40630,13 +40877,13 @@
"max_output_tokens": 8000
},
"openrouter/minimax/minimax-m2": {
"input_cost_per_token": 2.55e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 204800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.02e-06,
"output_cost_per_token": 1.2e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,
@ -40843,7 +41090,7 @@
},
"openrouter/nvidia/nemotron-3.5-lightning": {
"cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 7e-08,
"input_cost_per_token": 8e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 235929,
@ -41484,6 +41731,12 @@
},
"openrouter/qwen/qwen3-coder-plus": {
"cache_creation_input_token_cost": 8.125e-07,
"cache_creation_input_token_cost_above_128k_tokens": 2.4375e-06,
"cache_read_input_token_cost_above_128k_tokens": 3.9e-07,
"input_cost_per_token_above_32k_tokens": 1.17e-06,
"cache_creation_input_token_cost_above_32k_tokens": 1.4625e-06,
"cache_read_input_token_cost_above_32k_tokens": 2.34e-07,
"output_cost_per_token_above_32k_tokens": 5.85e-06,
"cache_read_input_token_cost": 1.3e-07,
"input_cost_per_token": 6.5e-07,
"input_cost_per_token_above_128k_tokens": 1.95e-06,
@ -41546,6 +41799,9 @@
},
"openrouter/qwen/qwen3.6-plus": {
"cache_creation_input_token_cost": 4.0625e-07,
"input_cost_per_token_above_256k_tokens": 1.3e-06,
"cache_creation_input_token_cost_above_256k_tokens": 1.625e-06,
"output_cost_per_token_above_256k_tokens": 3.9e-06,
"input_cost_per_token": 3.25e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
@ -41643,14 +41899,14 @@
},
"openrouter/qwen/qwen3.5-plus-02-15": {
"input_cost_per_token": 2.6e-07,
"input_cost_per_token_above_256k_tokens": 5e-07,
"input_cost_per_token_above_256k_tokens": 3.25e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.56e-06,
"output_cost_per_token_above_256k_tokens": 3e-06,
"output_cost_per_token_above_256k_tokens": 1.95e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,
@ -55923,7 +56179,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 4e-06,
@ -55954,7 +56213,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
@ -55985,7 +56247,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
@ -56016,7 +56281,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
@ -56047,7 +56315,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
@ -56078,7 +56349,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/responses"
]
},
"bedrock_mantle/openai.gpt-6-astra": {
"input_cost_per_token": 1.1e-05,
@ -56224,7 +56498,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-6-sol": {
"input_cost_per_token": 2.2e-06,
@ -56256,7 +56533,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-6-luna": {
"input_cost_per_token": 1.1e-07,
@ -56288,7 +56568,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-6-astra": {
"input_cost_per_token": 1e-05,
@ -56320,6 +56603,41 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"openai.gpt-6-sol": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4e-07,
"output_cost_per_token": 1e-05,
"output_cost_per_token_above_272k_tokens": 1.5e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_tool_choice": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-sol": {
@ -56352,6 +56670,41 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"openai.gpt-6-luna": {
"input_cost_per_token": 1e-07,
"input_cost_per_token_above_272k_tokens": 2e-07,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"output_cost_per_token": 5e-07,
"output_cost_per_token_above_272k_tokens": 7.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_tool_choice": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-luna": {
@ -56384,7 +56737,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/responses"
]
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,
@ -64072,7 +64428,7 @@
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"source": "https://www.baseten.co/library/glm-53-fast/",
"source": "https://inference.baseten.co/v1/models",
"supported_modalities": [
"text",
"image"
@ -64109,6 +64465,10 @@
},
"openrouter/qwen/qwen3.7-plus": {
"input_cost_per_token": 3.2e-07,
"input_cost_per_token_above_256k_tokens": 9.6e-07,
"cache_creation_input_token_cost_above_256k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_256k_tokens": 1.92e-07,
"output_cost_per_token_above_256k_tokens": 3.84e-06,
"output_cost_per_token": 1.28e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
@ -64358,6 +64718,24 @@
"supports_prompt_caching": true,
"supports_web_search": false
},
"openrouter/qwen/qwen3.8-max-prime": {
"input_cost_per_token": 4e-06,
"output_cost_per_token": 1.2e-05,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_video_input": true,
"supports_prompt_caching": true
},
"openrouter/deepseek/deepseek-v4-flash-0731": {
"input_cost_per_token": 4e-08,
"output_cost_per_token": 6.4e-07,
@ -64381,6 +64759,10 @@
},
"openrouter/qwen/qwen3.7-flash": {
"input_cost_per_token": 3e-08,
"input_cost_per_token_above_32k_tokens": 1e-07,
"cache_creation_input_token_cost_above_32k_tokens": 1.25e-07,
"cache_read_input_token_cost_above_32k_tokens": 2e-08,
"output_cost_per_token_above_32k_tokens": 4e-07,
"output_cost_per_token": 1.3e-07,
"cache_read_input_token_cost": 6e-09,
"cache_creation_input_token_cost": 3.8e-08,
@ -64929,9 +65311,9 @@
"supports_web_search": true
},
"openrouter/deepseek/deepseek-v4-flash": {
"input_cost_per_token": 8.246e-08,
"output_cost_per_token": 1.6492e-07,
"cache_read_input_token_cost": 1.6492e-08,
"input_cost_per_token": 8.8606e-08,
"output_cost_per_token": 1.77212e-07,
"cache_read_input_token_cost": 1.77212e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
@ -65271,6 +65653,8 @@
},
"openrouter/qwen/qwen3-max-thinking": {
"input_cost_per_token": 7.8e-07,
"input_cost_per_token_above_32k_tokens": 1.56e-06,
"output_cost_per_token_above_32k_tokens": 7.8e-06,
"output_cost_per_token": 3.9e-06,
"input_cost_per_token_above_128k_tokens": 1.95e-06,
"output_cost_per_token_above_128k_tokens": 9.75e-06,
@ -65717,6 +66101,10 @@
},
"openrouter/qwen/qwen3-max": {
"input_cost_per_token": 7.8e-07,
"input_cost_per_token_above_32k_tokens": 1.56e-06,
"cache_creation_input_token_cost_above_32k_tokens": 1.95e-06,
"cache_read_input_token_cost_above_32k_tokens": 3.12e-07,
"output_cost_per_token_above_32k_tokens": 7.8e-06,
"output_cost_per_token": 3.9e-06,
"cache_read_input_token_cost": 1.56e-07,
"cache_creation_input_token_cost": 9.75e-07,
@ -65763,6 +66151,10 @@
},
"openrouter/qwen/qwen3-coder-flash": {
"input_cost_per_token": 1.95e-07,
"input_cost_per_token_above_32k_tokens": 3.25e-07,
"cache_creation_input_token_cost_above_32k_tokens": 4.0625e-07,
"cache_read_input_token_cost_above_32k_tokens": 6.5e-08,
"output_cost_per_token_above_32k_tokens": 1.625e-06,
"output_cost_per_token": 9.75e-07,
"cache_read_input_token_cost": 3.9e-08,
"cache_creation_input_token_cost": 2.4375e-07,
@ -65806,7 +66198,7 @@
"supports_web_search": false
},
"openrouter/qwen/qwen3-next-80b-a3b-instruct": {
"input_cost_per_token": 9e-08,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 1.1e-06,
"cache_read_input_token_cost": 7e-08,
"litellm_provider": "openrouter",
@ -66978,6 +67370,15 @@
"output_cost_per_token": 1.2e-06,
"source": "https://api.together.ai/v1/models"
},
"together_ai/together/Tev1-4B-experimental": {
"cache_read_input_token_cost": 4.2e-08,
"input_cost_per_token": 4.2e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 0.0,
"source": "https://api.together.ai/v1/models"
},
"azure/eu/codex-mini": {
"deprecation_date": "2026-11-15",
"cache_read_input_token_cost": 4.13e-07,
@ -72219,14 +72620,15 @@
"supports_web_search": false
},
"openrouter/stealth/space-bunny-alpha": {
"input_cost_per_token": 0,
"deprecation_date": "2098-12-31",
"input_cost_per_token": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0,
"source": "https://openrouter.ai/stealth/space-bunny-alpha",
"output_cost_per_token": 0.0,
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -72689,14 +73091,14 @@
"supports_web_search": false
},
"openrouter/z-ai/glm-5.3:batch": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 7.2e-07,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.4e-06,
"output_cost_per_token": 2e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,
@ -72727,8 +73129,29 @@
"supports_vision": true,
"supports_web_search": false
},
"openrouter/z-ai/glm-5.3-prime": {
"cache_read_input_token_cost": 5.6e-07,
"input_cost_per_token": 2.8e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8.8e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false,
"supports_web_search": false
},
"openrouter/z-ai/glm-5.3-flashx": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost": 9e-08,
"deprecation_date": "2098-12-31",
"input_cost_per_token": 3.7e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
@ -73682,6 +74105,55 @@
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": {
"cache_read_input_token_cost": 6e-07,
"cache_read_input_token_cost_priority": 6e-07,
"deprecation_date": "2026-08-27",
"input_cost_per_token": 1.2e-06,
"input_cost_per_token_priority": 1.2e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"output_cost_per_token_priority": 1.2e-06,
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/minimax-m2p7": {
"cache_read_input_token_cost_priority": 6e-07,
"deprecation_date": "2026-08-27",
"input_cost_per_token_priority": 1.2e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 196608,
"max_output_tokens": 196608,
"max_tokens": 196608,
"mode": "chat",
"output_cost_per_token_priority": 1.2e-06,
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/ember-1": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/anthropic/claude-opus-5.5:batch": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
@ -73923,5 +74395,105 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"vertex_ai/meta/llama-3.3-70b-instruct-maas": {
"input_cost_per_token": 7.2e-07,
"input_cost_per_token_batches": 3.6e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 128000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"output_cost_per_token_batches": 3.6e-07,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"text",
"code"
],
"supports_function_calling": true,
"supports_tool_choice": true
},
"vertex_ai/veo-3.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
"mode": "video_generation",
"output_cost_per_second": 0.4,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
]
},
"vertex_ai/veo-3.0-fast-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
"mode": "video_generation",
"output_cost_per_second": 0.1,
"output_cost_per_second_1080p": 0.12,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
]
},
"vertex_ai/veo-2.0-generate-001": {
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
"mode": "video_generation",
"output_cost_per_second": 0.5,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
]
},
"vertex_ai/virtual-try-on-001": {
"deprecation_date": "2027-03-15",
"litellm_provider": "vertex_ai",
"mode": "image_generation",
"output_cost_per_image": 0.06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_modalities": [
"image"
],
"supported_output_modalities": [
"image"
]
},
"vertex_ai/gemini-2.5-flash-tts": {
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "vertex_ai",
"mode": "audio_speech",
"output_cost_per_audio_token": 1e-05,
"output_cost_per_token": 1e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/gemini-2.5-pro-tts": {
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "vertex_ai",
"mode": "audio_speech",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
}
}

4
litellm/ocr/main.py
View file

@ -173,9 +173,7 @@ def _prepare_ocr_request(
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
provider_config=ocr_provider_config,
optional_params=cast(
dict[str, object], optional_params
), # cast-ok: provider configs return heterogeneous OCR options
optional_params=cast(dict[str, object], optional_params),
litellm_params=dict(litellm_params),
effective_timeout=effective_timeout,
litellm_logging_obj=litellm_logging_obj,

12
litellm/passthrough/main.py
View file

@ -428,9 +428,7 @@ def llm_passthrough_route(
_is_async: Final = bool(kwargs.get("allm_passthrough_route", False))
litellm_logging_obj: Final = cast(
LiteLLMLoggingObj, kwargs.get("litellm_logging_obj")
) # cast-ok: logging obj is constructed upstream; tests inject mocks
litellm_logging_obj: Final = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj"))
model, custom_llm_provider, api_key, api_base = get_llm_provider(
model=model,
@ -516,9 +514,7 @@ def llm_passthrough_route(
forward_headers=False,
)
_request_data: dict | None = (
data if isinstance(data, dict) else (json if isinstance(json, dict) else None)
) # rebind-ok: conditional
_request_data: dict | None = data if isinstance(data, dict) else (json if isinstance(json, dict) else None)
headers, signed_json_body = provider_config.sign_request(
headers=headers,
litellm_params=litellm_params_dict,
@ -544,9 +540,7 @@ def llm_passthrough_route(
)
## IS STREAMING REQUEST
_streaming_request_data: dict = (
data if isinstance(data, dict) else (json if isinstance(json, dict) else {})
) # rebind-ok: conditional
_streaming_request_data: dict = data if isinstance(data, dict) else (json if isinstance(json, dict) else {})
is_streaming_request: Final = provider_config.is_streaming_request(
endpoint=endpoint,
request_data=_streaming_request_data,

16
litellm/proxy/_experimental/mcp_server/contracts.py
View file

@ -57,24 +57,20 @@ class OperationContext:
) -> tuple[
UserAPIKeyAuth | None,
str | None,
list[str] | None, # mutable-ok: detached legacy server-list payload
dict[str, dict[str, str]] | None, # mutable-ok: legacy auth dispatch requires concrete dict headers
dict[str, str] | None, # mutable-ok: detached legacy header payload
dict[str, str] | None, # mutable-ok: detached legacy header payload
list[str] | None,
dict[str, dict[str, str]] | None,
dict[str, str] | None,
dict[str, str] | None,
str | None,
]:
return (
self.user_api_key_auth,
self.mcp_auth_header,
list(self.mcp_servers) if self.mcp_servers is not None else None, # mutable-ok: legacy policy list input
{
key: dict(value) for key, value in self.mcp_server_auth_headers.items()
} # mutable-ok: legacy auth dispatch checks concrete dict headers
{key: dict(value) for key, value in self.mcp_server_auth_headers.items()}
if self.mcp_server_auth_headers is not None
else None,
dict(self.oauth2_headers)
if self.oauth2_headers is not None
else None, # mutable-ok: legacy OAuth header input
dict(self.oauth2_headers) if self.oauth2_headers is not None else None,
dict(self.raw_headers) if self.raw_headers is not None else None, # mutable-ok: legacy request header input
self.client_ip,
)

223
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -3,6 +3,7 @@ import binascii
import hashlib
import json
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast
@ -64,6 +65,7 @@ if TYPE_CHECKING:
class _UserEnvVarsTransactionClient(Protocol):
litellm_mcpuserenvvars: "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
litellm_mcpservertable: "TableActions[prisma_db_models.LiteLLM_MCPServerTable]"
async def execute_raw(self, query: str, *args: object) -> int: ...
@ -74,6 +76,19 @@ class _UserEnvVarsTransaction(Protocol):
async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
@dataclass(frozen=True, slots=True)
class McpIdentifierConflict:
"""An incoming ``server_name``/``alias`` already belongs to another MCP server row.
``field`` is the incoming identifier that collided, ``value`` the submitted
string, and ``server_id`` the existing row that owns it.
"""
field: Literal["server_name", "alias"]
value: str
server_id: str
_AUTH_FLOW_SCOPED_FIELDS: Final["frozenset[str]"] = frozenset(
{
"issuer",
@ -500,6 +515,121 @@ def _db_transaction_manager(prisma_client: PrismaClient) -> _UserEnvVarsTransact
return manager
def _identifier_where(value: str, exclude_server_id: str | None) -> "prisma_db_types.LiteLLM_MCPServerTableWhereInput":
own_row_guard: Final = (
({"NOT": [{"server_id": exclude_server_id}]},) # mutable-ok: prisma where-inputs must be plain dicts
if exclude_server_id is not None
else ()
)
where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = {
"AND": [ # mutable-ok: prisma where-inputs must be plain dicts
{
"OR": [ # mutable-ok: prisma where-inputs must be plain dicts
{"server_name": {"equals": value, "mode": "insensitive"}},
{"alias": {"equals": value, "mode": "insensitive"}},
]
},
{
"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]
}, # mutable-ok: prisma where-inputs must be plain dicts
*own_row_guard,
]
}
return where
def _identifier_field(data_dict: "Mapping[str, object]", field: str) -> str | None:
value: Final = data_dict.get(field)
return value if isinstance(value, str) else None
async def _find_mcp_server_identifier_conflict(
table: "TableActions[prisma_db_models.LiteLLM_MCPServerTable]",
*,
server_name: str | None,
alias: str | None,
exclude_server_id: str | None,
) -> McpIdentifierConflict | None:
"""Return the collision between an incoming identifier and a stored row, else None.
Each non-empty incoming identifier is compared case-insensitively against
BOTH the ``server_name`` and ``alias`` columns, because a value that matches
either column would still share the tool prefix another server answers to.
``alias`` is checked first so the reported field is deterministic. Draft
rows back the transient OAuth session flow and never reach the registry, so
they cannot collide. NULL ``approval_status`` predates the approval
workflow and is kept via the inner OR, matching ``get_all_mcp_servers``.
"""
candidates: Final[tuple[tuple[Literal["alias", "server_name"], str | None], ...]] = (
("alias", alias),
("server_name", server_name),
)
for field_name, value in candidates:
if not value:
continue
if (row := await table.find_first(where=_identifier_where(value, exclude_server_id))) is not None:
return McpIdentifierConflict(field=field_name, value=value, server_id=row.server_id)
return None
async def find_mcp_server_identifier_conflict(
prisma_client: PrismaClient,
*,
server_name: str | None,
alias: str | None,
exclude_server_id: str | None,
) -> McpIdentifierConflict | None:
"""Unlocked identifier-collision check, for callers outside a write path."""
return await _find_mcp_server_identifier_conflict(
_mcp_server_table_actions(prisma_client),
server_name=server_name,
alias=alias,
exclude_server_id=exclude_server_id,
)
def _mcp_identifier_lock_keys(*identifiers: str | None) -> tuple[int, ...]:
"""Deterministic advisory-lock keys for the lowercased identifiers, sorted
so concurrent requests for the same pair always lock in the same order."""
return tuple(
int.from_bytes(
hashlib.blake2b(f"mcp_identifier:{normalized}".encode(), digest_size=8).digest(),
"big",
signed=True,
)
for normalized in sorted(frozenset(value.lower() for value in identifiers if value))
)
async def _mcp_server_write_if_identifier_free(
prisma_client: PrismaClient,
*,
server_name: str | None,
alias: str | None,
exclude_server_id: str | None,
write: "Callable[[TableActions[prisma_db_models.LiteLLM_MCPServerTable]], Awaitable[prisma_db_models.LiteLLM_MCPServerTable | None]]",
) -> "prisma_db_models.LiteLLM_MCPServerTable | McpIdentifierConflict | None":
"""Run ``write`` only when no other live row owns ``server_name``/``alias``.
The conflict check and the write share a transaction guarded by per-identifier
advisory locks, so two concurrent requests for the same name cannot both
pass the check and both insert.
"""
lock_keys: Final = _mcp_identifier_lock_keys(server_name, alias)
async with _db_transaction_manager(prisma_client) as tx:
for lock_key in lock_keys:
await tx.execute_raw("SELECT pg_advisory_xact_lock($1::bigint)", lock_key)
conflict: Final = await _find_mcp_server_identifier_conflict(
tx.litellm_mcpservertable,
server_name=server_name,
alias=alias,
exclude_server_id=exclude_server_id,
)
if conflict is not None:
return conflict
return await write(tx.litellm_mcpservertable)
async def _db_find_mcp_server_rows(
prisma_client: PrismaClient,
where: "prisma_db_types.LiteLLM_MCPServerTableWhereInput | None" = None,
@ -636,8 +766,6 @@ async def get_all_mcp_servers(
where: Final[prisma_db_types.LiteLLM_MCPServerTableWhereInput] = (
{"approval_status": approval_status}
if approval_status is not None
# mutable-ok: prisma where-inputs must be plain dicts, and both `NOT` and `not` drop
# NULL rows (measured), so the OR is the only NULL-preserving way to exclude drafts
else {"OR": [{"approval_status": None}, {"approval_status": {"not": MCPApprovalStatus.draft}}]}
)
mcp_servers: Final = await _db_find_mcp_server_rows(prisma_client, where)
@ -882,6 +1010,43 @@ async def create_mcp_server(
return LiteLLM_MCPServerTable.model_validate(new_mcp_server.model_dump())
async def create_mcp_server_if_identifier_free(
prisma_client: PrismaClient, data: NewMCPServerRequest, touched_by: str
) -> LiteLLM_MCPServerTable | McpIdentifierConflict:
"""Create the row only when no other live server owns ``server_name``/``alias``.
Returns the McpIdentifierConflict instead of inserting when the collision
check finds an existing row; the advisory-lock transaction keeps two
concurrent creates of the same identifier from both passing.
"""
if data.server_id is None:
data.server_id = str(uuid.uuid4())
data_dict: Final = _prepare_mcp_server_data(data)
data_dict["created_by"] = touched_by
data_dict["updated_by"] = touched_by
async def _create(
table: "TableActions[prisma_db_models.LiteLLM_MCPServerTable]",
) -> "prisma_db_models.LiteLLM_MCPServerTable | None":
return await table.create(data=data_dict)
written: Final = await _mcp_server_write_if_identifier_free(
prisma_client,
server_name=_identifier_field(data_dict, "server_name"),
alias=_identifier_field(data_dict, "alias"),
exclude_server_id=None,
write=_create,
)
if isinstance(written, McpIdentifierConflict):
return written
if written is None:
raise RuntimeError("inserted MCP server row missing")
_decrypt_env_vars_on_returned_row(written)
return LiteLLM_MCPServerTable.model_validate(written.model_dump())
async def create_draft_mcp_server(
prisma_client: PrismaClient,
data: NewMCPServerRequest,
@ -972,14 +1137,57 @@ async def get_draft_mcp_server(
return table
async def _update_mcp_server_row(
prisma_client: PrismaClient,
*,
server_id: str,
data_dict: Mapping[str, object],
) -> "prisma_db_models.LiteLLM_MCPServerTable | McpIdentifierConflict | None":
identifier_write: Final = any(field in data_dict for field in ("server_name", "alias"))
async def _update(
table: "TableActions[prisma_db_models.LiteLLM_MCPServerTable]",
) -> "prisma_db_models.LiteLLM_MCPServerTable | None":
return await table.update(
where={"server_id": server_id}, # mutable-ok: prisma where-inputs must be plain dicts
data=data_dict,
)
if not identifier_write:
return await _update(_mcp_server_table_actions(prisma_client))
if "alias" in data_dict and not data_dict["alias"] and "server_name" not in data_dict:
# Clearing the alias drops the prefix to the stored server_name, which
# may already belong to another row, so that name needs the check too.
existing: Final = await _db_find_mcp_server_row(prisma_client, server_id)
if existing is None:
return await _update(_mcp_server_table_actions(prisma_client))
return await _mcp_server_write_if_identifier_free(
prisma_client,
server_name=existing.server_name,
alias=None,
exclude_server_id=server_id,
write=_update,
)
return await _mcp_server_write_if_identifier_free(
prisma_client,
server_name=_identifier_field(data_dict, "server_name"),
alias=_identifier_field(data_dict, "alias"),
exclude_server_id=server_id,
write=_update,
)
async def update_mcp_server(
prisma_client: PrismaClient,
data: UpdateMCPServerRequest,
touched_by: str,
fields_set: set[str] | None = None,
) -> LiteLLM_MCPServerTable | None:
) -> LiteLLM_MCPServerTable | McpIdentifierConflict | None:
"""
Update a new mcp server record in the db
Returns McpIdentifierConflict instead of writing when the update would put
``server_name``/``alias`` onto identifiers another live row already owns.
"""
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -1088,11 +1296,14 @@ async def update_mcp_server(
data_dict["credentials"] = Json(None)
updated_mcp_server: Final = await MCPServerRepository(prisma_client).table.update(
where={"server_id": data.server_id},
data=data_dict,
updated_mcp_server: Final = await _update_mcp_server_row(
prisma_client,
server_id=data.server_id,
data_dict=data_dict,
)
if isinstance(updated_mcp_server, McpIdentifierConflict):
return updated_mcp_server
_decrypt_env_vars_on_returned_row(updated_mcp_server)
return LiteLLM_MCPServerTable.model_validate(updated_mcp_server.model_dump()) if updated_mcp_server else None

3
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -1570,6 +1570,7 @@ async def _persist_dcr_client_registration(
}
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids circular import
McpIdentifierConflict,
update_mcp_server,
upsert_mcp_server_oauth_client_credentials,
)
@ -1601,7 +1602,7 @@ async def _persist_dcr_client_registration(
),
touched_by="mcp_oauth_dcr",
)
if updated_row is not None:
if updated_row is not None and not isinstance(updated_row, McpIdentifierConflict):
await global_mcp_server_manager.update_server(updated_row)
return "persisted"
if global_mcp_server_manager.is_config_declared_server(mcp_server.server_id):

4
litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
View file

@ -55,9 +55,7 @@ def create_sampling_callback(
params=params,
default_model=getattr(litellm, "default_mcp_sampling_model", None),
user_api_key_auth=captured.user_api_key_auth,
raw_headers=dict(captured.raw_headers)
if captured.raw_headers is not None
else None, # mutable-ok: handler consumes an owned request header dict
raw_headers=dict(captured.raw_headers) if captured.raw_headers is not None else None,
client_ip=captured.client_ip,
)

8
litellm/proxy/_experimental/mcp_server/mcp_debug.py
View file

@ -174,9 +174,7 @@ class MCPAuthDiagnostics:
{
"x-mcp-debug-auth-resolution": AuthResolution.multiple.value,
"x-mcp-debug-auth-resolutions": json.dumps(
{
server_id: source.value for server_id, source in self._outcomes[:32]
}, # mutable-ok: JSON encoder requires a concrete dict
{server_id: source.value for server_id, source in self._outcomes[:32]},
separators=(",", ":"),
ensure_ascii=True,
),
@ -597,9 +595,7 @@ async def capture_upstream_error_response(response: httpx.Response | httpx2.Resp
)
except (asyncio.TimeoutError, httpx.HTTPError, httpx.StreamError, httpx2.HTTPError, httpx2.StreamError):
response._content = b"" # pyright: ignore[reportPrivateUsage] # rebind-ok: httpx auth retries must survive diagnostic read failures
response.extensions[_CAPTURE_EXTENSION] = (
"(unavailable: error body read failed)" # rebind-ok: httpx response hooks communicate through extensions
)
response.extensions[_CAPTURE_EXTENSION] = "(unavailable: error body read failed)"
return
response.extensions[_CAPTURE_EXTENSION] = preview # rebind-ok: httpx response hooks communicate through extensions

30
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1453,6 +1453,35 @@ def _warn_on_server_name_fields(
_warn("server_name", server_name)
def _warn_on_shared_identifier_prefixes(servers: Iterable[MCPServer]) -> None:
"""Warn once per identifier that several servers share.
``get_server_prefix`` resolves alias first, so two servers sharing a
lowercased ``alias or server_name`` publish the same tool prefix and calls
routed by that prefix are ambiguous. A write-time uniqueness check keeps
new collisions out; this surfaces the ones already stored.
"""
pairs: Final = tuple(
((server.alias or server.server_name or "").lower(), server.server_id)
for server in servers
if server.alias or server.server_name
)
groups: Final = MappingProxyType(
{
identifier: tuple(sorted(server_id for key, server_id in pairs if key == identifier))
for identifier in frozenset(key for key, _server_id in pairs)
}
)
for identifier, server_ids in groups.items():
if len(server_ids) > 1:
verbose_logger.warning(
"MCP servers %s share the identifier '%s'; tool routing for that prefix is ambiguous. "
"Rename or delete all but one.",
sorted(server_ids),
identifier,
)
def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None:
"""Direct legacy delegated OAuth configurations to the admitted replacement."""
if server.auth_type != MCPAuth.oauth2:
@ -6613,6 +6642,7 @@ class MCPServerManager:
if previous_registry.get(server_id) != registered_registry.get(server_id):
self._invalidate_discovery_lists(server_id)
self.registry = registered_registry
_warn_on_shared_identifier_prefixes(registered_registry.values())
# A discovery task may have published into ``previous_registry`` while
# this replacement was being staged. Reconcile every published entry
# synchronously after the swap so a lost publication cannot also leave

45
litellm/proxy/_experimental/mcp_server/operations.py
View file

@ -1721,6 +1721,39 @@ async def _check_byok_credential(
)
def _challenge_missing_token_exchange_subject(
server: MCPServer | None,
requested_server: MCPServer | None,
allowed_mcp_servers: list[MCPServer],
user_api_key_auth: UserAPIKeyAuth | None,
oauth2_headers: dict[str, str] | None,
raw_headers: dict[str, str] | None,
) -> None:
"""Raise the RFC 9728 challenge when a token-exchange server is called without a subject token.
The listing that fills a cold catalog absorbs the upstream 401 by design, so without this
check a missing subject surfaces as an unknown-tool error instead of the challenge the
warm path already raises. Gated to servers the key may reach so an unauthorized caller
learns nothing about the catalog.
"""
if server is None or server.auth_type != MCPAuth.oauth2_token_exchange:
return
if requested_server is not None and requested_server.server_id != server.server_id:
return
if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers):
return
if global_mcp_server_manager._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) is not None:
return
from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # lazy: adapter pulls MCP subgraph
raise_token_exchange_challenge,
)
from litellm.proxy.middleware.per_request_root_path_middleware import ( # noqa: PLC0415 # lazy: middleware imports proxy utils
get_request_root_path,
)
raise_token_exchange_challenge(server, root_path=get_request_root_path())
async def _list_tools_before_first_call(
server: MCPServer | None,
tool_name: str,
@ -1864,6 +1897,14 @@ async def _execute_mcp_tool(
if first_call_target is None or (requested_server is not None and not name_is_prefixed)
else strip_known_server_prefix(name, first_call_target)
)
_challenge_missing_token_exchange_subject(
server=first_call_target,
requested_server=requested_server,
allowed_mcp_servers=allowed_mcp_servers,
user_api_key_auth=user_api_key_auth,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
await _list_tools_before_first_call(
server=first_call_target,
tool_name=first_call_tool_name,
@ -3062,9 +3103,7 @@ class GatewayOperations:
return await _execute_mcp_tool(
name=operation.name,
arguments=dict(operation.arguments), # mutable-ok: existing tool hooks own mutable argument data
allowed_mcp_servers=list(
operation.allowed_mcp_servers
), # mutable-ok: legacy dispatch list contract
allowed_mcp_servers=list(operation.allowed_mcp_servers),
start_time=operation.start_time,
user_api_key_auth=auth,
mcp_auth_header=token,

10
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -83,6 +83,8 @@ _MCP_GUARDRAIL_REJECTIONS: Final = (
HTTPException,
)
_CLIENT_FORWARDED_TOKEN_AUTH_TYPES: Final = frozenset((MCPAuth.true_passthrough, MCPAuth.oauth_delegate))
def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str:
reference: Final = uuid4().hex
@ -1153,6 +1155,7 @@ if MCP_AVAILABLE:
route_type=CallTypes.call_mcp_tool.value,
proxy_logging_obj=proxy_logging_obj,
general_settings=general_settings,
skip_guardrails=True,
)
# Extract MCP auth headers from request and add to data dict
@ -1186,6 +1189,11 @@ if MCP_AVAILABLE:
)
if target_server is not None:
user_oauth_extra_headers = await _get_user_oauth_extra_headers(target_server, user_api_key_dict)
caller_oauth2_headers: Final = (
MCPRequestHandler._get_oauth2_headers_from_headers(request.headers)
if target_server is not None and target_server.auth_type in _CLIENT_FORWARDED_TOKEN_AUTH_TYPES
else None
)
# Call execute_mcp_tool directly (permission checks already done)
_tool_start_time: Final = datetime.now()
@ -1197,7 +1205,7 @@ if MCP_AVAILABLE:
user_api_key_auth=data.get("user_api_key_auth"),
mcp_auth_header=data.get("mcp_auth_header"),
mcp_server_auth_headers=data.get("mcp_server_auth_headers"),
oauth2_headers=user_oauth_extra_headers or data.get("oauth2_headers"),
oauth2_headers=user_oauth_extra_headers or caller_oauth2_headers,
raw_headers=data.get("raw_headers"),
client_ip=IPAddressUtils.get_mcp_client_ip(request),
litellm_logging_obj=data.get("litellm_logging_obj"),

6
litellm/proxy/_experimental/mcp_server/tool_search.py
View file

@ -103,7 +103,7 @@ def _tool_result(tool: Tool) -> ToolSearchResult:
"name": tool.name,
"description": tool.description or "",
"inputSchema": tool.input_schema,
} # mutable-ok: wire schema payload
}
def _scored_result(tool: Tool, score: float) -> ToolSearchResult:
@ -112,7 +112,7 @@ def _scored_result(tool: Tool, score: float) -> ToolSearchResult:
"description": tool.description or "",
"inputSchema": tool.input_schema,
"score": score,
} # mutable-ok: wire schema payload
}
_MCP_PROXY_IDENTITY_META_KEY: Final[str] = "litellm.ai/proxy_tool_identity"
@ -120,7 +120,7 @@ _MCP_PROXY_IDENTITY_META_KEY: Final[str] = "litellm.ai/proxy_tool_identity"
def with_mcp_proxy_identity(tool: Tool, server_id: str) -> Tool:
identity: Final[MCPProxyToolIdentity] = {"server_id": server_id, "tool_name": tool.name}
return tool.model_copy( # mutable-ok: Pydantic requires mutable update and metadata mappings
return tool.model_copy(
update={ # mutable-ok: Pydantic update payload
"meta": {**(tool.meta or {}), _MCP_PROXY_IDENTITY_META_KEY: identity} # mutable-ok: metadata mapping
}

1
litellm/proxy/_types.py
View file

@ -1133,6 +1133,7 @@ class ModelInfo(LiteLLMPydanticObjectBase):
]
| None
)
discoverable: bool | None = None
model_config = ConfigDict(protected_namespaces=(), extra="allow")

8
litellm/proxy/agent_endpoints/agent_registry.py
View file

@ -135,9 +135,7 @@ def _dump_agent_params(raw: Mapping[str, object]) -> dict[str, object]:
_AGENT_PARAMS_MASKER: Final = SensitiveDataMasker()
_REDACT_AGENT_PARAMS_MAX_DEPTH: Final = 10
_AGENT_PARAMS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(
dict[str, object]
) # mutable-ok: safe_dumps() and AgentResponse.litellm_params both require a real dict, not a Mapping
_AGENT_PARAMS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
_AGENT_PARAMS_SEQUENCE_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
_EMPTY_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
@ -189,7 +187,7 @@ def _redact_agent_params_tree(value: object, _depth: int) -> object:
else _redact_agent_params_tree(nested_value, _depth + 1)
)
for key, nested_value in typed_params.items()
} # mutable-ok: consumed by json.dumps()/AgentResponse.litellm_params, both of which require a real dict
}
def parse_agent_litellm_params(value: object) -> Mapping[str, object]:
@ -318,7 +316,7 @@ def _restore_redacted_litellm_params(
key: value
for key in all_keys
if (value := _resolved_agent_param_value(key, incoming, existing, _depth)) is not _MISSING_AGENT_PARAM
} # mutable-ok: fed to safe_dumps() for JSON-column storage, which requires a real dict
}
class GrantMigrationResult(NamedTuple):

2
litellm/proxy/auth/auth_utils.py
View file

@ -1410,7 +1410,7 @@ def log_once_if_budget_reservation_disabled(
"Set disable_budget_reservation to False or remove it to restore "
"hard per-request budget enforcement."
)
constants.budget_reservation_disabled_info_emitted = True # rebind-ok: process-wide one-shot sentinel
constants.budget_reservation_disabled_info_emitted = True
def is_pass_through_provider_route(route: str) -> bool:

4
litellm/proxy/client/cli/commands/agents.py
View file

@ -241,9 +241,7 @@ def prepare_codex(
_Preparer: TypeAlias = Callable[[str, str, Mapping[str, str]], Sequence[str]]
_PREPARERS: Final[Mapping[str, _Preparer]] = MappingProxyType(
{"pi": prepare_pi, "codex": prepare_codex} # mutable-ok: MappingProxyType freezes the provider registry
)
_PREPARERS: Final[Mapping[str, _Preparer]] = MappingProxyType({"pi": prepare_pi, "codex": prepare_codex})
def agent_launch_args(command: str, base_url: str) -> list[str]:

2
litellm/proxy/client/cli/commands/claude_settings.py
View file

@ -621,7 +621,7 @@ def unconfigure_claude_settings(
)
target: Final = _write_target(settings_path)
file_removed: Final = not settings and not (receipt.file_existed and target.exists())
kept_receipt: Final = ( # mutable-ok: pydantic serializes the update as given and rejects a mappingproxy
kept_receipt: Final = (
receipt.model_copy(update={"written": {item.key: _fingerprint(absent) for item in withheld}})
if withheld
else None

1
litellm/proxy/client/cli/commands/codex_settings.py
View file

@ -106,7 +106,6 @@ def _with(document: TOMLDocument, path: str, snapshot: str | None) -> TOMLDocume
if section and section not in document and snapshot is not None:
contents: Final = tomlkit.parse(tomlkit.dumps(MappingProxyType({key: tomlkit.parse(snapshot).item("value")})))
return tomlkit.parse(document.as_string() + "\n" + tomlkit.dumps(MappingProxyType({section: contents})))
# mutable-ok: TOMLKit editing requires private node mutation to preserve comments and order
updated: Final = tomlkit.parse(document.as_string())
parent: Final = _table(_mapping(updated).get(section)) if section else updated
if parent is None:

6
litellm/proxy/client/cli/commands/pi.py
View file

@ -175,7 +175,7 @@ def _model_entry(
)
output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field
{"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {}
) # mutable-ok: JSON field
)
return {"id": model_id, **context, **output} # mutable-ok: JSON serialization requires a mutable object
@ -208,9 +208,7 @@ def sync_models_json(
) -> PiSyncError | None:
"""Replace only the litellm provider entry, leaving the rest of the file intact."""
try:
current: Final = ( # mutable-ok: JSON object default
_MODELS_FILE_ADAPTER.validate_json(path.read_text()) if path.exists() else {}
)
current: Final = _MODELS_FILE_ADAPTER.validate_json(path.read_text()) if path.exists() else {}
except (OSError, ValidationError) as e:
return PiSyncError(f"Could not read {path} as a JSON object: {e}. Fix or move the file, then retry.")
existing_providers: Final = current.get("providers", {}) # mutable-ok: JSON object default

4
litellm/proxy/common_request_processing.py
View file

@ -1999,6 +1999,7 @@ class ProxyBaseLLMRequestProcessing:
model: str | None = None,
llm_router: Router | None = None,
rate_limited_model: str | None = None,
skip_guardrails: bool = False,
) -> tuple[dict, LiteLLMLoggingObj]:
start_time: Final = datetime.now() # start before calling guardrail hooks
@ -2187,6 +2188,7 @@ class ProxyBaseLLMRequestProcessing:
user_api_key_dict=user_api_key_dict,
data=self.data,
call_type=route_type,
skip_guardrails=skip_guardrails,
)
await _enforce_guardrail_added_tag_budgets(
data=self.data,
@ -2206,7 +2208,7 @@ class ProxyBaseLLMRequestProcessing:
# Refresh AFTER pre_call_hook: guardrails (e.g. Presidio PII masking) may
# have mutated `self.data` in place, and the audit-trail snapshot taken in
# add_litellm_data_to_request predates that mutation.
refresh_proxy_server_request_body_snapshot(self.data)
refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True)
verbose_proxy_logger.debug("receiving data: %s", self.data)
if "messages" in self.data and self.data["messages"]:

8
litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py
View file

@ -184,17 +184,17 @@ class AuthCacheInvalidationSubscriber:
backoff_seconds = _BACKOFF_INITIAL_SECONDS # rebind-ok: exponential backoff accumulator across reconnects
while True:
try:
client = _pubsub_capable_client(self._redis_cache) # rebind-ok: re-resolved on every reconnect
client = _pubsub_capable_client(self._redis_cache)
if client is None:
verbose_proxy_logger.warning(
"auth cache invalidation subscriber disabled: cluster redis client has no pub/sub support; "
"cross-worker eviction falls back to the local cache TTL"
)
return
pubsub = client.pubsub() # rebind-ok: fresh pubsub per reconnect
pubsub = client.pubsub()
try:
await pubsub.subscribe(auth_cache_invalidation_channel(self._redis_cache))
backoff_seconds = _BACKOFF_INITIAL_SECONDS # rebind-ok: reset after successful subscribe
backoff_seconds = _BACKOFF_INITIAL_SECONDS
await self._consume(pubsub)
finally:
await self._close_pubsub(pubsub)
@ -207,7 +207,7 @@ class AuthCacheInvalidationSubscriber:
backoff_seconds,
)
await asyncio.sleep(backoff_seconds)
backoff_seconds = min(backoff_seconds * 2, _BACKOFF_MAX_SECONDS) # rebind-ok: backoff accumulator
backoff_seconds = min(backoff_seconds * 2, _BACKOFF_MAX_SECONDS)
async def _consume(self, pubsub: _ConfigSyncPubSub) -> None:
while True:

118
litellm/proxy/common_utils/discoverable_model_filter.py Normal file
View file

@ -0,0 +1,118 @@
from __future__ import annotations
import re
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider
from litellm.proxy._types import UserAPIKeyAuth, user_api_key_has_admin_view
if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.router import RouterModelGroupAliasItem
_PATTERN_DEPLOYMENTS: Final = TypeAdapter(Mapping[str, tuple[Mapping[str, object], ...]])
def is_undiscoverable_deployment(deployment: Mapping[str, object]) -> bool:
model_info: Final = deployment.get("model_info")
if not isinstance(model_info, Mapping):
return False
return "discoverable" in model_info and model_info["discoverable"] is False
def is_undiscoverable_model_name(model_name: str, llm_router: Router | None, team_id: str | None) -> bool:
if llm_router is None:
return False
deployments: Final = llm_router.get_model_list(model_name=model_name, team_id=team_id)
if not deployments:
return False
return all(is_undiscoverable_deployment(deployment) for deployment in deployments)
def _team_public_model_name(deployment: Mapping[str, object]) -> object:
model_info: Final = deployment.get("model_info")
return model_info.get("team_public_model_name") if isinstance(model_info, Mapping) else None
def _alias_target(alias: str | RouterModelGroupAliasItem) -> str:
return alias if isinstance(alias, str) else alias["model"]
def _undiscoverable_served_names(
undiscoverable_rows: Iterable[Mapping[str, object]],
model_group_alias: Mapping[str, str | RouterModelGroupAliasItem],
) -> frozenset[str]:
served: Final = frozenset(
name
for row in undiscoverable_rows
for name in (row.get("model_name"), _team_public_model_name(row))
if isinstance(name, str)
)
aliases: Final = frozenset(alias for alias, target in model_group_alias.items() if _alias_target(target) in served)
return served | aliases
def _undiscoverable_patterns(llm_router: Router, team_id: str | None) -> tuple[re.Pattern[str], ...]:
team_pattern_router: Final = llm_router.team_pattern_routers.get(team_id) if team_id is not None else None
pattern_routers: Final = (
(llm_router.pattern_router,)
if team_pattern_router is None
else (llm_router.pattern_router, team_pattern_router)
)
return tuple(
re.compile(regex)
for pattern_router in pattern_routers
for regex, deployments in _PATTERN_DEPLOYMENTS.validate_python(pattern_router.patterns).items()
if any(is_undiscoverable_deployment(deployment) for deployment in deployments)
)
def _resolved_provider(model_name: str) -> str | None:
try:
return get_llm_provider(model=model_name)[1]
except Exception: # noqa: BLE001 # get_llm_provider raises when the provider is unknown; the name then routes as-is
return None
def _matches_undiscoverable_pattern(model_name: str, patterns: tuple[re.Pattern[str], ...]) -> bool:
if not patterns:
return False
if any(pattern.match(model_name) for pattern in patterns):
return True
provider: Final = declared_authenticating_provider(model_name) or _resolved_provider(model_name)
return any(pattern.match(f"{provider}/{model_name}") for pattern in patterns)
def undiscoverable_model_names(
model_names: Iterable[str],
llm_router: Router | None,
user_api_key_dict: UserAPIKeyAuth,
team_id: str | None,
) -> frozenset[str]:
if llm_router is None or user_api_key_has_admin_view(user_api_key_dict):
return frozenset()
undiscoverable_rows: Final = tuple(
row for row in llm_router.get_model_list() or () if is_undiscoverable_deployment(row)
)
if not undiscoverable_rows:
return frozenset()
served_names: Final = _undiscoverable_served_names(undiscoverable_rows, llm_router.model_group_alias)
patterns: Final = _undiscoverable_patterns(llm_router, team_id)
return frozenset(
name
for name in model_names
if (name in served_names or _matches_undiscoverable_pattern(name, patterns))
and is_undiscoverable_model_name(name, llm_router, team_id)
)
def discoverable_rows(
rows: Iterable[Mapping[str, object]],
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[Mapping[str, object], ...]:
if user_api_key_has_admin_view(user_api_key_dict):
return tuple(rows)
return tuple(row for row in rows if not is_undiscoverable_deployment(row))

20
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -240,12 +240,8 @@ def _queue_budget_linked_resets(
one transaction, so the reverse order lets the zero re-match a row the
decrement just moved into the (0, cap] range and erase its carried spend."""
for budget_id, cap in cascade.rollover_caps.items():
writes.queue_spend_zero(
where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}}
) # mutable-ok: prisma where filter must be a dict
writes.queue_spend_decrement(
where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap
) # mutable-ok: prisma where filter must be a dict
writes.queue_spend_zero(where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}})
writes.queue_spend_decrement(where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap)
plain_ids: Final = tuple(bid for bid in cascade.budget_ids if bid not in cascade.rollover_caps)
if plain_ids:
writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra))
@ -267,16 +263,10 @@ def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCasca
return
cap: Final = cascade.rollover_caps.get(default_budget_id)
if cap is None:
writes.queue_spend_zero(
where={"budget_id": None, **_SPENT_ROWS_WHERE}
) # mutable-ok: prisma where filter must be a dict
writes.queue_spend_zero(where={"budget_id": None, **_SPENT_ROWS_WHERE})
return
writes.queue_spend_zero(
where={"budget_id": None, "spend": {"gt": 0, "lte": cap}}
) # mutable-ok: prisma where filter must be a dict
writes.queue_spend_decrement(
where={"budget_id": None, "spend": {"gt": cap}}, amount=cap
) # mutable-ok: prisma where filter must be a dict
writes.queue_spend_zero(where={"budget_id": None, "spend": {"gt": 0, "lte": cap}})
writes.queue_spend_decrement(where={"budget_id": None, "spend": {"gt": cap}}, amount=cap)
@dataclass(frozen=True, slots=True)

8
litellm/proxy/common_utils/sse_keepalive.py
View file

@ -65,9 +65,7 @@ async def _keepalive_ping_stream(
ping_interval_seconds: float,
ping_chunk: str,
) -> AsyncGenerator[str, None]:
pending = asyncio.ensure_future(
stream.__anext__()
) # rebind-ok: re-armed with the next __anext__ after each delivered chunk
pending = asyncio.ensure_future(stream.__anext__())
try:
while True:
await asyncio.wait({pending}, timeout=ping_interval_seconds)
@ -125,9 +123,7 @@ async def _keepalive_ping_byte_stream(
stream: AsyncGenerator[bytes, None],
ping_interval_seconds: float,
) -> AsyncGenerator[bytes, None]:
pending = asyncio.ensure_future(
stream.__anext__()
) # rebind-ok: re-armed with the next __anext__ after each delivered chunk
pending = asyncio.ensure_future(stream.__anext__())
# The tail of the bytes relayed so far, long enough to hold any delimiter.
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
# across chunks because a delimiter can be split between two transport reads,

4
litellm/proxy/common_utils/user_api_key_cache.py
View file

@ -86,6 +86,10 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
def update_in_memory_max_size(self, max_size: int | None) -> None:
super().update_in_memory_max_size(max_size)
self.key_object_cache.update_in_memory_max_size(max_size)
def attach_redis_cache(
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
) -> None:

6
litellm/proxy/db/baseline_accounting.py
View file

@ -491,7 +491,7 @@ class BaselineAccountingStore:
tuple(await db.query_raw(_READ_PAGE, scope, after_revision, cursor, _PAGE_TIMESTAMPS, withdraw_from))
):
yield page
cursor = page[-1].started_at # rebind-ok: keyset pagination advances after each complete timestamp group
cursor = page[-1].started_at
async def _withdraw(self, db: SupportsRawQueries, scope: str, started_at: float) -> None:
async for page in self._pages(db, scope, 0, withdraw_from=started_at):
@ -623,9 +623,7 @@ async def flush_baseline_accounting(client: PrismaClient) -> None:
store: Final = BaselineAccountingStore.for_client(client)
async with client.baseline_accounting_lock:
batch: Final = tuple(client.baseline_accounting_transactions[:32])
client.baseline_accounting_transactions = client.baseline_accounting_transactions[
32:
] # rebind-ok: drain under lock
client.baseline_accounting_transactions = client.baseline_accounting_transactions[32:]
more_queued: Final = bool(client.baseline_accounting_transactions)
try:
remaining: Final = await asyncio.wait_for(_flush_records(store, batch), timeout=5)

2
litellm/proxy/db/shadow_eval_funnel.py
View file

@ -40,7 +40,7 @@ def pending_shadow_eval_funnel_events() -> int:
def record_shadow_eval_funnel_event(job_id: str, stage: ShadowEvalFunnelStage) -> None:
"""Count one skipped request for one job leg; synchronous so the hook's read-modify-
write cannot interleave with the flush's snapshot on the shared event loop."""
counters: Final = _pending.setdefault(job_id, dict.fromkeys(FUNNEL_STAGES, 0)) # mutable-ok: queue entry
counters: Final = _pending.setdefault(job_id, dict.fromkeys(FUNNEL_STAGES, 0))
counters[stage] += 1

Some files were not shown because too many files have changed in this diff Show more