scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit6874_fireworks_perplexity_off_peak_pricing

Browse source
This commit is contained in:
mateo-berri 2026-09-03 15:13:44 -07:00
commit 0bd2fd2a3b
234 changed files with 11055 additions and 2139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.github/workflows/test-litellm-ui-build.yml vendored
View file

@ -19,9 +19,6 @@ jobs:
build-ui:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
@ -35,18 +32,11 @@ jobs:
with:
category: ui
- name: Setup Node.js
# Built through the image stage rather than the checkout, because the
# stage copies ui/litellm-dashboard/ alone: an import reaching above the
# dashboard root resolves in a checkout and fails in every image we ship.
# Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
# stage verbatim, so building one covers all three.
- name: Build the dashboard as the shipped images build it
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: npm ci
- name: Build
if: steps.changes.outputs.decision != 'skip'
run: npm run build
run: docker build --target ui-builder -f Dockerfile .

7
.github/workflows/test-rust.yml vendored
View file

@ -74,12 +74,19 @@ jobs:
- name: Run Clippy with Bedrock auth
run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
- name: Run Clippy with all gateway features
run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
- name: Run Rust tests
run: cargo test --workspace --locked
- name: Run core tests with Bedrock auth
run: cargo test -p litellm-core --features bedrock-auth --locked
# Not --all-features: python-config links libpython, which this job does not install.
- name: Run gateway tests with the server feature
run: cargo test -p litellm-ai-gateway --features server --locked
release-wheel:
name: release wheel
runs-on: ubuntu-latest

1
.github/workflows/test-unit.yml vendored
View file

@ -151,6 +151,7 @@ jobs:
tests/test_litellm/proxy/google_endpoints
tests/test_litellm/proxy/openai_files_endpoint
tests/test_litellm/proxy/batches_endpoints
tests/test_litellm/proxy/container_endpoints
tests/test_litellm/proxy/fine_tuning_endpoints
tests/test_litellm/proxy/vector_store_files_endpoints
tests/test_litellm/proxy/video_endpoints

4
basedpyright-code-budget.json
View file

@ -3,7 +3,7 @@
"limit": 14074
},
"reportArgumentType": {
"limit": 2215
"limit": 2214
},
"reportAssignmentType": {
"limit": 319
@ -42,7 +42,7 @@
"limit": 12
},
"reportIndexIssue": {
"limit": 25
"limit": 24
},
"reportInvalidTypeForm": {
"limit": 34

13
helm/litellm/templates/_helpers.tpl
View file

@ -428,3 +428,16 @@ envFrom:
{{- end }}
{{- end }}
{{- end -}}
{{/*
ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path
(strict-validate-path-type) and serves ImplementationSpecific as a plain
prefix location, so a dotted path takes that type there.
*/}}
{{- define "litellm.ingress.pathType" -}}
{{- if and (eq .controller "nginx") (contains "." .path) -}}
ImplementationSpecific
{{- else -}}
{{- .pathType -}}
{{- end -}}
{{- end -}}

45
helm/litellm/templates/ingress.yaml
View file

@ -5,6 +5,10 @@
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{- $controller := .Values.ingress.controller | default "alb" -}}
{{- if not (has $controller (list "alb" "nginx")) }}
{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }}
{{- end }}
{{/*
Backends addressable from ingress.extraPaths, keyed by the `service` field.
*/}}
@ -27,10 +31,11 @@
/litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
→ 404 → client-side navigation never settles and the login flow spins in an
infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
from the export; the rule only routes the request to it. Needs an ingress
controller whose ImplementationSpecific path is a wildcard pattern
(AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
Controller.
from the export; the rule only routes the request to it. It needs an
ingress controller whose ImplementationSpecific path is a wildcard pattern
(AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb
only: ingress-nginx serves ImplementationSpecific as a literal prefix
location, where /*.txt can never match.
*/}}
{{- $uiPaths := list
(dict "path" "/" "pathType" "Exact")
@ -38,8 +43,10 @@
(dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
(dict "path" "/_next" "pathType" "Prefix")
(dict "path" "/ui" "pathType" "Prefix")
(dict "path" "/*.txt" "pathType" "ImplementationSpecific")
-}}
{{- if eq $controller "alb" }}
{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }}
{{- end }}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
@ -83,12 +90,6 @@
adding to it.
*/}}
{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
{{- range $uiPaths }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
{{- end }}
{{- range $gatewayPrefixes }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
{{- end }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
@ -115,8 +116,10 @@ spec:
paths:
# --- UI (Next.js static export) ---
{{- range $uiPaths }}
{{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }}
- path: {{ .path }}
pathType: {{ .pathType }}
pathType: {{ $pathType }}
backend:
service:
name: {{ $uiName }}
@ -134,8 +137,10 @@ spec:
port:
number: {{ $gatewayPort }}
{{- range $gatewayPrefixes }}
{{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
- path: {{ . }}
pathType: Prefix
pathType: {{ $pathType }}
backend:
service:
name: {{ $gatewayName }}
@ -147,10 +152,11 @@ spec:
Rendered after every built-in path so an entry can never take
precedence over a default, and before the backend catch-all.
Position only decides the match on controllers that honour manifest
order: the AWS Load Balancer Controller this chart targets sorts
Exact paths first and Prefix paths longest-first, but keeps
order: the AWS Load Balancer Controller (ingress.controller=alb)
sorts Exact paths first and Prefix paths longest-first, but keeps
ImplementationSpecific paths in manifest order, which is what the
/*.txt rule above already depends on.
/*.txt rule above already depends on. ingress-nginx ignores order
and serves the longest matching location.
*/}}
{{- range $idx, $extra := .Values.ingress.extraPaths }}
{{- if not (kindIs "map" $extra) }}
@ -164,10 +170,11 @@ spec:
{{- if not $target }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
{{- end }}
{{- $pathType := $extra.pathType | default "Prefix" }}
{{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
{{- $requestedPathType := $extra.pathType | default "Prefix" }}
{{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }}
{{- end }}
{{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }}
{{- if eq $extra.path "/" }}
{{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
{{- end }}

205
helm/litellm/tests/ingress_controller_tests.yaml Normal file
View file

@ -0,0 +1,205 @@
suite: test ingress.controller
templates:
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: keeps the AWS Load Balancer Controller path types by default
set:
ingress.enabled: true
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /favicon.ico
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /eu.assemblyai
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /*.txt
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
set:
ingress.enabled: true
ingress.controller: nginx
asserts:
- notMatchRegexRaw:
pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n'
- contains:
path: spec.rules[0].http.paths
content:
path: /favicon.ico
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /eu.assemblyai
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is
set:
ingress.enabled: true
ingress.controller: nginx
asserts:
- notContains:
path: spec.rules[0].http.paths
content:
path: /*.txt
any: true
- contains:
path: spec.rules[0].http.paths
content:
path: /ui
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /test
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-1]
value:
path: /
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with
set:
ingress.enabled: true
ingress.controller: nginx
ingress.extraPaths:
- path: /favicon.ico
service: ui
pathType: ImplementationSpecific
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects a controller it has no path types for
set:
ingress.enabled: true
ingress.controller: traefik
asserts:
- failedTemplate:
errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx'
- it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType
set:
ingress.enabled: true
ingress.controller: nginx
ingress.extraPaths:
- path: /favicon.ico
service: ui
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx
set:
ingress.enabled: true
ingress.controller: nginx
ingress.extraPaths:
- path: /eu.assemblyai.custom
service: gateway
- path: /robots.txt
service: ui
pathType: Exact
asserts:
- notMatchRegexRaw:
pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n'
- contains:
path: spec.rules[0].http.paths
content:
path: /eu.assemblyai.custom
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /robots.txt
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller
set:
ingress.enabled: true
ingress.extraPaths:
- path: /eu.assemblyai.custom
service: gateway
- path: /robots.txt
service: ui
pathType: Exact
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /eu.assemblyai.custom
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /robots.txt
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000

15
helm/litellm/values.yaml
View file

@ -10,6 +10,18 @@ imagePullSecrets: []
ingress:
enabled: false
className: ""
# Which ingress controller serves this Ingress. Controllers disagree on the
# pathTypes they accept, so this picks the pathType of the dotted paths, the
# built-in ones and any dotted extraPaths entry alike:
# alb AWS Load Balancer Controller (default): Exact and Prefix paths plus
# the /*.txt wildcard that routes the UI's RSC payloads.
# nginx ingress-nginx: its admission webhook rejects a dot in an Exact or
# Prefix path (strict-validate-path-type, on by default from v1.12.0
# until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and
# /eu.assemblyai render as ImplementationSpecific, which nginx serves
# as a plain prefix location. /*.txt is dropped: nginx has no
# wildcard pathType, so that rule could never match there.
controller: alb
annotations: {}
host: "" # optional; if set, becomes the rule's host
tls: []
@ -26,7 +38,8 @@ ingress:
#
# path required; the HTTP path to route
# service which component serves it: gateway (default), backend, or ui
# pathType Prefix (default), Exact, or ImplementationSpecific
# pathType Prefix (default), Exact, or ImplementationSpecific; a dotted
# path renders as ImplementationSpecific when controller is nginx
#
# The target component only answers paths its own route allowlist keeps, so
# a path here still has to be one that component serves.

69
litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py
View file

@ -18,10 +18,15 @@ recoverable one.
constant: it grows with the number of pending migrations, so a fresh database
that has to replay every migration this package ships overruns a per-command
budget sized for the short bookkeeping commands, on a laptop as much as on a
slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
as separate children, so killing the wrapper on timeout leaves them running:
the retry then contends with that orphan for Prisma's advisory lock and cannot
finish any sooner. Migrate deploy therefore runs under its own budget.
slow CI runner. Migrate deploy therefore runs under its own budget.
The Python ``prisma`` wrapper spawns Node, which spawns the Rust schema
engine, so killing only the wrapper on timeout leaves the engine running with
no parent: it keeps mutating the database after the proxy has given up, holds
Prisma's advisory lock so every retry and every later boot queues behind it,
and dies mid-migration once its pipes close, leaving a half-applied ledger row.
Every Prisma command therefore runs in a process group of its own, and a
timeout kills the whole group.
All three budgets are overridable so an operator can widen them without a
release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
@ -35,10 +40,12 @@ the deploy override says otherwise.
import math
import os
import shutil
import signal
import subprocess
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
from typing import IO, Optional, Union
from litellm_proxy_extras._logging import logger
@ -167,6 +174,49 @@ def heal_incomplete_nodeenv_cache() -> bool:
return True
def _kill_process_group(process: "subprocess.Popen[str]") -> None:
if os.name == "nt":
process.kill()
return
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
def run_prisma(
argv: Sequence[str],
*,
timeout: float,
env: Mapping[str, str],
stdout: Union[IO[str], int, None] = subprocess.PIPE,
stderr: Optional[int] = subprocess.PIPE,
) -> "subprocess.CompletedProcess[str]":
"""Run one Prisma CLI command in its own process group, bounded by ``timeout``.
Raises ``subprocess.TimeoutExpired`` once the budget is spent, after killing
the command together with every process it spawned, and
``subprocess.CalledProcessError`` on a non-zero exit. Output is captured as
text unless ``stdout``/``stderr`` say otherwise.
"""
with subprocess.Popen(
argv,
env=env,
stdout=stdout,
stderr=stderr,
text=True,
start_new_session=True,
) as process:
try:
out, err = process.communicate(timeout=timeout)
except BaseException:
_kill_process_group(process)
raise
if process.returncode:
raise subprocess.CalledProcessError(process.returncode, process.args, out, err)
return subprocess.CompletedProcess(process.args, process.returncode, out, err)
def ensure_prisma_toolchain(
prisma_command: str, prisma_env: dict[str, str]
) -> ToolchainBootstrap:
@ -179,14 +229,7 @@ def ensure_prisma_toolchain(
timeout = prisma_bootstrap_timeout()
logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
try:
subprocess.run(
[prisma_command, BOOTSTRAP_ARG],
timeout=timeout,
check=True,
capture_output=True,
text=True,
env=prisma_env,
)
run_prisma([prisma_command, BOOTSTRAP_ARG], timeout=timeout, env=prisma_env)
except subprocess.TimeoutExpired:
logger.warning(
"Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "

7
litellm-proxy-extras/litellm_proxy_extras/replica_identity.py
View file

@ -16,7 +16,7 @@ import tempfile
from pathlib import Path
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout, run_prisma
REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
@ -66,7 +66,7 @@ def apply_replica_identity_full(
with tempfile.TemporaryDirectory(prefix="litellm_replica_identity_") as tmp_dir:
sql_path = Path(tmp_dir) / "replica_identity_full.sql"
sql_path.write_text(REPLICA_IDENTITY_FULL_SQL)
subprocess.run(
run_prisma(
[
prisma_command,
"db",
@ -77,9 +77,6 @@ def apply_replica_identity_full(
schema_path,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=prisma_env,
)
except subprocess.CalledProcessError as e:

59
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -10,6 +10,7 @@ from dataclasses import dataclass, replace
from pathlib import Path
from typing import Optional
from litellm_proxy_extras import prisma_toolchain
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
@ -231,7 +232,7 @@ class ProxyExtrasDBManager:
# 1. Generate migration SQL file by comparing empty state to current db state
logger.info("Generating baseline migration...")
migration_file = init_dir / "migration.sql"
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -242,14 +243,13 @@ class ProxyExtrasDBManager:
"--script",
],
stdout=open(migration_file, "w"),
check=True,
timeout=prisma_command_timeout(),
env=prisma_env,
)
# 3. Mark the migration as applied since it represents current state
logger.info("Marking baseline migration as applied...")
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -257,7 +257,6 @@ class ProxyExtrasDBManager:
"--applied",
"0_init",
],
check=True,
timeout=prisma_command_timeout(),
env=prisma_env,
)
@ -286,7 +285,7 @@ class ProxyExtrasDBManager:
"""Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -295,8 +294,6 @@ class ProxyExtrasDBManager:
migration_name,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
)
@ -348,11 +345,9 @@ class ProxyExtrasDBManager:
def _resolve_specific_migration(migration_name: str):
"""Mark a specific migration as applied"""
prisma_env = _get_prisma_env()
subprocess.run(
prisma_toolchain.run_prisma(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
)
@ -436,7 +431,7 @@ class ProxyExtrasDBManager:
try:
logger.info("Generating migration diff between DB and schema.prisma...")
with open(diff_sql_path, "w") as f:
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -447,7 +442,6 @@ class ProxyExtrasDBManager:
schema_path,
"--script",
],
check=True,
timeout=prisma_command_timeout(),
stdout=f,
env=_get_prisma_env(),
@ -470,7 +464,7 @@ class ProxyExtrasDBManager:
migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
for mig_file in migration_files:
try:
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"db",
@ -481,9 +475,6 @@ class ProxyExtrasDBManager:
schema_path,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"Applied migration: {mig_file.parent.name}")
@ -516,7 +507,7 @@ class ProxyExtrasDBManager:
applied_ok = False
try:
logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run(
result = prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"db",
@ -527,9 +518,6 @@ class ProxyExtrasDBManager:
schema_path,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"prisma db execute stdout: {result.stdout}")
@ -558,7 +546,7 @@ class ProxyExtrasDBManager:
for migration_name in migration_names:
try:
logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -567,9 +555,6 @@ class ProxyExtrasDBManager:
migration_name,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.debug(f"Resolved migration: {migration_name}")
@ -762,11 +747,12 @@ class ProxyExtrasDBManager:
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
subprocess.run(
prisma_toolchain.run_prisma(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=prisma_command_timeout(),
check=True,
env=_get_prisma_env(),
stdout=None,
stderr=None,
)
return True
except (
@ -789,12 +775,9 @@ class ProxyExtrasDBManager:
try:
while not budget.exhausted:
try:
result = subprocess.run(
result = prisma_toolchain.run_prisma(
[_get_prisma_command(), "migrate", "deploy"],
timeout=deploy_timeout,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -1031,12 +1014,9 @@ class ProxyExtrasDBManager:
logger.info("Running prisma migrate deploy")
try:
# Set migrations directory for Prisma
result = subprocess.run(
result = prisma_toolchain.run_prisma(
[_get_prisma_command(), "migrate", "deploy"],
timeout=prisma_migrate_deploy_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -1108,7 +1088,7 @@ class ProxyExtrasDBManager:
f"Found failed migration: {failed_migration}, marking as rolled back"
)
# Mark the failed migration as rolled back
subprocess.run(
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"migrate",
@ -1117,9 +1097,6 @@ class ProxyExtrasDBManager:
failed_migration,
],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(
@ -1244,10 +1221,12 @@ class ProxyExtrasDBManager:
if ProxyExtrasDBManager.spend_logs_is_partitioned():
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
# Use prisma db push with increased timeout
subprocess.run(
prisma_toolchain.run_prisma(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=prisma_command_timeout(),
check=True,
stdout=None,
stderr=None,
env=_get_prisma_env(),
)
return True
except subprocess.TimeoutExpired:

33
litellm-proxy-extras/tests/test_setup_database_fail_fast.py
View file

@ -42,7 +42,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
"Error: P3018\nMigration name: 20250326162113_baseline\n"
"Database error code: 42501\npermission denied for schema public"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="permission"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -60,7 +60,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
"Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
'Reason: syntax error at or near "BRKN" LINE 42'
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -124,7 +124,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
def fake_resolve(*args, **kwargs):
resolve_called["n"] += 1
monkeypatch.setattr("subprocess.run", fake_run)
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", fake_run)
monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set
@ -139,7 +139,7 @@ def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_pat
(tmp_path / "schema.prisma").write_text("// stub")
stderr = "db push error"
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="prisma db push failed"):
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
@ -209,7 +209,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error(
"Error: P3009\nMigration `20260101000000_some_migration` failed\n"
"relation already exists"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(
RuntimeError, match="Failed to mark migration .* as applied"
):
@ -228,7 +228,7 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
stdout = "Applied migration.\n"
stderr = ""
monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult())
resolve_called = {"n": 0}
monkeypatch.setattr(
@ -296,7 +296,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
"_resolve_specific_migration",
lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
)
monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR))
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR))
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
@ -309,7 +309,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
with patch(
"subprocess.run",
"litellm_proxy_extras.prisma_toolchain.run_prisma",
side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR),
):
with pytest.raises(RuntimeError, match="after 4 attempts"):
@ -343,7 +343,7 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
"_resolve_specific_migration",
lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
)
monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
@ -372,7 +372,7 @@ def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path
"_resolve_specific_migration",
lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
)
monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
@ -395,7 +395,7 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
"_roll_back_migration",
lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
)
monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -417,7 +417,7 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
lambda name: 'ERROR: syntax error at or near "BRKN"',
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -427,7 +427,7 @@ def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path):
waiter as victim) is retried, not fatal."""
_stub_v2_env(monkeypatch, tmp_path)
monkeypatch.setattr(
"subprocess.run", _succeed_after(1, "Database error: deadlock detected")
"litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, "Database error: deadlock detected")
)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@ -446,7 +446,10 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path):
"""v2: the advisory-lock waiter that times out while a peer's retry holds
the lock retries instead of dying."""
_stub_v2_env(monkeypatch, tmp_path)
monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR))
monkeypatch.setattr(
"litellm_proxy_extras.prisma_toolchain.run_prisma",
_succeed_after(2, _P1002_ADVISORY_LOCK_STDERR),
)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
@ -456,7 +459,7 @@ def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_pa
"""v2: a plain P1002 (database unreachable) stays fatal."""
_stub_v2_env(monkeypatch, tmp_path)
stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out."
monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)

2
litellm-rust/ADDING_A_PROVIDER.md
View file

@ -26,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider
is a few declarative lines, not a new file of duplicated flow. Only diverge from
the base when behavior is genuinely different, and say so explicitly in the PR.
**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md).

8
litellm-rust/CLAUDE.md
View file

@ -174,10 +174,14 @@ for changes under `litellm-rust/`.
```bash
cd litellm-rust
cargo fmt --check
cargo clippy --workspace --all-targets -- -D warnings
cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings
# the ai-gateway binary + server code is behind the `server` feature
cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings
cargo test --workspace
cargo test -p litellm-core --features bedrock-auth
# the `auth`, `routes`, `state` and `realtime` tests only exist under `server`
cargo test -p litellm-ai-gateway --features server
```
When a Rust path is exposed through Python, add Python parity tests that compare

11
litellm-rust/README.md
View file

@ -49,11 +49,6 @@ function per top-level route, mirroring the core entrypoints.
## Checks
Run these before pushing Rust changes. GitHub Actions runs the same checks for
changes under `litellm-rust/`.
```bash
cargo fmt --check
cargo clippy --workspace --all-targets -- -D warnings
cargo test --workspace
```
Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust
changes. That list is the single source of truth and matches what GitHub Actions
runs for changes under `litellm-rust/`.

10
litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md
View file

@ -49,11 +49,5 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
## Checks before push
25. Run, and keep green:
```bash
cd litellm-rust
cargo fmt --check
cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
cargo test --workspace
```
25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`.
That list is the single source of truth and matches what GitHub Actions runs.

4
litellm-rust/crates/ai-gateway/Dockerfile
View file

@ -36,12 +36,12 @@ FROM chef AS builder
# whenever only gateway source changes.
COPY --from=planner /build/litellm-rust/recipe.json recipe.json
RUN cargo chef cook --locked --release \
-p litellm-ai-gateway --features python-config \
-p litellm-ai-gateway --features server,python-config \
--recipe-path recipe.json
# Now copy the real sources and build the gateway binary. Deps are already cooked
# above, so this step only recompiles the gateway crate.
COPY litellm-rust/ .
RUN cargo build --locked --release -p litellm-ai-gateway --features python-config
RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config
# ---- Runtime ----------------------------------------------------------------
# python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3

6
litellm-rust/crates/ai-gateway/README.md
View file

@ -100,7 +100,7 @@ Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096),
## Build & run with Docker
The image is built `--features python-config` and installs litellm **from this
The image is built `--features server,python-config` and installs litellm **from this
repo's source** (the config reader is newer than any PyPI release), so the build
**context is the repo root**:
@ -135,10 +135,10 @@ docker run --rm -p 4001:4001 \
```bash
# config.yaml mode — needs litellm importable in the active python env
LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \
cargo run --release -p litellm-ai-gateway --features python-config
cargo run --release -p litellm-ai-gateway --features server,python-config
# env stand-in mode — no python, no config
cargo run --release -p litellm-ai-gateway
cargo run --release -p litellm-ai-gateway --features server
```
## Deploy on Render

44
litellm-rust/crates/ai-gateway/src/realtime/streaming.rs
View file

@ -106,16 +106,16 @@ impl RealTimeStreaming {
/// `litellm_call_id`, replacing the gateway-generated fallback.
fn on_session(&mut self, event: &RealtimeEvent) {
let session = event.data.get("session").and_then(Value::as_object);
if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) {
if !id.is_empty() {
self.id = id.to_string();
self.litellm_call_id = id.to_string();
}
if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str)
&& !id.is_empty()
{
self.id = id.to_string();
self.litellm_call_id = id.to_string();
}
if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) {
if !model.is_empty() {
self.model = model.to_string();
}
if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str)
&& !model.is_empty()
{
self.model = model.to_string();
}
}
@ -323,6 +323,32 @@ mod tests {
assert_eq!(streaming.dropped(), 0);
}
#[test]
fn blank_session_id_and_model_keep_the_gateway_fallbacks() {
let mut streaming = RealTimeStreaming::new(
Vec::new(),
"call_fallback".to_string(),
"gpt-realtime".to_string(),
RequestMetadata::default(),
);
streaming.observe(&event(
r#"{"type":"session.created","session":{"id":"","model":""}}"#,
));
let payload = streaming.build_payload();
assert_eq!(payload.id, "call_fallback");
assert_eq!(payload.litellm_call_id, "call_fallback");
assert_eq!(payload.model, "gpt-realtime");
streaming.observe(&event(
r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#,
));
let payload = streaming.build_payload();
assert_eq!(payload.id, "sess_002");
assert_eq!(payload.litellm_call_id, "sess_002");
assert_eq!(payload.model, "gpt-realtime");
}
#[test]
fn payload_serializes_with_camelcase_times_and_realtime_call_type() {
let mut streaming = RealTimeStreaming::new(

23
litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs
View file

@ -50,18 +50,17 @@ where
provider_model,
params.api_key.as_deref(),
params.api_base.as_deref(),
) {
if let Some(handoff) = pool.take(&key) {
return crate::io::realtime::realtime_warm(
provider_model,
handoff,
idle_timeout,
observe,
client_in,
client_out,
)
.await;
}
) && let Some(handoff) = pool.take(&key)
{
return crate::io::realtime::realtime_warm(
provider_model,
handoff,
idle_timeout,
observe,
client_in,
client_out,
)
.await;
}
// Cold path: fresh dial (the original behavior).

115
litellm/_service_logger.py
View file

@ -1,4 +1,5 @@
import asyncio
from collections.abc import Callable, Coroutine
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final
@ -83,6 +84,47 @@ class ServiceLogging(CustomLogger):
return open_telemetry_logger
return None
@staticmethod
def _sync_dispatch_loop() -> asyncio.AbstractEventLoop | None:
"""The event loop a blocking caller can dispatch on, or ``None`` if it has none."""
try:
loop: Final = asyncio.get_event_loop()
except RuntimeError:
return None
return None if loop.is_closed() else loop
@staticmethod
async def _emit_guarded(hook: Callable[[], Coroutine[object, object, None]]) -> None:
"""Emit one service event, absorbing anything the callbacks raise.
Monitoring must not break the call it monitors. Sync callers are the ones that
swallow their own service failures (a Redis batch read returns an empty dict),
so an exception from a misconfigured callback would replace a Redis outage with
a callback error and skip the caller's fallback handling.
"""
try:
await hook()
except Exception as e:
verbose_logger.exception("Error emitting service event - %s", e)
@staticmethod
def _dispatch_from_sync(hook: Callable[[], Coroutine[object, object, None]]) -> None:
"""Run an async service hook from a blocking caller, whatever event loop it holds.
Takes a factory rather than a coroutine so the hook is built on the path that
runs it, and only ever once.
"""
loop: Final = ServiceLogging._sync_dispatch_loop()
try:
if loop is None:
asyncio.run(ServiceLogging._emit_guarded(hook))
elif loop.is_running():
loop.create_task(ServiceLogging._emit_guarded(hook))
else:
loop.run_until_complete(ServiceLogging._emit_guarded(hook))
except Exception as e:
verbose_logger.exception("Error dispatching service event - %s", e)
def service_success_hook(
self,
service: ServiceTypes,
@ -99,54 +141,45 @@ class ServiceLogging(CustomLogger):
if self.mock_testing:
self.mock_testing_sync_success_hook += 1
try:
# Try to get the current event loop
loop: Final = asyncio.get_event_loop()
# Check if the loop is running
if loop.is_running():
# If we're in a running loop, create a task
loop.create_task(
self.async_service_success_hook(
service=service,
duration=duration,
call_type=call_type,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
)
)
else:
# Loop exists but not running, we can use run_until_complete
loop.run_until_complete(
self.async_service_success_hook(
service=service,
duration=duration,
call_type=call_type,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
)
)
except RuntimeError:
# No event loop exists, create a new one and run
asyncio.run(
self.async_service_success_hook(
service=service,
duration=duration,
call_type=call_type,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
)
self._dispatch_from_sync(
lambda: self.async_service_success_hook(
service=service,
duration=duration,
call_type=call_type,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
)
)
def service_failure_hook(self, service: ServiceTypes, duration: float, error: Exception, call_type: str):
def service_failure_hook(
self,
service: ServiceTypes,
duration: float,
error: Exception,
call_type: str,
parent_otel_span: Span | None = None,
start_time: datetime | float | None = None,
end_time: float | datetime | None = None,
):
"""
[TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
Handles both sync and async monitoring by checking for existing event loop.
"""
if self.mock_testing:
self.mock_testing_sync_failure_hook += 1
self._dispatch_from_sync(
lambda: self.async_service_failure_hook(
service=service,
duration=duration,
error=error,
call_type=call_type,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
)
)
async def async_service_success_hook(
self,
service: ServiceTypes,

58
litellm/caching/dual_cache.py
View file

@ -8,10 +8,9 @@ Has 4 primary methods:
- async_get_cache
"""
import asyncio
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from collections.abc import Sequence
from threading import Lock
from typing import TYPE_CHECKING, Any, Final
@ -188,31 +187,38 @@ class DualCache(BaseCache):
local_only: bool = False,
**kwargs,
):
received_args: Final = locals()
received_args.pop("self")
def run_in_new_loop():
"""Run the coroutine in a new event loop within this thread."""
new_loop: Final = asyncio.new_event_loop()
try:
asyncio.set_event_loop(new_loop)
return new_loop.run_until_complete(self.async_batch_get_cache(**received_args))
finally:
new_loop.close()
asyncio.set_event_loop(None)
try:
# First, try to get the current event loop
_ = asyncio.get_running_loop()
# If we're already in an event loop, run in a separate thread
# to avoid nested event loop issues
with ThreadPoolExecutor(max_workers=1) as executor:
future: Final = executor.submit(run_in_new_loop)
return future.result()
in_memory_result: Final = (
self.in_memory_cache.batch_get_cache(keys, **kwargs) if self.in_memory_cache is not None else None
)
result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in keys)
except RuntimeError:
# No running event loop, we can safely run in this thread
return run_in_new_loop()
if None not in result or self.redis_cache is None or local_only:
return result
sublist_keys, previous_access_times = self._reserve_redis_batch_keys(time.time(), keys, result)
if len(sublist_keys) == 0:
return result
try:
redis_result: Final = self.redis_cache.batch_get_cache(
key_list=sublist_keys, parent_otel_span=parent_otel_span
)
except Exception:
# Do not throttle subsequent callers if the Redis read fails.
self._rollback_redis_batch_key_reservations(previous_access_times)
raise
if self.in_memory_cache is not None:
for key, value in redis_result.items():
if value is not None:
self.in_memory_cache.set_cache(key, value, **self._backfill_kwargs(kwargs))
return list( # mutable-ok: public list contract
redis_result.get(key) if value is None else value for key, value in zip(keys, result)
)
except Exception:
verbose_logger.error(traceback.format_exc())
async def async_get_cache(
self,
@ -251,7 +257,7 @@ class DualCache(BaseCache):
self,
current_time: float,
keys: list[str],
result: list[Any],
result: Sequence[Any],
) -> tuple[list[str], dict[str, float | None]]:
"""
Atomically choose keys to fetch from Redis and reserve their access time.

90
litellm/caching/redis_cache.py
View file

@ -78,10 +78,18 @@ class _AsyncRedisCommands(Protocol):
def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
_BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
{"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
)
def _get_call_stack_info(num_frames: int = 2) -> str:
"""
Get the function names from the previous 1-2 functions in the call stack.
Frames belonging to this module's circuit-breaker guards are skipped so the
reported callers stay the real ones even on guarded methods.
Args:
num_frames: Number of previous frames to include (default: 2)
@ -102,11 +110,11 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
return "unknown"
function_names: Final = []
for _ in range(num_frames):
if frame is None:
break
func_name = frame.f_code.co_name
function_names.append(func_name)
while frame is not None and len(function_names) < num_frames:
if frame.f_code.co_name in _BREAKER_GUARD_FRAME_NAMES and frame.f_globals.get("__name__") == __name__:
frame = frame.f_back
continue
function_names.append(frame.f_code.co_name)
frame = frame.f_back
if not function_names:
@ -241,6 +249,23 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep
_swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1)
def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int:
"""Reject the call if the breaker is open, else return the swallowed-failure count to compare against."""
if breaker.is_open():
raise Exception(f"Redis circuit breaker is open — skipping {name}")
return _swallowed_redis_failures.get()
def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None:
"""Record success only when nothing failed while the call ran.
Several Redis methods catch their own connection errors and return a default, so a
method that returned is not on its own proof of a healthy Redis.
"""
if _swallowed_redis_failures.get() == swallowed_before:
breaker.record_success()
async def _run_under_circuit_breaker(
breaker: RedisCircuitBreaker,
name: str,
@ -249,20 +274,33 @@ async def _run_under_circuit_breaker(
"""Run one Redis coroutine under a circuit breaker.
Shared by the method decorator and the Lua script executor so both feed the same
health signal. Success is recorded only when nothing failed while ``call`` ran,
because several Redis methods catch their own connection errors and return a default.
health signal.
"""
if breaker.is_open():
raise Exception(f"Redis circuit breaker is open — skipping {name}")
swallowed_before: Final = _swallowed_redis_failures.get()
swallowed_before: Final = _enter_circuit_breaker(breaker, name)
try:
result: Final = await call()
except Exception as e:
if _is_redis_health_failure(e):
breaker.record_failure()
raise
if _swallowed_redis_failures.get() == swallowed_before:
breaker.record_success()
_exit_circuit_breaker(breaker, swallowed_before)
return result
def _run_under_circuit_breaker_sync(
breaker: RedisCircuitBreaker,
name: str,
call: Callable[[], _RedisCallResult],
) -> _RedisCallResult:
"""Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path."""
swallowed_before: Final = _enter_circuit_breaker(breaker, name)
try:
result: Final = call()
except Exception as e:
if _is_redis_health_failure(e):
breaker.record_failure()
raise
_exit_circuit_breaker(breaker, swallowed_before)
return result
@ -288,6 +326,14 @@ def _redis_circuit_breaker_guard(method):
return wrapper
def _redis_circuit_breaker_guard_sync(method: Callable[..., _RedisCallResult]) -> Callable[..., _RedisCallResult]:
return functools.wraps(method)(
lambda self, *args, **kwargs: _run_under_circuit_breaker_sync(
self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
)
)
class RedisCache(BaseCache):
# if users don't provider one, use the default litellm cache
@ -1146,14 +1192,13 @@ class RedisCache(BaseCache):
"""
key_value_dict = {}
_key_list: Final = [key for key in key_list if key is not None]
start_time: Final = time.time()
try:
_keys: Final = []
for cache_key in _key_list:
cache_key = self.check_and_fix_namespace(key=cache_key or "")
_keys.append(cache_key)
start_time: Final = time.time()
swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache")
_keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list]
results: Final = self._run_redis_mget_operation(keys=_keys)
_exit_circuit_breaker(self._circuit_breaker, swallowed_before)
end_time: Final = time.time()
_duration: Final = end_time - start_time
self.service_logger_obj.service_success_hook(
@ -1178,7 +1223,18 @@ class RedisCache(BaseCache):
return decoded_results
except Exception as e:
failed_at: Final = time.time()
self.service_logger_obj.service_failure_hook(
service=ServiceTypes.REDIS,
duration=failed_at - start_time,
error=e,
call_type=f"batch_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=failed_at,
parent_otel_span=parent_otel_span,
)
verbose_logger.error("Error occurred in batch get cache - %s", e)
_record_swallowed_redis_failure(self._circuit_breaker, e)
return key_value_dict
@_redis_circuit_breaker_guard

2
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -932,7 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
if role == "user" or role == "system" or role == "tool":
if role in ("user", "system", "developer", "tool"):
return {"type": "input_text", "text": content}
else:
return {"type": "output_text", "text": content}

2
litellm/constants.py
View file

@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran
DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
@ -39,6 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
"router_general_settings",
"ignore_invalid_deployments",
"fallback_access_check",
"heuristic_v2_router_limit",
}
)
DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))

6
litellm/integrations/custom_guardrail.py
View file

@ -831,10 +831,10 @@ class CustomGuardrail(CustomLogger):
# should run guardrail
litellm_guardrails: Final = request_data.get("guardrails")
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
return response
return None
if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
return response
return None
# CHECK IF GUARDRAIL REJECTS THE REQUEST
result: Final = await self.async_post_call_success_hook(
@ -850,7 +850,7 @@ class CustomGuardrail(CustomLogger):
)
if not self._is_valid_response_type(result):
return response
return None
return result

76
litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py
View file

@ -6,7 +6,8 @@ It searches the vector store for relevant context and appends it to the messages
"""
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any, Final, cast
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
import litellm
import litellm.vector_stores
@ -24,10 +25,35 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
class ProxyRuntime(Protocol):
def llm_router(self) -> "Router | None": ...
def prisma_client(self) -> "PrismaClient | None": ...
@dataclass(frozen=True, slots=True)
class ProxyServerRuntime:
def llm_router(self) -> "Router | None":
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
return None
return llm_router
def prisma_client(self) -> "PrismaClient | None":
try:
from litellm.proxy.proxy_server import prisma_client
except ImportError:
return None
return prisma_client
class VectorStorePreCallHook(CustomLogger):
CONTENT_PREFIX_STRING = "Context:\n\n"
"""
@ -39,8 +65,9 @@ class VectorStorePreCallHook(CustomLogger):
3. Appends the search results as context to the messages
"""
def __init__(self):
def __init__(self, proxy_runtime: ProxyRuntime | None = None):
super().__init__()
self.proxy_runtime: Final[ProxyRuntime] = proxy_runtime or ProxyServerRuntime()
async def async_get_chat_completion_prompt(
self,
@ -79,21 +106,8 @@ class VectorStorePreCallHook(CustomLogger):
if litellm.vector_store_registry is None:
return model, messages, non_default_params
# Get prisma_client for database fallback
prisma_client = None
llm_router = None
try:
from litellm.proxy.proxy_server import (
llm_router as _llm_router,
)
from litellm.proxy.proxy_server import (
prisma_client as _prisma_client,
)
prisma_client = _prisma_client
llm_router = _llm_router
except ImportError:
pass
prisma_client: Final = self.proxy_runtime.prisma_client()
llm_router: Final = self.proxy_runtime.llm_router()
# Use database fallback to ensure synchronization across instances
vector_stores_to_run: list[
@ -136,15 +150,23 @@ class VectorStorePreCallHook(CustomLogger):
Callable[..., Awaitable[VectorStoreSearchResponse]],
litellm.vector_stores.asearch,
)
search_response = await search_function(
**{
"vector_store_id": vector_store_id,
"query": query,
"custom_llm_provider": custom_llm_provider,
"metadata": request_metadata,
**litellm_params_for_vector_store,
},
)
try:
search_response = await search_function(
**{
"vector_store_id": vector_store_id,
"query": query,
"custom_llm_provider": custom_llm_provider,
"metadata": request_metadata,
**litellm_params_for_vector_store,
},
)
except Exception as search_error:
verbose_logger.warning(
"Vector store search failed for vector_store_id=%s, continuing without its context: %s",
vector_store_id,
search_error,
)
continue
verbose_logger.debug("search_response: %s", search_response)
@ -153,7 +175,7 @@ class VectorStorePreCallHook(CustomLogger):
# Process search results and append as context
modified_messages = self._append_search_results_to_messages(
messages=messages, search_response=search_response
messages=modified_messages, search_response=search_response
)
# Get the number of results for logging

15
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -18,9 +18,11 @@ caller's identity metadata, minus two things that must never be forwarded as-is:
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, NON_INFERENCE_CALL_TYPES
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, InternalCallOrigin
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
@ -142,6 +144,19 @@ def forwarded_internal_call_metadata(
}
def parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, str]:
kwargs: Final = request_kwargs or MappingProxyType({})
return MappingProxyType(
{k: v for k in ("litellm_session_id", "litellm_trace_id") if isinstance(v := kwargs.get(k), str)}
)
def effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | None) -> bool | None:
return initialize_standard_callback_dynamic_params(dict(request_kwargs) if request_kwargs else None).get(
"turn_off_message_logging"
)
def sanitized_forwardable_call_metadata(
parent_metadata: Mapping[str, object],
call_origin: InternalCallOrigin,

34
litellm/litellm_core_utils/litellm_logging.py
View file

@ -414,6 +414,11 @@ def _resolve_vertex_location_for_cost(
return VertexBase.get_vertex_region(configured_location, model)
def _provider_response_id(source: object) -> str | None:
candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
return candidate if isinstance(candidate, str) and candidate else None
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -429,6 +434,7 @@ class Logging(LiteLLMLoggingBaseClass):
custom_pricing: bool = False
stream_options = None
litellm_request_debug: bool = False
streamed_anthropic_message_id: str | None = None
def __init__(
self,
@ -2136,7 +2142,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["cache_hit"] = cache_hit
if self.call_type == CallTypes.anthropic_messages.value:
result = self._handle_anthropic_messages_response_logging(result=result)
result = self._anthropic_messages_logged_response(result=result)
elif (
self.call_type == CallTypes.generate_content.value
or self.call_type == CallTypes.agenerate_content.value
@ -3806,6 +3812,23 @@ class Logging(LiteLLMLoggingBaseClass):
)
return None
def record_streamed_anthropic_message_id(self, message_id: str) -> None:
self.streamed_anthropic_message_id = message_id
def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
"""
The ModelResponse a /v1/messages spend_logs row is built from.
A streaming call bridged onto the Responses API is the one case where the `msg_` id the
caller was served is minted locally rather than issued upstream, so it is absent from the
response the row would otherwise be keyed on and has to be carried over here.
"""
logged: Final = self._handle_anthropic_messages_response_logging(result=result)
streamed_message_id: Final = self.streamed_anthropic_message_id
if streamed_message_id is None:
return logged
return logged.model_copy(update={"id": streamed_message_id})
def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
"""
Handles logging for Anthropic messages responses.
@ -3832,11 +3855,12 @@ class Logging(LiteLLMLoggingBaseClass):
if isinstance(result, ResponsesAPIResponse):
return self._translate_responses_api_response_to_model_response(result)
provider_response_id: Final = _provider_response_id(result)
httpx_response: Final = self.model_call_details.get("httpx_response", None)
if httpx_response and isinstance(httpx_response, httpx.Response):
result = litellm.AnthropicConfig().transform_response(
raw_response=httpx_response,
model_response=litellm.ModelResponse(),
model_response=litellm.ModelResponse(id=provider_response_id),
model=self.model,
messages=[],
logging_obj=self,
@ -3859,7 +3883,7 @@ class Logging(LiteLLMLoggingBaseClass):
status_code=200,
headers={},
),
model_response=litellm.ModelResponse(),
model_response=litellm.ModelResponse(id=provider_response_id),
json_mode=None,
speed=self.optional_params.get("speed") if self.optional_params else None,
)
@ -3882,7 +3906,7 @@ class Logging(LiteLLMLoggingBaseClass):
return LiteLLMResponsesTransformationHandler().transform_response(
model=self.model,
raw_response=result,
model_response=litellm.ModelResponse(),
model_response=litellm.ModelResponse(id=_provider_response_id(result)),
logging_obj=self,
request_data={},
messages=[],
@ -3897,7 +3921,7 @@ class Logging(LiteLLMLoggingBaseClass):
"usage-only ModelResponse to keep the spend_logs row.",
str(e),
)
model_response: Final = litellm.ModelResponse()
model_response: Final = litellm.ModelResponse(id=_provider_response_id(result))
model_response.model = self.model
usage: Final = getattr(result, "usage", None)
if usage is not None and ResponseAPILoggingUtils._is_response_api_usage(usage):

34
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -5,6 +5,8 @@ Helper utilities for tracking the cost of built-in tools.
from collections.abc import Mapping
from typing import Final, Literal
from pydantic import ValidationError
import litellm
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import (
@ -13,6 +15,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
from litellm.types.llms.openai import (
FileSearchTool,
ResponsesAPIResponse,
ResponsesToolUsage,
WebSearchOptions,
)
from litellm.types.utils import (
@ -32,6 +35,17 @@ def _output_item_type(output_item: object) -> str | None:
return item_type if isinstance(item_type, str) else None
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = getattr(response_object, "tool_usage", None)
if tool_usage is None:
return None
try:
web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
except ValidationError:
return None
return None if web_search is None else web_search.num_requests
def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
details: Final = getattr(usage, "server_side_tool_usage_details", None)
if not isinstance(details, Mapping):
@ -182,15 +196,19 @@ class StandardBuiltInToolCostTracking:
Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by
get_cost_for_web_search_request and never reach here. This path prices per call, so it must count
the web_search_call items. Chat-completions responses only expose url_citation annotations with no
count, so they floor to a single billable search.
the web_search_call items, unless the response reports the billable count itself
(Bedrock's tool_usage.web_search.num_requests, which excludes open_page fetches). Chat-completions
responses only expose url_citation annotations with no count, so they floor to a single billable search.
"""
if isinstance(response_object, ResponsesAPIResponse):
count = sum(
1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
)
return max(count, 1)
return 1
if not isinstance(response_object, ResponsesAPIResponse):
return 1
reported: Final = _reported_web_search_requests(response_object)
if reported is not None:
return reported
count: Final = sum(
1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
)
return max(count, 1)
@staticmethod
def _handle_file_search_cost(

8
litellm/litellm_core_utils/secret_redaction.py
View file

@ -11,7 +11,7 @@ from typing import Final
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
_REDACTED: Final = "REDACTED"
REDACTED: Final = "REDACTED"
def _build_secret_patterns() -> "re.Pattern[str]":
@ -89,7 +89,7 @@ _SECRET_RE: Final = _build_secret_patterns()
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(_REDACTED, value)
return _SECRET_RE.sub(REDACTED, value)
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
@ -110,7 +110,7 @@ def redact_internal_details(value: str) -> str:
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
marker_index: Final = value.find(_TRACEBACK_MARKER)
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
return _INTERNAL_DETAIL_RE.sub(_REDACTED, redact_string(without_traceback))
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
def redact_structured_value(key: str | None, value: str) -> str:
@ -126,4 +126,4 @@ def redact_structured_value(key: str | None, value: str) -> str:
if scrubbed != value or key is None:
return scrubbed
rendered: Final = f"'{key}': '{value}'"
return _REDACTED if redact_string(rendered) != rendered else value
return REDACTED if redact_string(rendered) != rendered else value

45
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,9 +1,10 @@
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import BaseModel
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.litellm_core_utils.secret_redaction import REDACTED
class SensitiveDataMasker:
@ -214,6 +215,46 @@ def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dic
return masked
def redact_credentials_in_payload(data: Mapping[str, object]) -> Mapping[str, object]:
"""Return a copy of ``data`` where every value under a credential-named key is
replaced by the shared ``REDACTED`` marker, nested mappings are recursed into,
and every other value is preserved by identity.
Sensitive-key detection is delegated to the shared :class:`SensitiveDataMasker`,
so the credential names stay in one place. Unlike
:func:`mask_credentials_in_payload`, no prefix or suffix of the secret survives
and non-string secrets are covered too, which is what a payload rendered
straight to stdout needs. ``None`` is preserved so an unset credential still
reads as unset, and lists and tuples are rebuilt element by element so a
credential nested inside one is caught as well. The walk is bounded only to stop
runaway recursion, and a container sitting at that bound is replaced wholesale
rather than passed through, so burying a credential deeper than the walk goes
hides it instead of exposing it.
"""
return _redact_mapping(data, 0)
def _redact_mapping(data: Mapping[str, object], depth: int) -> Mapping[str, object]:
return {key: _redact_entry(key, value, depth) for key, value in data.items()}
def _redact_entry(key: str, value: object, depth: int) -> object:
if value is not None and _default_masker.is_sensitive_key(key):
return REDACTED
if not isinstance(value, (Mapping, list, tuple)):
return value
if depth >= DEFAULT_MAX_RECURSE_DEPTH:
return REDACTED
if isinstance(value, Mapping):
return _redact_mapping(value, depth + 1)
return _redact_sequence(value, depth + 1)
def _redact_sequence(values: Sequence[object], depth: int) -> Sequence[object]:
redacted: Final = tuple(_redact_entry("", item, depth) for item in values)
return redacted if isinstance(values, tuple) else list(redacted)
# Usage example:
"""
masker = SensitiveDataMasker()

3
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2337,6 +2337,9 @@ class CustomStreamWrapper:
else:
self.sent_last_chunk = True
processed_chunk: Final = self.finish_reason_handler()
if self.stream_options is None:
usage: Final = calculate_total_usage(chunks=self.chunks)
processed_chunk._hidden_params["usage"] = usage # pyright: ignore[reportPrivateUsage] # sync parity
# see sync __next__'s sibling branch: deliberately do NOT restore
# here - this chunk is still this call's own data, and restoring
# before returning it would corrupt the caller's own log

3
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
)
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
litellm_logging_obj_from_kwargs,
local_model_name,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
@ -621,6 +622,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=True,
litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
)
if transformed_stream is not None:
return transformed_stream
@ -755,6 +757,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=False,
litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
)
if transformed_stream is not None:
return transformed_stream

9
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -31,6 +31,7 @@ from litellm.types.llms.anthropic import (
from litellm.types.utils import AdapterCompletionStreamWrapper, Delta
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.types.utils import ModelResponseStream
@ -287,12 +288,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
applied_edits: list[AppliedEdit] | None = None,
compaction_block: CompactionBlock | None = None,
iterations_usage: list[UsageIteration] | None = None,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
):
# Wrap the upstream stream so chunks that carry both content and a
# finish_reason (fake-streamed providers) are split into two — see
# _CombinedChunkSplitter.
super().__init__(_CombinedChunkSplitter(completion_stream))
self.model = model
self._message_id: str = f"msg_{uuid.uuid4()}"
if litellm_logging_obj is not None:
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
# Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
self.tool_name_mapping = tool_name_mapping or {}
# Polyfill applied_edits on final message_delta.
@ -507,7 +512,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
{
"type": "message_start",
"message": {
"id": f"msg_{uuid.uuid4()}",
"id": self._message_id,
"type": "message",
"role": "assistant",
"content": [],
@ -741,7 +746,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
{
"type": "message_start",
"message": {
"id": f"msg_{uuid.uuid4()}",
"id": self._message_id,
"type": "message",
"role": "assistant",
"content": [],

3
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -174,6 +174,7 @@ from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
from .streaming_iterator import AnthropicStreamWrapper
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
@ -264,6 +265,7 @@ class AnthropicAdapter:
tool_name_mapping: dict[str, str] | None = None,
polyfill_result: PolyfillResult | None = None,
is_async: bool = True,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
) -> AsyncIterator[bytes] | Iterator[bytes] | None:
"""
Translate OpenAI streaming response to Anthropic format.
@ -290,6 +292,7 @@ class AnthropicAdapter:
applied_edits=applied_edits,
compaction_block=compaction_block,
iterations_usage=iterations_usage,
litellm_logging_obj=litellm_logging_obj,
)
# Return the SSE-wrapped version for proper event formatting.
if is_async:

10
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
View file

@ -20,7 +20,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
)
from litellm.types.llms.openai import ResponsesAPIResponse
from ..utils import local_model_name
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
@ -186,7 +186,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
)
return wrapper.async_anthropic_sse_wrapper()
@ -266,7 +268,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
)
return wrapper.async_anthropic_sse_wrapper()

8
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -4,7 +4,7 @@ import json
import traceback
from collections import deque
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from typing import TYPE_CHECKING, Any, Final
from litellm import verbose_logger
from litellm._uuid import uuid
@ -12,6 +12,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUs
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
class AnthropicResponsesStreamWrapper:
"""
@ -31,10 +34,13 @@ class AnthropicResponsesStreamWrapper:
self,
responses_stream: Any,
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
) -> None:
self.responses_stream = responses_stream
self.model = model
self._message_id: str = f"msg_{uuid.uuid4()}"
if litellm_logging_obj is not None:
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
self._current_block_index: int = -1
# Map item_id -> content_block_index so we can stop the right block later
self._item_id_to_block_index: dict[str, int] = {}

13
litellm/llms/anthropic/experimental_pass_through/utils.py
View file

@ -1,11 +1,14 @@
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.types.utils import ModelInfo
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
_EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
@ -24,6 +27,14 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None":
"""The logging object the bridged call logs through, when the caller supplied one."""
from litellm.litellm_core_utils.litellm_logging import Logging
candidate: Final = kwargs.get("litellm_logging_obj")
return candidate if isinstance(candidate, Logging) else None
def local_model_name(model: str, custom_llm_provider: object) -> str:
"""The id the provider itself knows, for reporting back to the caller in ``message_start``."""
return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model

22
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -7,6 +7,7 @@ from litellm.exceptions import UnsupportedParamsError
from litellm.llms.openai.chat.gpt_5_transformation import (
OpenAIGPT5Config,
_get_effort_level,
is_gpt_reasoning_series_name,
)
from litellm.types.llms.openai import AllMessageValues
@ -35,26 +36,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:
"""Check if the Azure model string refers to a gpt-5 variant.
Accepts both explicit gpt-5 model names and the ``gpt5_series/`` prefix
used for manual routing.
"""
# The gpt-5-chat* family (gpt-5-chat, gpt-5-chat-latest, gpt-5-chat-2025-08-07,
# …) are regular chat models: they support temperature and tool_choice but NOT
# reasoning_effort. They must NOT be routed through the GPT-5 reasoning path.
#
# Versioned chat models such as gpt-5.3-chat and gpt-5.1-chat ARE reasoning
# models and must stay on the GPT-5 path. The distinguishing feature is that
# the gpt-5-chat family has a literal "-chat" immediately after "gpt-5"
# (i.e. "gpt-5-chat…"), while versioned chat models interpose a minor version
# number (i.e. "gpt-5.<digit>-chat").
#
# Using a startswith("gpt-5-chat") prefix check on the normalized name (rather
# than a substring check) makes this boundary explicit and avoids any ambiguity
# if future model names coincidentally contain "gpt-5-chat" as an interior run.
_normalized: Final = model.split("/")[-1] # strip provider prefix, e.g. "azure/"
return ("gpt-5" in model and not _normalized.startswith("gpt-5-chat")) or "gpt5_series" in model
return is_gpt_reasoning_series_name(model) or "gpt5_series" in model
def get_supported_openai_params(self, model: str) -> list[str]:
"""Get supported parameters for Azure OpenAI GPT-5 models.

3
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_azure_openai_messages,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.chat.gpt_5_transformation import GPT_REASONING_SERIES_MARKERS
from litellm.types.llms.azure import (
API_VERSION_MONTH_SUPPORTED_RESPONSE_FORMAT,
API_VERSION_YEAR_SUPPORTED_RESPONSE_FORMAT,
@ -139,7 +140,7 @@ class AzureOpenAIConfig(BaseConfig):
name family needs the rename, including the ``gpt-5-chat*`` models that are excluded from
the reasoning path by https://github.com/BerriAI/litellm/issues/13781.
"""
return "gpt-5" in model or "gpt5_series" in model
return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) or "gpt5_series" in model
def _is_response_format_supported_model(self, model: str) -> bool:
"""

23
litellm/llms/azure_ai/chat/transformation.py
View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
from litellm.llms.openai.openai import OpenAIConfig
@ -207,20 +208,18 @@ class AzureAIStudioConfig(OpenAIConfig):
message["content"] = texts
return stripped_messages
def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
try:
if "/" in model:
model = model.split("/", 1)[1]
if (
model in litellm.open_ai_chat_completion_models
or model in litellm.open_ai_text_completion_models
or model in litellm.open_ai_embedding_models
):
return True
def _is_foundry_model_inference_base(self, api_base: str) -> bool:
return is_foundry_model_inference_base(api_base)
except Exception:
def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
if api_base is None or self._is_foundry_model_inference_base(api_base):
return False
return False
stripped_model: Final = model.split("/", 1)[1] if "/" in model else model
return (
stripped_model in litellm.open_ai_chat_completion_models
or stripped_model in litellm.open_ai_text_completion_models
or stripped_model in litellm.open_ai_embedding_models
)
def _get_openai_compatible_provider_info(
self,

9
litellm/llms/azure_ai/common_utils.py
View file

@ -1,5 +1,6 @@
from collections.abc import Mapping
from typing import Final, Literal
from urllib.parse import urlparse
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
@ -10,6 +11,14 @@ from litellm.types.router import GenericLiteLLMParams
AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
def is_foundry_model_inference_base(api_base: str) -> bool:
parsed: Final = urlparse(api_base)
host: Final = parsed.hostname
if host is None or not host.endswith(".services.ai.azure.com"):
return False
return "/openai/deployments" not in parsed.path
def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
"""
Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.

19
litellm/llms/azure_ai/embed/handler.py
View file

@ -1,8 +1,10 @@
from typing import Final
from urllib.parse import urlsplit, urlunsplit
from openai import OpenAI
import litellm
from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -16,6 +18,16 @@ from litellm.utils import convert_to_model_response_object
from .cohere_transformation import AzureAICohereConfig
def _foundry_models_route_base(api_base: str | None) -> str | None:
if api_base is None or not is_foundry_model_inference_base(api_base):
return api_base
parts: Final = urlsplit(api_base)
path: Final = parts.path.rstrip("/")
if path.endswith("/models"):
return api_base
return urlunsplit((parts.scheme, parts.netloc, f"{path}/models", parts.query, parts.fragment))
class AzureAIEmbedding(OpenAIChatCompletion):
def _process_response(
self,
@ -214,6 +226,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
assemble result in-order, and return
"""
resolved_api_base: Final = _foundry_models_route_base(api_base)
if aembedding is True:
return self.async_embedding(
model,
@ -223,7 +236,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
model_response,
optional_params,
api_key,
api_base,
resolved_api_base,
client,
)
@ -245,7 +258,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
model_response=model_response,
optional_params=optional_params,
api_key=api_key,
api_base=api_base,
api_base=resolved_api_base,
client=client,
)
@ -262,7 +275,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
model_response,
optional_params,
api_key,
api_base,
resolved_api_base,
client=(client if client is not None and isinstance(client, OpenAI) else None),
aembedding=aembedding,
shared_session=shared_session,

63
litellm/llms/bedrock/base_aws_llm.py
View file

@ -7,7 +7,7 @@ import urllib.parse
from collections.abc import Callable
from datetime import datetime
from threading import Lock
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args, overload
import httpx
from pydantic import BaseModel, ValidationError
@ -48,12 +48,24 @@ _STS_REGION_FROM_ENDPOINT_PATTERN: Final = re.compile(
SIGV4_COMPUTED_HEADERS: Final = frozenset({"authorization", "x-amz-date", "x-amz-security-token", "date"})
class Boto3CredentialsInfo(BaseModel):
credentials: Credentials
class BedrockRequestTarget(BaseModel):
aws_region_name: str
aws_bedrock_runtime_endpoint: str | None
class Boto3CredentialsInfo(BedrockRequestTarget):
credentials: Credentials
class BearerRequestTarget(BedrockRequestTarget):
credentials: None = None
def bedrock_bearer_token(api_key: str | None) -> str | None:
token: Final = api_key if api_key is not None else get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
return token or None
class _WebIdentityTokenClaims(BaseModel):
aud: str | list[str] | None = None
iss: str | None = None
@ -1387,9 +1399,26 @@ class BaseAWSLLM:
else:
return f"https://bedrock-runtime.{aws_region_name}.{dns_suffix}"
@overload
def _get_boto_credentials_from_optional_params(
self, optional_params: dict, model: str | None = None
) -> Boto3CredentialsInfo:
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
model: str | None = None,
bearer_token: None = None,
) -> Boto3CredentialsInfo: ...
@overload
def _get_boto_credentials_from_optional_params(
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
model: str | None = None,
*,
bearer_token: str,
) -> BearerRequestTarget: ...
def _get_boto_credentials_from_optional_params(
self, optional_params: dict, model: str | None = None, bearer_token: str | None = None
) -> Boto3CredentialsInfo | BearerRequestTarget:
"""
Get boto3 credentials from optional params
@ -1420,6 +1449,12 @@ class BaseAWSLLM:
) # https://bedrock-runtime.{region_name}.amazonaws.com
aws_external_id: Final = optional_params.pop("aws_external_id", None)
if bearer_token is not None:
return BearerRequestTarget(
aws_region_name=aws_region_name,
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
)
credentials: Final[Credentials] = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
@ -1432,7 +1467,6 @@ class BaseAWSLLM:
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
return Boto3CredentialsInfo(
credentials=credentials,
aws_region_name=aws_region_name,
@ -1451,14 +1485,9 @@ class BaseAWSLLM:
api_key: str | None = None,
supports_bearer_token: bool = True,
) -> AWSPreparedRequest:
if not supports_bearer_token:
aws_bearer_token: str | None = None
elif api_key is not None:
aws_bearer_token = api_key
else:
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
aws_bearer_token: Final = bedrock_bearer_token(api_key) if supports_bearer_token else None
if aws_bearer_token:
if aws_bearer_token is not None:
try:
from botocore.awsrequest import AWSRequest
except ImportError:
@ -1555,13 +1584,9 @@ class BaseAWSLLM:
Returns:
Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request
"""
if api_key is not None:
aws_bearer_token: str | None = api_key
else:
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
aws_bearer_token: Final = bedrock_bearer_token(api_key)
# If aws bearer token is set, use it directly in the header
if aws_bearer_token:
if aws_bearer_token is not None:
headers = headers or {}
headers["Content-Type"] = "application/json"
headers["Authorization"] = f"Bearer {aws_bearer_token}"

28
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -21,7 +21,7 @@ from litellm.rust_bridge.chat_completions import rust_chat_completions_accepts
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
from ..base_aws_llm import BaseAWSLLM, Credentials
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token
from ..common_utils import BedrockError, _get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
@ -349,17 +349,21 @@ class BedrockConverseLLM(BaseAWSLLM):
litellm_params["aws_region_name"] = aws_region_name # [DO NOT DELETE] important for async calls
credentials: Final[Credentials | None] = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
credentials: Final[Credentials | None] = (
None
if bedrock_bearer_token(api_key) is not None
else self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
)
### SET RUNTIME ENDPOINT ###

8
litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py
View file

@ -149,19 +149,15 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
- Temperature and parameter validation
"""
# Filter out AWS credentials using the existing method from BaseAWSLLM
self._get_boto_credentials_from_optional_params(optional_params, model)
inference_params: Final = {k: v for k, v in optional_params.items() if k not in self.aws_authentication_params}
# Strip routing prefixes to get the actual model ID
clean_model_id: Final = self._get_model_id(model)
# Use Moonshot's transform_request which handles message transformation
# and tool_choice="required" workaround
return MoonshotChatConfig.transform_request(
self,
model=clean_model_id,
messages=messages,
optional_params=optional_params,
optional_params=inference_params,
litellm_params=litellm_params,
headers=headers,
)

59
litellm/llms/bedrock/embed/embedding.py
View file

@ -6,7 +6,7 @@ import copy
import json
import urllib.parse
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Final, get_args
from typing import TYPE_CHECKING, Final, get_args, overload
import httpx
@ -26,7 +26,7 @@ from litellm.types.llms.bedrock import (
)
from litellm.types.utils import EmbeddingResponse, LlmProviders
from ..base_aws_llm import BaseAWSLLM
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token
from ..common_utils import BedrockError
from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
from .amazon_titan_g1_transformation import AmazonTitanG1Config
@ -42,14 +42,25 @@ if TYPE_CHECKING:
class BedrockEmbedding(BaseAWSLLM):
@overload
def _load_credentials(
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
bearer_token: None = None,
) -> tuple[Credentials, str]: ...
@overload
def _load_credentials(
self,
optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
bearer_token: str,
) -> tuple[None, str]: ...
def _load_credentials(
self,
optional_params: dict,
) -> tuple[Any, str]:
try:
from botocore.credentials import Credentials
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
bearer_token: str | None = None,
) -> tuple[Credentials | None, str]:
## CREDENTIALS ##
# pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
aws_secret_access_key: Final = optional_params.pop("aws_secret_access_key", None)
@ -78,17 +89,21 @@ class BedrockEmbedding(BaseAWSLLM):
if aws_region_name is None:
aws_region_name = "us-west-2"
credentials: Final[Credentials] = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
credentials: Final[Credentials | None] = (
None
if bearer_token is not None
else self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
)
return credentials, aws_region_name
@ -233,7 +248,7 @@ class BedrockEmbedding(BaseAWSLLM):
client: HTTPHandler | None,
timeout: float | httpx.Timeout | None,
batch_data: list[dict],
credentials: Any,
credentials: Credentials | None,
extra_headers: dict | None,
endpoint_url: str,
aws_region_name: str,
@ -301,7 +316,7 @@ class BedrockEmbedding(BaseAWSLLM):
client: AsyncHTTPHandler | None,
timeout: float | httpx.Timeout | None,
batch_data: list[dict],
credentials: Any,
credentials: Credentials | None,
extra_headers: dict | None,
endpoint_url: str,
aws_region_name: str,
@ -383,7 +398,9 @@ class BedrockEmbedding(BaseAWSLLM):
litellm_params: dict,
api_key: str | None = None,
) -> EmbeddingResponse:
credentials, aws_region_name = self._load_credentials(optional_params)
credentials, aws_region_name = self._load_credentials(
optional_params, bearer_token=bedrock_bearer_token(api_key)
)
### TRANSFORMATION ###
unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed

6
litellm/llms/bedrock/image_edit/handler.py
View file

@ -29,7 +29,7 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.types.utils import ImageResponse
from ..base_aws_llm import BaseAWSLLM
from ..base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from ..common_utils import BedrockError
if TYPE_CHECKING:
@ -198,7 +198,9 @@ class BedrockImageEdit(BaseAWSLLM):
Returns:
BedrockImageEditPreparedRequest: The prepared request object
"""
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(
optional_params, model, bearer_token=bedrock_bearer_token(api_key)
)
# Use the existing ARN-aware provider detection method
bedrock_provider: Final = self.get_bedrock_invoke_provider(model)

6
litellm/llms/bedrock/image_generation/image_handler.py
View file

@ -29,7 +29,7 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.types.utils import ImageResponse
from ..base_aws_llm import BaseAWSLLM
from ..base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from ..common_utils import BedrockError
if TYPE_CHECKING:
@ -220,7 +220,9 @@ class BedrockImageGeneration(BaseAWSLLM):
prepped (httpx.Request): The prepared request object
body (bytes): The request body
"""
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(
optional_params, model, bearer_token=bedrock_bearer_token(api_key)
)
# Use the existing ARN-aware provider detection method
bedrock_provider: Final = self.get_bedrock_invoke_provider(model)

14
litellm/llms/custom_httpx/container_handler.py
View file

@ -139,7 +139,7 @@ def _build_query_params(
return {name: value if isinstance(value, str) else str(value) for name, value in supplied if value is not None}
def _error_message_from_response(response: httpx.Response) -> str:
def error_message_from_response(response: httpx.Response) -> str:
try:
body: Final = response.json()
except ValueError:
@ -153,6 +153,16 @@ def _error_message_from_response(response: httpx.Response) -> str:
return response.text
def raise_for_error_status(response: httpx.Response, container_provider_config: "BaseContainerConfig") -> None:
if not httpx.codes.is_error(response.status_code):
return
raise container_provider_config.get_error_class(
error_message=error_message_from_response(response),
status_code=response.status_code,
headers=response.headers,
)
def _transform_response(
response: httpx.Response,
returns_binary: bool,
@ -163,7 +173,7 @@ def _transform_response(
if httpx.codes.is_error(response.status_code):
raise BaseLLMException(
status_code=response.status_code,
message=_error_message_from_response(response),
message=error_message_from_response(response),
headers=dict(response.headers),
)

197
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -77,6 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import (
BaseVectorStoreFilesConfig,
)
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.custom_httpx.container_handler import raise_for_error_status
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -99,9 +100,12 @@ from litellm.types.containers.main import (
)
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
from litellm.types.integrations.custom_logger import (
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
converted_stream_requested,
is_interception_internal_key,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -2760,6 +2764,7 @@ class BaseLLMHTTPHandler:
)
if self._has_agentic_completion_hook(logging_obj):
agentic_kwargs: Final = dict(litellm_params) # mutable-ok: agentic hooks mutate kwargs in place
final_response: Final = run_async_function(
self._call_agentic_completion_hooks,
response=initial_response,
@ -2770,10 +2775,19 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(litellm_params),
kwargs=agentic_kwargs,
api_surface="responses",
)
return final_response if final_response is not None else initial_response
result: Final = final_response if final_response is not None else initial_response
if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
return self._wrap_responses_response_as_fake_stream(
result=result,
model=model,
responses_api_provider_config=responses_api_provider_config,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
)
return result
return initial_response
@ -2939,6 +2953,7 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
agentic_kwargs: Final = dict(litellm_params) # mutable-ok: agentic hooks mutate kwargs in place
final_response: Final = await self._call_agentic_completion_hooks(
response=initial_response,
model=model,
@ -2948,15 +2963,12 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(litellm_params),
kwargs=agentic_kwargs,
api_surface="responses",
)
result: Final = final_response if final_response is not None else initial_response
interception_converted_stream: Final = litellm_params.get(
"_code_interpreter_interception_converted_stream"
) or litellm_params.get("_websearch_interception_converted_stream")
if interception_converted_stream and not litellm_params.get("_agentic_loop_depth"):
if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
return self._wrap_responses_response_as_fake_stream(
result=result,
model=model,
@ -5420,8 +5432,7 @@ class BaseLLMHTTPHandler:
kwargs_for_followup: Final = {
k: v
for k, v in kwargs.items()
if not k.startswith("_websearch_interception")
and not k.startswith("_compression_interception")
if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES)
and k != "_code_interpreter_interception_converted_stream"
and k not in internal_keys
and k not in optional_params
@ -8753,17 +8764,19 @@ class BaseLLMHTTPHandler:
json=data,
timeout=timeout,
)
return container_provider_config.transform_container_create_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_create_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_create_handler(
self,
@ -8829,17 +8842,19 @@ class BaseLLMHTTPHandler:
json=data,
timeout=timeout,
)
return container_provider_config.transform_container_create_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_create_response(
raw_response=response,
logging_obj=logging_obj,
)
def container_list_handler(
self,
@ -8919,17 +8934,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_list_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_list_handler(
self,
@ -8996,17 +9013,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_list_response(
raw_response=response,
logging_obj=logging_obj,
)
def container_retrieve_handler(
self,
@ -9084,17 +9103,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_retrieve_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_retrieve_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_retrieve_handler(
self,
@ -9161,17 +9182,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_retrieve_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_retrieve_response(
raw_response=response,
logging_obj=logging_obj,
)
def container_delete_handler(
self,
@ -9249,17 +9272,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_delete_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_delete_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_delete_handler(
self,
@ -9326,17 +9351,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_delete_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_delete_response(
raw_response=response,
logging_obj=logging_obj,
)
def container_file_list_handler(
self,
@ -9418,17 +9445,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_file_list_handler(
self,
@ -9497,17 +9526,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
def container_file_content_handler(
self,
@ -9583,17 +9614,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
async def async_container_file_content_handler(
self,
@ -9659,17 +9692,19 @@ class BaseLLMHTTPHandler:
headers=headers,
params=params or None,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
raise_for_error_status(
response=response,
container_provider_config=container_provider_config,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
###### VECTOR STORE HANDLER ######
@staticmethod

120
litellm/llms/dashscope/cost_calculator.py
View file

@ -7,11 +7,13 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
"""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.litellm_core_utils.llm_cost_calc.utils import (
apply_off_peak_pricing,
parse_completion_tokens_details,
parse_prompt_tokens_details,
)
@ -32,6 +34,19 @@ class TokenBreakdown:
return self.text_tokens + self.cached_tokens + self.cache_creation_tokens
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
cache_read_rate: float
cache_creation_rate: float
output_rate: float
reasoning_rate: float | None
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
prompt_details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = prompt_details["cache_hit_tokens"]
@ -57,69 +72,75 @@ def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) ->
return float(value)
def _calculate_prompt_cost(
breakdown: TokenBreakdown,
model_info: ModelInfo,
tier: dict | None,
) -> float:
if tier is not None:
return (
(breakdown.text_tokens * tier_rate(tier, "input_cost_per_token"))
+ (breakdown.cached_tokens * tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"))
+ (
breakdown.cache_creation_tokens
* tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
)
)
input_cost: Final = float(model_info.get("input_cost_per_token") or 0.0)
cache_read_cost: Final = _flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token")
cache_creation_cost: Final = _flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token")
return (
(breakdown.text_tokens * input_cost)
+ (breakdown.cached_tokens * cache_read_cost)
+ (breakdown.cache_creation_tokens * cache_creation_cost)
def _flat_rates(model_info: ModelInfo) -> TokenRates:
reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
return TokenRates(
input_rate=float(model_info.get("input_cost_per_token") or 0.0),
cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"),
output_rate=float(model_info.get("output_cost_per_token") or 0.0),
reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
)
def _calculate_completion_cost(
breakdown: TokenBreakdown,
model_info: ModelInfo,
tier: dict | None,
) -> float:
def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
# spelling out only input rates would serve every completion for free, so there the model's
# own output rates stand in
tier_declares_output: Final = tier is not None and "output_cost_per_token" in tier
output_cost: Final = (
tier_rate(tier, "output_cost_per_token")
if tier_declares_output
else float(model_info.get("output_cost_per_token") or 0.0)
)
tier_declares_reasoning: Final = tier is not None and "output_cost_per_reasoning_token" in tier
model_reasoning_rate: Final = None if tier_declares_output else model_info.get("output_cost_per_reasoning_token")
reasoning_cost: Final = (
tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
if tier_declares_reasoning
else float(model_reasoning_rate)
if model_reasoning_rate is not None
else output_cost
flat_rates: Final = _flat_rates(model_info)
tier_declares_output: Final = "output_cost_per_token" in tier
tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier
return TokenRates(
input_rate=tier_rate(tier, "input_cost_per_token"),
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"),
output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate,
reasoning_rate=(
tier_rate(tier, "output_cost_per_reasoning_token")
if tier_declares_reasoning
else None
if tier_declares_output
else flat_rates.reasoning_rate
),
)
return (breakdown.completion_tokens * output_cost) + (breakdown.reasoning_tokens * reasoning_cost)
def _off_peak_rates(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates:
input_rate, output_rate, cache_read_rate = apply_off_peak_pricing(
model_info, current_time, rates.input_rate, rates.output_rate, rates.cache_read_rate
)
return replace(rates, input_rate=input_rate, output_rate=output_rate, cache_read_rate=cache_read_rate)
def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashscope") -> tuple[float, float]:
def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
prompt_cost: Final = (
(breakdown.text_tokens * rates.input_rate)
+ (breakdown.cached_tokens * rates.cache_read_rate)
+ (breakdown.cache_creation_tokens * rates.cache_creation_rate)
)
completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
breakdown.reasoning_tokens * rates.billed_reasoning_rate
)
return prompt_cost, completion_cost
def cost_per_token(
model: str,
usage: Usage,
custom_llm_provider: str = "dashscope",
current_time: datetime | None = None,
) -> tuple[float, float]:
"""
Calculate cost per token for Dashscope models.
Supports both tiered and flat pricing with cached and reasoning tokens.
Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the
model's off_peak_pricing rates while one of its windows is open.
Args:
model: Model name without provider prefix
usage: LiteLLM Usage block
custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases
current_time: The moment the request is billed at; defaults to now, UTC
Returns:
Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
@ -133,8 +154,7 @@ def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashsco
if tiered_pricing
else None
)
standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
rates: Final = _off_peak_rates(model_info, current_time, standard_rates)
prompt_cost: Final = _calculate_prompt_cost(breakdown=breakdown, model_info=model_info, tier=tier)
completion_cost: Final = _calculate_completion_cost(breakdown=breakdown, model_info=model_info, tier=tier)
return prompt_cost, completion_cost
return _bill(breakdown, rates)

26
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -61,6 +61,14 @@ def _get_effort_level(value: str | dict | None) -> str | None:
return None
GPT_REASONING_SERIES_MARKERS: Final = ("gpt-5", "gpt-6")
def is_gpt_reasoning_series_name(model: str) -> bool:
normalized: Final = model.split("/")[-1]
return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) and not normalized.startswith("gpt-5-chat")
class OpenAIGPT5Config(OpenAIGPTConfig):
"""Configuration for gpt-5 models including GPT-5-Codex variants.
@ -73,21 +81,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:
# The gpt-5-chat* family (gpt-5-chat, gpt-5-chat-latest, gpt-5-chat-2025-08-07,
# …) are regular chat models: they support temperature and tool_choice but NOT
# reasoning_effort. They must NOT be routed through the GPT-5 reasoning path.
#
# Versioned chat models such as gpt-5.3-chat and gpt-5.1-chat ARE reasoning
# models and must stay on the GPT-5 path. The distinguishing feature is that
# the gpt-5-chat family has a literal "-chat" immediately after "gpt-5"
# (i.e. "gpt-5-chat…"), while versioned chat models interpose a minor version
# number (i.e. "gpt-5.<digit>-chat").
#
# Using a startswith("gpt-5-chat") prefix check on the normalized name (rather
# than a substring check) makes this boundary explicit and avoids any ambiguity
# if future model names coincidentally contain "gpt-5-chat" as an interior run.
_normalized: Final = model.split("/")[-1] # strip provider prefix, e.g. "openai/"
return "gpt-5" in model and not _normalized.startswith("gpt-5-chat")
return is_gpt_reasoning_series_name(model)
@classmethod
def is_model_gpt_5_search_model(cls, model: str) -> bool:
@ -122,6 +116,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
def is_model_gpt_5_4_plus_model(cls, model: str) -> bool:
"""Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro)."""
model_name: Final = model.split("/")[-1]
if model_name.startswith("gpt-6"):
return True
if not model_name.startswith("gpt-5."):
return False
try:

9
litellm/llms/openai/common_utils.py
View file

@ -11,6 +11,7 @@ import time
import uuid
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
from urllib.parse import urlsplit
import httpx
import openai
@ -43,6 +44,14 @@ _OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(OpenAI)
_AZURE_OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(AzureOpenAI)
_OPENAI_API_HOST: Final[str] = "api.openai.com"
def is_openai_backed_api_base(api_base: str) -> bool:
hostname: Final = urlsplit(api_base).hostname
return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
class OpenAIError(BaseLLMException):
def __init__(
self,

8
litellm/llms/openai/openai.py
View file

@ -2,7 +2,6 @@ import time
import types
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
from urllib.parse import urlparse
import httpx
@ -55,6 +54,7 @@ from .common_utils import (
OpenAIError,
build_output_token_limit_response,
drop_params_from_unprocessable_entity_error,
is_openai_backed_api_base,
is_output_token_limit_error,
)
from .workload_identity import resolve_openai_workload_identity_config
@ -1190,10 +1190,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"""
if stream_options is not None:
return {"stream_options": stream_options}
else:
# by default litellm will include usage for openai endpoints
if api_base is None or urlparse(api_base).hostname == "api.openai.com":
return {"stream_options": {"include_usage": True}}
if api_base is None or is_openai_backed_api_base(api_base):
return {"stream_options": {"include_usage": True}}
return {}
# Embedding

366
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -33,8 +33,9 @@ import time
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from pydantic import BaseModel, TypeAdapter
@ -42,6 +43,7 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import (
@ -74,6 +76,7 @@ from litellm.types.llms.openai import (
OutputTextDoneEvent,
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
@ -115,6 +118,199 @@ class ResponsesStreamChunk(TypedDict, total=False):
content_index: ReadOnly[int]
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "message": "content"}
)
_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
def _item_rewrite_field(item: Mapping[str, object]) -> str | None:
item_type: Final = item.get("type")
if item_type is None:
return "content" if "content" in item else None
if not isinstance(item_type, str):
return None
return _PATCHABLE_ITEM_FIELDS.get(item_type)
def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None:
field: Final = _item_rewrite_field(item)
if field is None or not isinstance(rewritten, Mapping):
return None
rewritten_content: Final = rewritten.get("content")
if isinstance(item.get(field), str) and isinstance(rewritten_content, str):
return {**item, field: rewritten_content} # mutable-ok: request input items must stay JSON-plain dicts
rewritten_row: Final = cast("AllMessageValues", rewritten) # cast-ok: guardrails hand back chat-shaped rows
converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
[rewritten_row] # mutable-ok: converter signature takes a list
)
if len(converted_items) != 1 or not isinstance(converted_items[0], Mapping):
return None
first_converted: Final = cast("Mapping[str, object]", converted_items[0]) # cast-ok: isinstance-checked above
converted_value: Final = first_converted.get(field)
if converted_value is None:
return None
return {**item, field: converted_value} # mutable-ok: request input items must stay JSON-plain dicts
def _is_function_call_item(item: object) -> bool:
return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call")
def _last_message_role(messages: Sequence[object]) -> str | None:
if not messages:
return None
last: Final = messages[-1]
role: Final = last.get("role") if isinstance(last, Mapping) else getattr(last, "role", None)
return role if isinstance(role, str) else None
def _provenance_unit_bounds(
raw_input: Sequence[object],
solo_conversions: Sequence[Sequence[object]],
) -> tuple[tuple[int, int], ...]:
trailing_roles: Final = tuple(
accumulate(
(_last_message_role(messages) for messages in solo_conversions),
lambda previous, current: current if current is not None else previous,
)
)
start_indexes: Final = tuple(
index
for index in range(len(raw_input))
if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant")
)
return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input))))
def _input_item_provenance(
raw_input: Sequence[object],
expected_messages: Sequence[object],
) -> tuple[Mapping[int, int], frozenset[int]] | None:
if not all(isinstance(item, Mapping) for item in raw_input):
return None
solo_conversions: Final = tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
)
for item in raw_input
)
full_conversion: Final = tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
)
)
if full_conversion != tuple(expected_messages):
return None
units: Final = _provenance_unit_bounds(raw_input, solo_conversions)
unit_messages: Final = tuple(
tuple(solo_conversions[start])
if end - start == 1
else tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
)
)
for start, end in units
)
if tuple(message for messages in unit_messages for message in messages) != full_conversion:
return None
boundaries: Final = tuple(accumulate((len(messages) for messages in unit_messages), initial=0))
item_for_message: Final = MappingProxyType(
{
message_index: start
for unit_index, (start, end) in enumerate(units)
if end - start == 1
for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
}
)
tainted: Final = frozenset(
message_index
for unit_index, (start, end) in enumerate(units)
if end - start > 1
for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
)
return item_for_message, tainted
class _RequestFields(NamedTuple):
input: tuple[object, ...]
instructions: str | None
class _ExtractedInputs(NamedTuple):
inputs: GenericGuardrailAPIInputs
task_mappings: tuple[tuple[int, int | None], ...]
def _patched_request_fields(
raw_input: object,
instructions: object,
original_messages: Sequence[object],
structured_messages: Sequence[object],
) -> _RequestFields | None:
if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
return None
offset: Final = 1 if instructions else 0
provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
if provenance is None:
return None
item_for_message, tainted = provenance
changed: Final = tuple(
(index, rewritten)
for index, (original, rewritten) in enumerate(zip(original_messages, structured_messages))
if original != rewritten
)
instruction_rewrites: Final = tuple(rewritten for index, rewritten in changed if index < offset)
rewritten_instructions: Final = (
instruction_rewrites[0].get("content")
if instruction_rewrites and isinstance(instruction_rewrites[0], Mapping)
else instructions
)
instructions_value: Final = rewritten_instructions if isinstance(rewritten_instructions, str) else None
if rewritten_instructions is not None and instructions_value is None:
return None
body_changes: Final = tuple((index - offset, rewritten) for index, rewritten in changed if index >= offset)
if any(message_index in tainted or message_index not in item_for_message for message_index, _ in body_changes):
return None
replacements: Final = MappingProxyType(
{
item_for_message[message_index]: _rewritten_input_item(
cast("Mapping[str, object]", raw_input[item_for_message[message_index]]), # cast-ok: checked Mappings
rewritten,
)
for message_index, rewritten in body_changes
}
)
if len(replacements) != len(body_changes) or any(item is None for item in replacements.values()):
return None
return _RequestFields(
input=tuple(replacements.get(index, item) for index, item in enumerate(raw_input)),
instructions=instructions_value,
)
def _patch_or_convert_request_fields(
raw_input: object,
instructions: object,
original_messages: Sequence[object],
structured_messages: Sequence[AllMessageValues],
) -> _RequestFields | None:
if not isinstance(structured_messages, list):
return None
patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages)
if patched is not None:
return patched
input_items, converted_instructions = (
LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
)
return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
def _next_stream_sequence_number(responses_so_far: Sequence[Any] | None) -> int:
sequence_numbers: Final = (
item.get("sequence_number") if isinstance(item, dict) else getattr(item, "sequence_number", None)
@ -162,9 +358,8 @@ class OpenAIResponsesHandler(BaseTranslation):
Handles both string input and list of message objects.
"""
input_data: Final[str | ResponseInputParam | None] = data.get("input")
if input_data is None:
if not isinstance(input_data, (str, list)):
return data
structured_messages: Final = self.get_structured_messages(data)
raw_tools: Final = data.get("tools")
original_tools: Final[tuple[Mapping[str, object], ...]] = (
@ -173,94 +368,93 @@ class OpenAIResponsesHandler(BaseTranslation):
flattened_tool_groups: Final = tuple(
form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools)
)
flattened_tools: Final = tuple(
cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list
for group in flattened_tool_groups
for tool in group
)
tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list
copy.deepcopy(flattened_tools)
)
# Handle simple string input
if isinstance(input_data, str):
inputs = GenericGuardrailAPIInputs(texts=[input_data])
if tools_to_check:
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages
# Include model information if available
model = data.get("model")
if model:
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
self._apply_guardrailed_tools_to_data(
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
)
verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
return data
# Handle list input (ResponseInputParam)
if not isinstance(input_data, list):
extracted: Final = self._extract_guardrail_inputs(data, input_data, flattened_tool_groups)
if not extracted.inputs.get("texts"):
return data
if structured_messages:
extracted.inputs["structured_messages"] = structured_messages
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=extracted.inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
self._apply_guardrailed_tools_to_data(
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
)
written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs)
if written_back is not None:
data["input"] = list(written_back.input) # mutable-ok: JSON body
if written_back.instructions is None:
data.pop("instructions", None)
else:
data["instructions"] = written_back.instructions # rebind-ok: data is an out-param
elif isinstance(input_data, str):
guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data # rebind-ok: data is an out-param
else:
await self._apply_guardrail_responses_to_input(
messages=input_data,
responses=guardrailed_inputs.get("texts") or (),
task_mappings=extracted.task_mappings,
)
verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
return data
def _extract_guardrail_inputs(
self,
data: Mapping[str, object],
input_data: "str | ResponseInputParam",
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
) -> _ExtractedInputs:
texts_to_check: Final[list[str]] = []
images_to_check: Final[list[str]] = []
task_mappings: Final[list[tuple[int, int | None]]] = []
# Step 1: Extract all text content, images, and tools
for msg_idx, message in enumerate(input_data):
self._extract_input_text_and_images(
message=message,
msg_idx=msg_idx,
texts_to_check=texts_to_check,
images_to_check=images_to_check,
task_mappings=task_mappings,
tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list
copy.deepcopy(
tuple(
cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list
for group in flattened_tool_groups
for tool in group
)
)
)
if isinstance(input_data, str):
texts_to_check.append(input_data)
else:
for msg_idx, message in enumerate(input_data):
self._extract_input_text_and_images(
message=message,
msg_idx=msg_idx,
texts_to_check=texts_to_check,
images_to_check=images_to_check,
task_mappings=task_mappings,
)
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tools_to_check:
inputs["tools"] = tools_to_check
model: Final = data.get("model")
if isinstance(model, str):
inputs["model"] = model
return _ExtractedInputs(inputs=inputs, task_mappings=tuple(task_mappings))
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tools_to_check:
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages
# Include model information if available
model = data.get("model")
if model:
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
self._apply_guardrailed_tools_to_data(
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
)
# Step 3: Map guardrail responses back to original input structure
await self._apply_guardrail_responses_to_input(
messages=input_data,
responses=guardrailed_texts,
task_mappings=task_mappings,
)
verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", input_data)
return data
@staticmethod
def _written_back_request_fields(
data: Mapping[str, object],
structured_messages: Sequence[AllMessageValues] | None,
guardrailed_inputs: GenericGuardrailAPIInputs,
) -> _RequestFields | None:
guardrailed: Final = guardrailed_inputs.get("structured_messages")
if guardrailed is None or guardrailed is structured_messages:
return None
return _patch_or_convert_request_fields(
data.get("input"),
data.get("instructions"),
structured_messages or (),
guardrailed,
)
def extract_request_tool_names(self, data: dict) -> list[str]:
"""Extract tool names from Responses API request (tools[].name for function
@ -331,8 +525,8 @@ class OpenAIResponsesHandler(BaseTranslation):
async def _apply_guardrail_responses_to_input(
self,
messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam
responses: list[str],
task_mappings: list[tuple[int, int | None]],
responses: Sequence[str],
task_mappings: Sequence[tuple[int, int | None]],
) -> None:
"""
Apply guardrail responses back to input messages.

3
litellm/llms/openai/responses/transformation.py
View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import *
@ -88,7 +89,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
parts: Final = model.split("/")
if len(parts) > 1 and parts[0] not in ("openai",):
return False
return "gpt-5" in model and "gpt-5-chat" not in model
return is_gpt_reasoning_series_name(model)
@staticmethod
def _supports_reasoning_effort_none(model: str) -> bool:

23
litellm/llms/vertex_ai/common_utils.py
View file

@ -998,6 +998,16 @@ def replace_project_and_location_in_route(requested_route: str, vertex_project:
return modified_route
def _api_version_for_route(requested_route: str) -> Literal["v1", "v1beta1"]:
return "v1beta1" if "cachedContent" in requested_route else "v1"
def _with_api_version(requested_route: str) -> str:
if not requested_route.startswith("/projects/"):
return requested_route
return f"/{_api_version_for_route(requested_route)}{requested_route}"
def construct_target_url(
base_url: str,
requested_route: str,
@ -1017,18 +1027,19 @@ def construct_target_url(
new_base_url: Final = httpx.URL(base_url)
if "locations" in requested_route: # contains the target project id + location
if vertex_project and vertex_location:
requested_route = replace_project_and_location_in_route(requested_route, vertex_project, vertex_location)
return new_base_url.copy_with(path=requested_route)
targeted_route: Final = (
replace_project_and_location_in_route(requested_route, vertex_project, vertex_location)
if vertex_project and vertex_location
else requested_route
)
return new_base_url.copy_with(path=_with_api_version(targeted_route))
"""
- Add endpoint version (e.g. v1beta for cachedContent, v1 for rest)
- Add default project id
- Add default location
"""
vertex_version: Literal["v1", "v1beta1"] = "v1"
if "cachedContent" in requested_route:
vertex_version = "v1beta1"
vertex_version: Literal["v1", "v1beta1"] = _api_version_for_route(requested_route)
# Check if the requested route starts with a version
# e.g. /v1beta1/publishers/google/models/gemini-3-pro-preview:streamGenerateContent

36
litellm/main.py
View file

@ -26,6 +26,7 @@ from copy import deepcopy
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
from urllib.parse import urlsplit
from litellm._logging import _redact_string
from litellm._uuid import uuid
@ -60,6 +61,7 @@ if TYPE_CHECKING:
from litellm.types.utils import TokenCountResponse
from litellm.constants import (
AZURE_OPENAI_AUDIO_PROVIDERS,
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
)
@ -984,6 +986,12 @@ def mock_completion(
_OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
_OPENAI_API_HOST: Final = "api.openai.com"
def _is_openai_backed_api_base(api_base: str) -> bool:
hostname: Final = urlsplit(api_base).hostname
return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
def _resolve_openai_api_base(api_base: str | None) -> str:
@ -1053,7 +1061,7 @@ def responses_api_bridge_check(
# natively by Chat Completions with reasoning on, so custom-only requests stay on
# chat and keep their native custom tool_call response shape.
# - The UNSET-effort arm only fires against endpoints known to enforce that
# constraint (the default OpenAI endpoint, or Azure OpenAI where api_base is
# constraint (any api.openai.com host, or Azure OpenAI where api_base is
# always set): chat-only OpenAI-compatible backends registered under the openai
# provider with a custom api_base and gpt-5.4+ model names serve tools without
# reasoning fine and have no /responses route, so they keep pre-existing
@ -1068,14 +1076,15 @@ def responses_api_bridge_check(
reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None
else:
reasoning_active = reasoning_effort != "none"
# The reasoning+tools constraint is enforced only by the real OpenAI endpoint (and Azure OpenAI).
# Resolve the effective base arg>global>env>default exactly as the chat handler does, so a custom
# base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and
# bridged to a /responses route it lacks. A whitespace-only base collapses to the default too.
resolved_api_base: Final = _resolve_openai_api_base(api_base)
on_constraint_enforcing_endpoint: Final = custom_llm_provider == "azure" or resolved_api_base.strip() in (
"",
_OPENAI_DEFAULT_API_BASE,
# The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com
# host (the default URL or a PrivateLink hostname such as <region>.privatelink.api.openai.com) and
# by Azure OpenAI. Resolve the effective base arg>global>env>default exactly as the chat handler
# does, so a custom base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread
# as the default and bridged to a /responses route it lacks. A whitespace-only base collapses to
# the default too.
resolved_api_base: Final = _resolve_openai_api_base(api_base).strip()
on_constraint_enforcing_endpoint: Final = (
custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base)
)
if (
custom_llm_provider in ("openai", "azure")
@ -7769,7 +7778,7 @@ def transcription(
provider=LlmProviders(custom_llm_provider),
)
if custom_llm_provider == "azure" and provider_config is None:
if custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS and provider_config is None:
# azure configs
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -8056,7 +8065,10 @@ def speech(
custom_llm_provider=custom_llm_provider,
)
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers:
if custom_llm_provider == "openai" or (
custom_llm_provider in litellm.openai_compatible_providers
and custom_llm_provider not in AZURE_OPENAI_AUDIO_PROVIDERS
):
if voice is None or not (isinstance(voice, str)):
raise litellm.BadRequestError(
message="'voice' is required to be passed as a string for OpenAI TTS",
@ -8110,7 +8122,7 @@ def speech(
aspeech=aspeech,
shared_session=shared_session,
)
elif custom_llm_provider == "azure":
elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
# Check if this is Azure Speech Service (Cognitive Services TTS)
if model.startswith("speech/"):
from litellm.llms.azure.text_to_speech.transformation import (

112
litellm/model_prices_and_context_window_backup.json
View file

@ -10305,6 +10305,24 @@
"supports_vision": true,
"supports_web_search": true
},
"azure_ai/grok-4.6": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/grok-4-6-comes-to-microsoft-foundry-models-built-for-long-horizon-reasoning-and-/4547578",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure_ai/grok-4-fast-non-reasoning": {
"deprecation_date": "2026-05-01",
"input_cost_per_token": 2e-07,
@ -29277,6 +29295,75 @@
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
"cache_creation_input_token_cost_flex": 6.25e-06,
"cache_creation_input_token_cost_priority": 2.5e-05,
"cache_read_input_token_cost": 1e-06,
"cache_read_input_token_cost_above_272k_tokens": 2e-06,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
"cache_read_input_token_cost_flex": 5e-07,
"cache_read_input_token_cost_priority": 2e-06,
"input_cost_per_token": 1e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"input_cost_per_token_above_272k_tokens_priority": 4e-05,
"input_cost_per_token_batches": 5e-06,
"input_cost_per_token_flex": 5e-06,
"input_cost_per_token_priority": 2e-05,
"litellm_provider": "openai",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"output_cost_per_token_above_272k_tokens": 7.5e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
"output_cost_per_token_above_272k_tokens_priority": 0.00015,
"output_cost_per_token_batches": 2.5e-05,
"output_cost_per_token_flex": 2.5e-05,
"output_cost_per_token_priority": 0.0001,
"regional_processing_uplift_multiplier_eu": 1.1,
"regional_processing_uplift_multiplier_us": 1.1,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
"supports_none_reasoning_effort": false,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
@ -52911,6 +52998,11 @@
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
"search_context_size_medium": 0.012
},
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -52945,6 +53037,11 @@
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
"output_cost_per_token": 1.32e-05,
"output_cost_per_token_above_272k_tokens": 1.98e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
"search_context_size_medium": 0.012
},
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -53007,6 +53104,11 @@
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
"output_cost_per_token": 1.32e-06,
"output_cost_per_token_above_272k_tokens": 1.98e-06,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
"search_context_size_medium": 0.012
},
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -53195,6 +53297,11 @@
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
"search_context_size_medium": 0.012
},
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -53226,6 +53333,11 @@
"cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
"output_cost_per_token": 1.65e-05,
"output_cost_per_token_above_272k_tokens": 2.475e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
"search_context_size_medium": 0.012
},
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,

43
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -448,6 +448,17 @@ def _append_query_params(url: str, params: dict[str, str]) -> str:
return urlunparse(parsed._replace(query=urlencode(query_params)))
def _resolve_mcp_server_by_name_or_id(lookup: str, client_ip: str | None) -> MCPServer | None:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
by_name: Final = global_mcp_server_manager.get_mcp_server_by_name(lookup, client_ip=client_ip)
if by_name is not None:
return by_name
return global_mcp_server_manager.get_mcp_server_by_id(lookup, client_ip=client_ip)
def _resolve_oauth2_server_for_root_endpoints(
client_ip: str | None = None,
) -> MCPServer | None:
@ -1766,10 +1777,6 @@ async def authorize(
resource: str | None = None,
):
# Redirect to real OAuth provider with PKCE support
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if mcp_server_name is None and client_id and is_gateway_dcr_client_id(client_id):
if is_proxy_api_resource(request, resource):
return await native_client_authorize(
@ -1797,9 +1804,7 @@ async def authorize(
lookup_name: Final[str | None] = mcp_server_name or client_id
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = (
global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip) if lookup_name else None
)
mcp_server = _resolve_mcp_server_by_name_or_id(lookup_name, client_ip) if lookup_name else None
if mcp_server is None and mcp_server_name is None:
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
@ -1855,10 +1860,6 @@ async def token_endpoint(
3. Return the token
4. Return a virtual key in this response
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if mcp_server_name is None and is_gateway_dcr_client_id(client_id):
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # circular import at module load
master_key,
@ -1882,7 +1883,7 @@ async def token_endpoint(
lookup_name: Final = mcp_server_name or client_id
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip)
mcp_server = _resolve_mcp_server_by_name_or_id(lookup_name, client_ip)
if mcp_server is None and mcp_server_name is None:
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
@ -2288,10 +2289,6 @@ async def _build_oauth_protected_resource_response(
Returns:
OAuth protected resource metadata dict
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
request_base_url: Final = get_request_base_url(request)
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
explicitly_named: Final = mcp_server_name is not None
@ -2304,7 +2301,7 @@ async def _build_oauth_protected_resource_response(
mcp_server: MCPServer | None = None
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
mcp_server = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
# Build resource URL based on the pattern
if mcp_server_name:
@ -2562,10 +2559,6 @@ def _build_oauth_authorization_server_response(
registry lookups; unlike :func:`_build_oauth_protected_resource_response`
it does not need to await any upstream IO.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
request_base_url: Final = get_request_base_url(request)
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
explicitly_named: Final = mcp_server_name is not None
@ -2583,7 +2576,7 @@ def _build_oauth_authorization_server_response(
mcp_server: MCPServer | None = None
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
mcp_server = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
_raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server")
@ -2709,10 +2702,6 @@ async def oauth_authorization_server_legacy(request: Request, mcp_server_name: s
@router.post("/{mcp_server_name}/register")
@router.post("/register")
async def register_client(request: Request, mcp_server_name: str | None = None):
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
# Get the correct base URL considering X-Forwarded-* headers
request_base_url: Final = get_request_base_url(request)
@ -2748,7 +2737,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
)
return dummy_return
mcp_server: Final = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
mcp_server: Final = _resolve_mcp_server_by_name_or_id(mcp_server_name, client_ip)
if mcp_server is None:
return dummy_return
return await register_client_with_server(

8
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -6147,13 +6147,13 @@ class MCPServerManager:
internal_networks = IPAddressUtils.parse_internal_networks(general_settings.get("mcp_internal_ip_ranges"))
return IPAddressUtils.is_internal_ip(client_ip, internal_networks)
def get_mcp_server_by_id(self, server_id: str) -> MCPServer | None:
"""
Get the MCP Server from the server id
"""
def get_mcp_server_by_id(self, server_id: str, client_ip: str | None = None) -> MCPServer | None:
"""Get the MCP Server from the server id."""
registry: Final = self.get_registry()
for server in registry.values():
if server.server_id == server_id:
if not self._is_server_accessible_from_ip(server, client_ip):
return None
return server
return None

2
litellm/proxy/_types.py
View file

@ -4239,6 +4239,8 @@ class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase):
access_group_id: str
access_group_name: str
models: tuple[str, ...]
mcp_server_ids: tuple[str, ...] = ()
agent_ids: tuple[str, ...] = ()
class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable):

53
litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
View file

@ -5,10 +5,13 @@ Handles agent permission checking for keys and teams using object_permission_id.
Follows the same pattern as MCP permission handling.
"""
import asyncio
from collections.abc import Awaitable, Callable, Sequence
from dataclasses import dataclass
from typing import Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts
from litellm.proxy._types import (
UI_TEAM_ID,
LiteLLM_ObjectPermissionTable,
@ -443,15 +446,47 @@ class AgentRequestHandler:
return []
async def accessible_agents(user_api_key_auth: UserAPIKeyAuth) -> tuple[AgentResponse, ...]:
"""Every registry agent for proxy admins, else the agents the key's and team's grants reach."""
def _granted_ids(access: AgentAccess) -> frozenset[str]:
match access:
case UnrestrictedAgentAccess():
return frozenset()
case RestrictedAgentAccess(agent_ids):
return agent_ids
ResolveAgentAccess: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[AgentAccess]]
EffectiveAuthContexts: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[Sequence[UserAPIKeyAuth]]]
async def _granted_agent_ids(
user_api_key_auth: UserAPIKeyAuth,
resolve_access: ResolveAgentAccess,
effective_contexts: EffectiveAuthContexts,
) -> frozenset[str]:
"""Union of the explicit grants reachable from the key, its team, or (for a dashboard session)
the user's real teams and user row. No grant anywhere yields the empty set, unlike the
open-by-default ``resolve_agent_access`` that guards direct access."""
accesses: Final = await asyncio.gather(
*(resolve_access(auth_context) for auth_context in await effective_contexts(user_api_key_auth))
)
return frozenset().union(*(_granted_ids(access) for access in accesses))
async def accessible_agents(
user_api_key_auth: UserAPIKeyAuth,
all_agents: tuple[AgentResponse, ...] | None = None,
resolve_access: ResolveAgentAccess | None = None,
effective_contexts: EffectiveAuthContexts = build_effective_auth_contexts,
) -> tuple[AgentResponse, ...]:
"""Every registry agent for proxy admins, else only the agents the caller was granted."""
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
all_agents: Final = global_agent_registry.get_agent_list()
agents: Final = global_agent_registry.get_agent_list() if all_agents is None else all_agents
if user_api_key_auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN.value):
return all_agents
match await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_auth):
case UnrestrictedAgentAccess():
return all_agents
case RestrictedAgentAccess(allowed_agent_ids):
return tuple(agent for agent in all_agents if agent.agent_id in allowed_agent_ids)
return agents
allowed_agent_ids: Final = await _granted_agent_ids(
user_api_key_auth,
AgentRequestHandler.resolve_agent_access if resolve_access is None else resolve_access,
effective_contexts,
)
return tuple(agent for agent in agents if agent.agent_id in allowed_agent_ids)

23
litellm/proxy/auth/litellm_license.py
View file

@ -16,6 +16,10 @@ if TYPE_CHECKING:
from litellm.proxy._types import EnterpriseLicenseData
AUTO_ROUTER_LICENSE_FEATURE: Final = "auto_router"
HEURISTIC_V2_LICENSE_REMEDY: Final = "A LiteLLM license with the 'auto_router' feature lifts the limit."
class LicenseCheck:
"""
- Check if license in env
@ -149,6 +153,19 @@ class LicenseCheck:
return False
return team_count > _max_teams_in_license
def heuristic_v2_router_limit(self) -> int | None:
"""
How many heuristic_v2 auto-routers this proxy may hold: unlimited (None) only when the
signed license lists the auto_router feature, otherwise one. A license verified through
the API carries no feature list, so it does not lift the limit either.
"""
if self.airgapped_license_data is None:
return 1
allowed_features: Final = self.airgapped_license_data.get("allowed_features")
if isinstance(allowed_features, list) and AUTO_ROUTER_LICENSE_FEATURE in allowed_features:
return None
return 1
def verify_license_without_api_request(self, public_key, license_key):
try:
from cryptography.hazmat.primitives import hashes
@ -179,19 +196,21 @@ class LicenseCheck:
# Decode and parse the data
license_data: Final = json.loads(message.decode())
self.airgapped_license_data = EnterpriseLicenseData(**license_data)
# debug information provided in license data
verbose_proxy_logger.debug("License data: %s", license_data)
# Check expiration date
expiration_date: Final = datetime.strptime(license_data["expiration_date"], "%Y-%m-%d")
if expiration_date < datetime.now():
self.airgapped_license_data = None
return False, "License has expired"
self.airgapped_license_data = EnterpriseLicenseData(**license_data)
return True
except Exception as e:
self.airgapped_license_data = None
verbose_proxy_logger.debug(
"litellm.proxy.auth.litellm_license.py::verify_license_without_api_request - Unable to verify License locally. - %s",
e,

71
litellm/proxy/common_request_processing.py
View file

@ -467,6 +467,42 @@ def _getattr_object(value: object, name: str, default: object = None) -> object:
return getattr(value, name, default)
_OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
{
status.HTTP_401_UNAUTHORIZED: "authentication_error",
status.HTTP_403_FORBIDDEN: "permission_error",
status.HTTP_429_TOO_MANY_REQUESTS: "rate_limit_error",
}
)
def _error_status_code(exc: object, default: int) -> int:
"""The HTTP status an exception carries, or ``default`` when it carries none."""
carried: Final = _getattr_object(exc, "status_code")
return carried if isinstance(carried, int) and not isinstance(carried, bool) else default
def _openai_error_type(exc: object, status_code: int) -> str:
"""OpenAI types ``error.type`` as a required string, so an exception carrying none
falls back to the type its status code stands for."""
carried: Final = _getattr_object(exc, "type")
if isinstance(carried, str):
return carried
mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code)
if mapped is not None:
return mapped
if status_code < status.HTTP_500_INTERNAL_SERVER_ERROR:
return "invalid_request_error"
return "internal_server_error"
def _openai_error_param(exc: object) -> str | None:
"""OpenAI types ``error.param`` as nullable, so an exception carrying none
serializes as JSON ``null``."""
carried: Final = _getattr_object(exc, "param")
return carried if isinstance(carried, str) else None
class _UpstreamHttpResponse(Protocol):
@property
def status_code(self) -> int: ...
@ -540,11 +576,12 @@ def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, s
message, structured_fields = serialize_http_exception_detail(raw_detail)
existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
error_status: Final = _error_status_code(exc, status.HTTP_400_BAD_REQUEST)
return ProxyException(
message=message,
type=getattr(exc, "type", "None"),
param=getattr(exc, "param", "None"),
code=getattr(exc, "status_code", status.HTTP_400_BAD_REQUEST),
type=_openai_error_type(exc, error_status),
param=_openai_error_param(exc),
code=error_status,
provider_specific_fields=merged_fields,
headers=headers,
)
@ -827,25 +864,22 @@ def sse_error_payload(exc: BaseException) -> tuple[int, Mapping[str, object]]:
are byte-identical.
"""
# Preserve status code from HTTPException (e.g. guardrail blocks)
error_status: Final = getattr(exc, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
error_status: Final = _error_status_code(exc, status.HTTP_500_INTERNAL_SERVER_ERROR)
raw_detail: Final = _getattr_object(exc, "detail", "Error processing stream start")
message, structured_fields = serialize_http_exception_detail(raw_detail)
existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
# Built in one statement then given its one optional key, rather than spread
# conditionally: the spread form costs two extra dict constructions, which
# type-discipline-budget.json's LIT002 ceiling has no room for.
error_obj: Final = {
"message": message,
"type": getattr(exc, "type", "None"),
"param": getattr(exc, "param", "None"),
"type": _openai_error_type(exc, error_status),
"param": _openai_error_param(exc),
"code": str(error_status),
}
if merged_fields:
error_obj["provider_specific_fields"] = merged_fields
return error_status, error_obj
if not merged_fields:
return error_status, error_obj
return error_status, {**error_obj, "provider_specific_fields": merged_fields}
def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
@ -922,7 +956,7 @@ async def create_response(
"error": {
"message": _CLIENT_DISCONNECT_DETAIL,
"type": "client_disconnect",
"param": "None",
"param": None,
"code": str(LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED),
}
},
@ -3417,8 +3451,8 @@ class ProxyBaseLLMRequestProcessing:
_code = status.HTTP_500_INTERNAL_SERVER_ERROR
raise ProxyException(
message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
type=_openai_error_type(e, _code),
param=_openai_error_param(e),
openai_code=getattr(e, "code", None),
code=_code,
provider_specific_fields=getattr(e, "provider_specific_fields", None),
@ -3628,11 +3662,12 @@ class ProxyBaseLLMRequestProcessing:
if isinstance(e, HTTPException):
raise e
stream_error_status: Final = _error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)
proxy_exception: Final = ProxyException(
message=redact_internal_details_from_client_message(getattr(e, "message", str(e))),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
type=_openai_error_type(e, stream_error_status),
param=_openai_error_param(e),
code=stream_error_status,
)
stream_completed = True
yield serialize_error(proxy_exception)

65
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -1,10 +1,12 @@
import json
import re
from collections.abc import Collection
from typing import Any, Final
from collections.abc import Collection, Mapping
from types import MappingProxyType, UnionType
from typing import Any, Final, Union, get_args, get_origin
import orjson
from fastapi import Request, UploadFile, status
from typing_extensions import ReadOnly
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
@ -40,6 +42,65 @@ def _is_json_content_type(content_type: str) -> bool:
return _normalize_media_type(content_type) == "application/json"
def _numeric_form_type(annotation: object) -> type[int] | type[float] | None:
"""The scalar to parse an ``int``/``float``-typed field as, else ``None``."""
unwrapped: Final = get_args(annotation)[0] if get_origin(annotation) is ReadOnly else annotation
candidates: Final = (
tuple(arg for arg in get_args(unwrapped) if arg is not type(None))
if get_origin(unwrapped) in (Union, UnionType)
else (unwrapped,)
)
if len(candidates) != 1:
return None
if candidates[0] is int:
return int
if candidates[0] is float:
return float
return None
def numeric_form_fields(annotations: Mapping[str, object]) -> Mapping[str, type[int] | type[float]]:
"""
The numeric fields of a request schema, mapped to the scalar to parse them as.
Only a bare ``int``/``float`` or an optional one qualifies, so container and
literal fields are left alone and ``bool`` is excluded on purpose.
"""
return MappingProxyType(
{
name: scalar
for name, annotation in annotations.items()
if (scalar := _numeric_form_type(annotation)) is not None
}
)
def _numeric_form_value(value: object, scalar: type[int] | type[float]) -> object:
if not isinstance(value, str):
return value
try:
return scalar(value)
except ValueError:
return value
def coerce_numeric_form_fields(
parsed_body: Mapping[str, object],
numeric_fields: Mapping[str, type[int] | type[float]],
) -> Mapping[str, object]:
"""
Parse the numeric fields of a form-encoded body back into numbers.
``request.form()`` yields every field as a string, so a provider that puts the
value in a JSON body would send a string where its API requires a number. A
value that will not parse is left as-is for the provider to reject as before.
"""
return {
name: _numeric_form_value(value, numeric_fields[name]) if name in numeric_fields else value
for name, value in parsed_body.items()
}
async def _read_request_body(request: Request | None) -> dict:
"""
Safely read the request body and parse it as JSON.

87
litellm/proxy/container_endpoints/endpoints.py
View file

@ -15,10 +15,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
)
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
from litellm.proxy.container_endpoints.ownership import (
assert_user_can_access_container,
filter_container_list_response,
get_container_forwarding_params,
list_owned_containers,
record_container_owner,
)
@ -173,6 +174,9 @@ async def list_containers(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
after: str | None = None,
limit: int | None = None,
order: str | None = None,
):
"""
Container list endpoint for retrieving a list of containers.
@ -206,55 +210,54 @@ async def list_containers(
version,
)
# Read query parameters
query_params: Final = dict(request.query_params)
data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")}
# Extract custom_llm_provider using priority chain
custom_llm_provider: Final = (
get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
data: Final[dict[str, Any]] = {
"query_params": query_params,
"model": query_params.get("model"),
"order": order,
"custom_llm_provider": custom_llm_provider,
}
# Add custom_llm_provider to data
data["custom_llm_provider"] = custom_llm_provider
async def fetch_page(page_after: str | None, page_limit: int | None) -> object:
processor: Final = ProxyBaseLLMRequestProcessing(data={**data, "after": page_after, "limit": page_limit})
try:
return await processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type="alist_containers",
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=select_data_generator,
model=None,
user_model=user_model,
user_temperature=user_temperature,
user_request_timeout=user_request_timeout,
user_max_tokens=user_max_tokens,
user_api_base=user_api_base,
version=version,
)
except Exception as e:
raise await processor._handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
# Process request using ProxyBaseLLMRequestProcessing
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
response: Final = await processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type="alist_containers",
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=select_data_generator,
model=None,
user_model=user_model,
user_temperature=user_temperature,
user_request_timeout=user_request_timeout,
user_max_tokens=user_max_tokens,
user_api_base=user_api_base,
version=version,
)
except Exception as e:
raise await processor._handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
# Ownership filtering runs OUTSIDE the LLM-exception scope: a DB error
# in the ownership lookup is not an LLM-API error and shouldn't be
# translated to a provider-shaped failure (which would also fire the
# post_call_failure_hook for what is in fact a successful upstream call).
return await filter_container_list_response(
response=response,
if is_proxy_admin(user_api_key_dict):
return await fetch_page(after, limit)
return await list_owned_containers(
fetch_page=fetch_page,
after=after,
limit=limit,
user_api_key_dict=user_api_key_dict,
custom_llm_provider=custom_llm_provider,
)

20
litellm/proxy/container_endpoints/handler_factory.py
View file

@ -6,7 +6,9 @@ FastAPI route handlers for ALL container file endpoints.
"""
import json
from collections.abc import Mapping, Sequence
from pathlib import Path
from types import MappingProxyType
from typing import Any, Final
from fastapi import APIRouter, Depends, Request, Response
@ -56,6 +58,7 @@ def _create_handler_for_path_params(
route_type: str,
returns_binary: bool = False,
is_multipart: bool = False,
query_param_names: Sequence[str] = (),
):
"""
Dynamically create a handler with the correct path parameter signature.
@ -114,6 +117,7 @@ def _create_handler_for_path_params(
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={"container_id": container_id},
query_param_names=query_param_names,
)
return handler_container_id
@ -133,6 +137,7 @@ def _create_handler_for_path_params(
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={"container_id": container_id, "file_id": file_id},
query_param_names=query_param_names,
)
return handler_container_file
@ -150,6 +155,7 @@ def _create_handler_for_path_params(
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={},
query_param_names=query_param_names,
)
return handler_no_params
@ -351,12 +357,17 @@ async def _process_multipart_upload_request(
)
def _declared_query_params(query_params: Mapping[str, str], query_param_names: Sequence[str]) -> Mapping[str, str]:
return MappingProxyType({name: query_params[name] for name in query_param_names if name in query_params})
async def _process_request(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth,
route_type: str,
path_params: dict[str, str],
query_param_names: Sequence[str] = (),
):
"""Common request processing logic."""
from litellm.proxy.proxy_server import (
@ -376,6 +387,7 @@ async def _process_request(
query_params: Final = dict(request.query_params)
data: Final[dict[str, Any]] = {
"query_params": query_params,
**_declared_query_params(query_params, query_param_names),
**path_params,
}
@ -452,7 +464,13 @@ def register_container_file_endpoints(router: APIRouter) -> None:
is_multipart = endpoint_config.get("is_multipart", False)
# Create handler with correct signature for path params
handler = _create_handler_for_path_params(path_params, route_type, returns_binary, is_multipart)
handler = _create_handler_for_path_params(
path_params,
route_type,
returns_binary,
is_multipart,
query_param_names=endpoint_config.get("query_params", ()),
)
# Register routes
route_method = getattr(router, method)

134
litellm/proxy/container_endpoints/ownership.py
View file

@ -1,9 +1,10 @@
import json
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, TypeAlias
from fastapi import HTTPException
from pydantic import BaseModel
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
@ -46,6 +47,12 @@ _CONTAINER_STORED_ID_CACHE: Final = InMemoryCache(max_size_in_memory=10000, defa
# different users with different scopes get disjoint cache entries.
_ALLOWED_CONTAINER_IDS_CACHE: Final = InMemoryCache(max_size_in_memory=2048, default_ttl=60)
DEFAULT_CONTAINER_LIST_LIMIT: Final = 20
OWNED_CONTAINER_LIST_PAGE_SIZE: Final = 100
OWNED_CONTAINER_LIST_MAX_PAGES: Final = 5
FetchContainerListPage: TypeAlias = Callable[[str | None, int | None], Awaitable[object]]
def _allowed_container_ids_cache_key(owner_scopes: Sequence[str]) -> str:
"""JSON-encode the sorted scope list — using a separator like ``|``
@ -337,27 +344,23 @@ def _get_container_list_data(response: object) -> Sequence[object] | None:
return data if isinstance(data, list) else None
def _set_container_list_data(response: Any, data: list[object], removed_filtered_items: bool = False) -> object:
def _get_has_more(response: object) -> bool:
if isinstance(response, dict):
response["data"] = data
if data:
response["first_id"] = _get_response_id(data[0])
response["last_id"] = _get_response_id(data[-1])
else:
response["first_id"] = None
response["last_id"] = None
response["has_more"] = False
if removed_filtered_items:
response["has_more"] = False
return response
return response.get("has_more") is True
return getattr(response, "has_more", None) is True
response.data = data
response.first_id = _get_response_id(data[0]) if data else None
response.last_id = _get_response_id(data[-1]) if data else None
if not data and hasattr(response, "has_more"):
response.has_more = False
if removed_filtered_items and hasattr(response, "has_more"):
response.has_more = False
def _with_container_list_page(response: object, data: Sequence[object], has_more: bool) -> object:
page: Final = {
"data": list(data),
"first_id": _get_response_id(data[0]) if data else None,
"last_id": _get_response_id(data[-1]) if data else None,
"has_more": has_more,
}
if isinstance(response, dict):
return {**response, **page}
if isinstance(response, BaseModel):
return response.model_copy(update=page)
return response
@ -366,16 +369,16 @@ async def _get_allowed_container_ids(
) -> AbstractSet[str]:
owner_scopes: Final = get_resource_owner_scopes(user_api_key_dict)
if not owner_scopes:
return set()
return frozenset()
cache_key: Final = _allowed_container_ids_cache_key(owner_scopes)
cached: Final = _ALLOWED_CONTAINER_IDS_CACHE.get_cache(cache_key)
if cached is not None:
return set(cached)
return frozenset(cached)
prisma_client: Final = await _get_prisma_client()
if prisma_client is None:
return set()
return frozenset()
table: Final = ManagedObjectRepository(prisma_client).table
rows: Final[Sequence[prisma_models.LiteLLM_ManagedObjectTable]] = await table.find_many(
@ -384,34 +387,69 @@ async def _get_allowed_container_ids(
"created_by": {"in": owner_scopes},
}
)
allowed_ids: Final = {row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None}
# ``InMemoryCache.get_cache`` attempts ``json.loads`` on the stored
# value; passing a set would round-trip through that path
# unnecessarily. Store as a list and rehydrate above.
_ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, list(allowed_ids))
allowed_ids: Final = frozenset(
row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None
)
_ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, tuple(allowed_ids))
return allowed_ids
async def filter_container_list_response(
response: object,
def _is_owned_container(item: object, allowed_container_ids: AbstractSet[str], custom_llm_provider: str) -> bool:
container_id: Final = _get_response_id(item)
if container_id is None:
return False
original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider)
return _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids
async def _collect_owned_containers(
fetch_page: FetchContainerListPage,
after: str | None,
needed: int,
allowed_container_ids: AbstractSet[str],
custom_llm_provider: str,
pages_left: int,
collected: tuple[object, ...],
) -> tuple[object, tuple[object, ...]]:
page: Final = await fetch_page(after, OWNED_CONTAINER_LIST_PAGE_SIZE)
page_data: Final = _get_container_list_data(page) or ()
owned: Final = collected + tuple(
item for item in page_data if _is_owned_container(item, allowed_container_ids, custom_llm_provider)
)
upstream_last_id: Final = _get_response_id(page_data[-1]) if page_data else None
if len(owned) >= needed or upstream_last_id is None or pages_left <= 1 or not _get_has_more(page):
return page, owned
return await _collect_owned_containers(
fetch_page=fetch_page,
after=upstream_last_id,
needed=needed,
allowed_container_ids=allowed_container_ids,
custom_llm_provider=custom_llm_provider,
pages_left=pages_left - 1,
collected=owned,
)
async def list_owned_containers(
fetch_page: FetchContainerListPage,
after: str | None,
limit: int | None,
user_api_key_dict: UserAPIKeyAuth,
custom_llm_provider: str,
) -> object:
if is_proxy_admin(user_api_key_dict):
return response
data: Final = _get_container_list_data(response)
if data is None:
return response
allowed_container_ids: Final = await _get_allowed_container_ids(user_api_key_dict)
filtered: Final[list[object]] = []
for item in data:
container_id = _get_response_id(item)
if container_id is None:
continue
original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider)
if _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids:
filtered.append(item)
return _set_container_list_data(response, filtered, removed_filtered_items=len(filtered) != len(data))
page_limit: Final = limit if limit is not None else DEFAULT_CONTAINER_LIST_LIMIT
last_page, owned = await _collect_owned_containers(
fetch_page=fetch_page,
after=after,
needed=page_limit + 1,
allowed_container_ids=allowed_container_ids,
custom_llm_provider=custom_llm_provider,
pages_left=OWNED_CONTAINER_LIST_MAX_PAGES,
collected=(),
)
return _with_container_list_page(
last_page,
owned[:page_limit],
has_more=len(owned) > page_limit or _get_has_more(last_page),
)

53
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -41,7 +41,7 @@ from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicM
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -56,7 +56,6 @@ from litellm.proxy.guardrails.anthropic_sse import (
is_raw_sse_stream,
model_response_text,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import (
BedrockChecksConfigModel,
BedrockGuardrailStreamingParams,
@ -713,9 +712,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# logic becomes shared across providers.
#### CALL HOOKS - proxy only ####
def _load_credentials(
self,
):
def _load_credentials(self, bearer_token: str | None = None):
try:
from botocore.credentials import Credentials
except ImportError:
@ -737,17 +734,21 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
aws_region_name=aws_region_name,
)
credentials: Final[Credentials] = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
credentials: Final[Credentials | None] = (
None
if bearer_token is not None
else self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=aws_region_name,
aws_session_name=aws_session_name,
aws_profile_name=aws_profile_name,
aws_role_name=aws_role_name,
aws_web_identity_token=aws_web_identity_token,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
)
return credentials, aws_region_name
@ -779,13 +780,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
proxy_endpoint_url = f"{proxy_endpoint_url}{request_path}"
encoded_data: Final = json.dumps(data).encode("utf-8")
# first check api-key, if none, fall back to sigV4
if api_key is not None:
aws_bearer_token: str | None = api_key
else:
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
aws_bearer_token: Final = bedrock_bearer_token(api_key)
if aws_bearer_token:
if aws_bearer_token is not None:
try:
from botocore.awsrequest import AWSRequest
except ImportError:
@ -916,7 +913,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
source,
)
return BedrockGuardrailResponse()
credentials, aws_region_name = self._load_credentials()
credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key))
allow_chunking: Final = not self._content_uses_contextual_grounding(content)
completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = [] # mutable-ok: billed-chunk usage accumulator
@ -958,7 +955,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
self,
content: Sequence[BedrockContentItem],
base_request_data: Mapping[str, object],
credentials: "Credentials",
credentials: "Credentials | None",
aws_region_name: str,
api_key: str | None,
request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper
@ -1096,7 +1093,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
self,
content: Sequence[BedrockContentItem],
base_request_data: Mapping[str, object],
credentials: "Credentials",
credentials: "Credentials | None",
aws_region_name: str,
api_key: str | None,
request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper
@ -1146,7 +1143,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
self,
content: Sequence[BedrockContentItem],
base_request_data: Mapping[str, object],
credentials: "Credentials",
credentials: "Credentials | None",
aws_region_name: str,
api_key: str | None,
request_data: dict | None, # mutable-ok: proxy request body dict, mutated by the logging helper
@ -1873,9 +1870,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# Nothing to scan (e.g. tool-only turn) -> allow, like ApplyGuardrail does.
return BedrockGuardrailResponse()
credentials, aws_region_name = self._load_credentials()
body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
api_key: Final[str | None] = request_data.get("api_key") if request_data else None
credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key))
body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
prepared_request: Final = self._prepare_request(
credentials=credentials,

7
litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py
View file

@ -916,9 +916,10 @@ class CompresrGuardrail(CustomGuardrail):
def _mirror_texts_channel(input_texts: object, applied: _CompressionResult) -> list[object] | None:
"""Compressed content mirrored into the Responses `texts` channel.
The chat/Anthropic handlers round-trip ``structured_messages``; the
Responses translation cannot rebuild its input from chat messages and
instead writes back through ``texts``. This matches by value, so a
The chat/Anthropic/Responses handlers round-trip
``structured_messages``; translations without that round-trip write
back through ``texts``, so the compressed content is mirrored there
too. This matches by value, so a
replacement is applied only when it is unambiguous: one compression per
text, and every occurrence in ``texts`` accounted for by a compressed
target. Anything else is left uncompressed rather than risk a wrong or

11
litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
View file

@ -50,6 +50,9 @@ if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
BYPASS_HEADER: Final = "x-headroom-bypass"
_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
(CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
)
HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve"
_HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})")
_HASH_CACHE_TTL_SECONDS: Final = 15 * 60
@ -725,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER)
return inputs
if request_data.get("background"):
verbose_proxy_logger.debug("Headroom: background request; skipping compression")
return inputs
structured_messages: Final = inputs.get("structured_messages")
if not _is_object_list(structured_messages) or not structured_messages:
return inputs
@ -826,9 +833,9 @@ class HeadroomGuardrail(CustomGuardrail):
) -> dict[str, Any] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict
base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type)
effective: Final = base_result if base_result is not None else kwargs
if call_type not in (CallTypes.completion, CallTypes.acompletion):
if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES:
return base_result
if not effective.get("stream"):
if not effective.get("stream") or effective.get("background"):
return base_result
if not has_headroom_retrieve_tool(effective.get("tools")):
return base_result

54
litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
View file

@ -121,9 +121,6 @@ def _a2a_jsonrpc_error_chunk(exc: HTTPException, request_id: str | None) -> Mapp
}
endpoint_guardrail_translation_mappings = None
def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> None:
"""Populate data['litellm_metadata'] from user_api_key_dict if absent."""
if "litellm_metadata" not in data:
@ -164,7 +161,6 @@ class UnifiedLLMGuardrails(CustomLogger):
Use this if you want to MODIFY the input
"""
global endpoint_guardrail_translation_mappings
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
@ -186,18 +182,15 @@ class UnifiedLLMGuardrails(CustomLogger):
)
return data
if endpoint_guardrail_translation_mappings is None:
endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
mappings: Final = load_guardrail_translation_mappings()
try:
if CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
if CallTypes(call_type) not in mappings:
return data
except ValueError:
return data # handle unmapped call types
endpoint_translation: Final = _as_endpoint_translation(
endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
)
endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
_ensure_litellm_metadata(data, user_api_key_dict)
@ -222,8 +215,6 @@ class UnifiedLLMGuardrails(CustomLogger):
This can NOT modify the input, only used to reject or accept a call before going to LLM API
"""
global endpoint_guardrail_translation_mappings
verbose_proxy_logger.debug("Running UnifiedLLMGuardrails moderation hook")
guardrail_to_apply: Final[CustomGuardrail] = data.pop("guardrail_to_apply", None)
@ -241,14 +232,11 @@ class UnifiedLLMGuardrails(CustomLogger):
)
return data
if endpoint_guardrail_translation_mappings is None:
endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
if call_type is not None and CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
mappings: Final = load_guardrail_translation_mappings()
if call_type is not None and CallTypes(call_type) not in mappings:
return data
endpoint_translation: Final = _as_endpoint_translation(
endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
)
endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
_ensure_litellm_metadata(data, user_api_key_dict)
@ -271,7 +259,6 @@ class UnifiedLLMGuardrails(CustomLogger):
Uses Enkrypt AI guardrails to check the response for policy violations, PII, and injection attacks
"""
global endpoint_guardrail_translation_mappings
# Local import avoids a module-level cyclic import with
# litellm.integrations.custom_guardrail.
from litellm.integrations.custom_guardrail import ModifyResponseException
@ -319,10 +306,9 @@ class UnifiedLLMGuardrails(CustomLogger):
)
return response
if endpoint_guardrail_translation_mappings is None:
endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
mappings: Final = load_guardrail_translation_mappings()
if CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
if CallTypes(call_type) not in mappings:
verbose_proxy_logger.warning(
"Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; "
"skipping post-call scanning.",
@ -332,9 +318,7 @@ class UnifiedLLMGuardrails(CustomLogger):
)
return response
endpoint_translation: Final = _as_endpoint_translation(
endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
)
endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]())
try:
response = await endpoint_translation.process_output_response(
@ -906,8 +890,6 @@ class UnifiedLLMGuardrails(CustomLogger):
sampling_rate=1 means every chunk, sampling_rate=5 means every 5th chunk, etc.
"""
global endpoint_guardrail_translation_mappings
# Local import avoids a module-level cyclic import with
# litellm.integrations.custom_guardrail.
from litellm.integrations.custom_guardrail import ModifyResponseException
@ -978,9 +960,7 @@ class UnifiedLLMGuardrails(CustomLogger):
yield item
return
# Initialize translation mappings if needed
if endpoint_guardrail_translation_mappings is None:
endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
mappings: Final = load_guardrail_translation_mappings()
# Streaming text transformation (incremental_diff) diverges enough from the
# block_only path that it runs as its own iterator. It requires a route we
@ -989,7 +969,7 @@ class UnifiedLLMGuardrails(CustomLogger):
if streaming_transform_mode == "incremental_diff":
transform_call_type: Final = self._resolve_transform_call_type(
user_api_key_dict=user_api_key_dict,
mappings=endpoint_guardrail_translation_mappings,
mappings=mappings,
)
if transform_call_type is not None:
async for transformed_item in self._run_incremental_transform_stream(
@ -1000,7 +980,7 @@ class UnifiedLLMGuardrails(CustomLogger):
call_type=transform_call_type,
sampling_rate=sampling_rate,
end_of_stream_only=end_of_stream_only,
mappings=endpoint_guardrail_translation_mappings,
mappings=mappings,
):
yield transformed_item
return
@ -1037,7 +1017,7 @@ class UnifiedLLMGuardrails(CustomLogger):
call_type = _infer_call_type(call_type=None, completion_response=item)
# If call type not supported, just pass through all chunks
if call_type is None or CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
if call_type is None or CallTypes(call_type) not in mappings:
yield item
async for remaining_item in response:
yield remaining_item
@ -1049,7 +1029,7 @@ class UnifiedLLMGuardrails(CustomLogger):
# moderation runs below.
if end_of_stream_only:
if not buffer_until_moderated:
endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
endpoint_translation = mappings[CallTypes(call_type)]()
stream_has_ended = hasattr(
endpoint_translation, "_check_streaming_has_ended"
) and endpoint_translation._check_streaming_has_ended(responses_so_far)
@ -1063,7 +1043,7 @@ class UnifiedLLMGuardrails(CustomLogger):
# Process chunk based on sampling rate
if chunk_counter % sampling_rate == 0:
endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
endpoint_translation = mappings[CallTypes(call_type)]()
scan_key = endpoint_translation.get_streaming_scan_key(responses_so_far)
if _is_redundant_scan(scan_key, last_scan_key):
verbose_proxy_logger.debug(
@ -1143,14 +1123,14 @@ class UnifiedLLMGuardrails(CustomLogger):
yield item
# Stream has ended - do final processing with all collected chunks
if call_type is not None and CallTypes(call_type) in endpoint_guardrail_translation_mappings:
if call_type is not None and CallTypes(call_type) in mappings:
verbose_proxy_logger.debug(
"Processing final streaming response with all %s chunks for guardrail %s",
len(responses_so_far),
guardrail_to_apply.guardrail_name,
)
endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]()
endpoint_translation = mappings[CallTypes(call_type)]()
# When buffering, snapshot the original chunks before moderation.
# A shallow copy suffices: end-of-stream

23
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -654,23 +654,16 @@ class InMemoryGuardrailHandler:
source: Literal["db", "config"] = "db",
) -> None:
"""
Update a guardrail in memory
- updates the guardrail in memory
- updates the guardrail params in litellm.callback_manager
Update a guardrail in memory: a changed name or litellm_params rebuilds the
live callback from the new row (fail-closed: an invalid row keeps the
previous instance and raises), anything else only refreshes the stored row
"""
self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail
self._sources[guardrail_id] = source
tracked_callbacks: Final = self._tracked_callbacks(guardrail_id)
if not tracked_callbacks:
updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id})
if self._has_guardrail_params_changed(guardrail_id, updated_guardrail):
self.reinitialize_guardrail(guardrail=updated_guardrail, source=source)
return
updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {}))
tracked_callbacks[0].update_in_memory_litellm_params(litellm_params=updated_litellm_params)
for sibling_callback in tracked_callbacks[1:]:
sibling_stage = sibling_callback.event_hook
sibling_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params)
sibling_callback.event_hook = sibling_stage
self.IN_MEMORY_GUARDRAILS[guardrail_id] = updated_guardrail
self._sources[guardrail_id] = source
def delete_in_memory_guardrail(self, guardrail_id: str) -> None:
"""

94
litellm/proxy/hooks/responses_id_security.py
View file

@ -5,10 +5,11 @@ This hook uses the DBSpendUpdateWriter to batch-write response IDs to the databa
instead of writing immediately on each request.
"""
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Callable, Mapping
from typing import TYPE_CHECKING, Any, Final, cast
from fastapi import HTTPException
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
@ -32,6 +33,44 @@ _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
_RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _response_payload(response_obj: object) -> Mapping[str, object] | None:
try:
return _RESPONSE_PAYLOAD_ADAPTER.validate_python(response_obj)
except ValidationError:
return None
def _rewrite_advertised_id(
event: BaseLiteLLMOpenAIResponseObject,
rewrite: Callable[[str], str],
) -> BaseLiteLLMOpenAIResponseObject:
event_id: Final = getattr(event, "id", None)
if isinstance(event_id, str) and event_id.startswith("resp_"):
setattr(event, "id", rewrite(event_id))
return event
nested: Final = getattr(event, "response", None)
if isinstance(nested, ResponsesAPIResponse):
setattr(nested, "id", rewrite(nested.id))
setattr(event, "response", nested)
return event
payload: Final = _response_payload(nested)
if payload is None:
return event
payload_id: Final = payload.get("id")
if not isinstance(payload_id, str):
return event
rewritten: Final = {**payload, "id": rewrite(payload_id)} # mutable-ok: pydantic cannot serialize a frozen map
setattr(event, "response", rewritten)
return event
def _is_responses_api_create_route(request_route: str | None) -> bool:
if request_route is None:
return False
@ -196,10 +235,6 @@ class ResponsesIDSecurity(CustomLogger):
user_api_key_dict: "UserAPIKeyAuth",
request_cache: dict[str, str] | None = None,
) -> BaseLiteLLMOpenAIResponseObject:
# encrypt the response id using the symmetric key
# encrypt the response id, and encode the user id and response id in base64
# Check if signing key is available
signing_key: Final = self._get_signing_key()
if signing_key is None:
verbose_proxy_logger.debug(
@ -210,43 +245,22 @@ class ResponsesIDSecurity(CustomLogger):
)
return response
response_id: Final = getattr(response, "id", None)
response_obj: Final = getattr(response, "response", None)
def encrypt(original_id: str) -> str:
cached: Final = request_cache.get(original_id) if request_cache is not None else None
if cached is not None:
return cached
if response_id and isinstance(response_id, str) and response_id.startswith("resp_"):
# Check request-scoped cache first (for streaming consistency)
if request_cache is not None and response_id in request_cache:
setattr(response, "id", request_cache[response_id])
else:
encrypted_response_id = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
response_id,
user_api_key_dict.user_id or "",
user_api_key_dict.team_id or "",
)
managed_id: Final = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
original_id,
user_api_key_dict.user_id or "",
user_api_key_dict.team_id or "",
)
encrypted_id: Final = f"resp_{encrypt_value_helper(value=managed_id)}"
if request_cache is not None:
request_cache[original_id] = encrypted_id
return encrypted_id
encoded_user_id_and_response_id = encrypt_value_helper(value=encrypted_response_id)
encrypted_id = f"resp_{encoded_user_id_and_response_id}"
if request_cache is not None:
request_cache[response_id] = encrypted_id
setattr(response, "id", encrypted_id)
elif response_obj and isinstance(response_obj, ResponsesAPIResponse):
# Check request-scoped cache first (for streaming consistency)
if request_cache is not None and response_obj.id in request_cache:
setattr(response_obj, "id", request_cache[response_obj.id])
else:
encrypted_response_id = SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format(
response_obj.id,
user_api_key_dict.user_id or "",
user_api_key_dict.team_id or "",
)
encoded_user_id_and_response_id = encrypt_value_helper(value=encrypted_response_id)
encrypted_id = f"resp_{encoded_user_id_and_response_id}"
if request_cache is not None:
request_cache[response_obj.id] = encrypted_id
setattr(response_obj, "id", encrypted_id)
setattr(response, "response", response_obj)
return response
return _rewrite_advertised_id(response, encrypt)
async def async_post_call_success_hook(
self,

16
litellm/proxy/image_endpoints/endpoints.py
View file

@ -2,7 +2,7 @@ import asyncio
import io
import traceback
from collections.abc import Sequence
from typing import Final
from typing import Final, get_type_hints
import orjson
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
@ -16,11 +16,18 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.http_parsing_utils import (
coerce_numeric_form_fields,
numeric_form_fields,
)
from litellm.proxy.route_llm_request import route_request
from litellm.types.images.main import ImageEditRequestParams
from litellm.types.llms.openai import ChatCompletionUserMessage
router: Final = APIRouter()
IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams))
async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO:
"""
@ -279,7 +286,12 @@ async def image_edit_api(
#########################################################
# Read request body and convert UploadFiles to BytesIO
#########################################################
data: Final = await _read_request_body(request=request)
data: Final = dict(
coerce_numeric_form_fields(
parsed_body=await _read_request_body(request=request),
numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS,
)
)
image_files: Final = await batch_to_bytesio(image)
mask_files: Final = await batch_to_bytesio(mask)
if image_files:

34
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -17,6 +17,7 @@ import json
import traceback
from collections.abc import Awaitable, Mapping, Sequence
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Any, Final, Literal, Protocol, cast, overload
import fastapi
@ -77,6 +78,7 @@ from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
BulkUpdateUserRequest,
BulkUpdateUserResponse,
UserListResponse,
UserSearchWhere,
UserUpdateResult,
)
from litellm.types.proxy.management_endpoints.scim_v2 import (
@ -2080,6 +2082,22 @@ async def _authorize_user_list_request(
return ",".join(allowed_org_ids)
_NO_SEARCH_WHERE: Final[Mapping[str, object]] = MappingProxyType({})
def _user_search_where(search: str | None) -> Mapping[str, object]:
"""Prisma predicate for `/user/list?search=`: user_id or user_email contains it, case-insensitive."""
if not search:
return _NO_SEARCH_WHERE
search_where: Final[UserSearchWhere] = {
"OR": (
{"user_id": {"contains": search, "mode": "insensitive"}},
{"user_email": {"contains": search, "mode": "insensitive"}},
)
}
return search_where
@router.get(
"/user/list",
tags=["Internal User management"],
@ -2091,6 +2109,10 @@ async def get_users(
user_ids: str | None = fastapi.Query(default=None, description="Get list of users by user_ids"),
sso_user_ids: str | None = fastapi.Query(default=None, description="Get list of users by sso_user_id"),
user_email: str | None = fastapi.Query(default=None, description="Filter users by partial email match"),
search: str | None = fastapi.Query(
default=None,
description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).",
),
team: str | None = fastapi.Query(default=None, description="Filter users by team id"),
page: int = fastapi.Query(default=1, ge=1, description="Page number"),
page_size: int = fastapi.Query(default=25, ge=1, le=100, description="Number of items per page"),
@ -2121,6 +2143,8 @@ async def get_users(
Get list of users by sso_ids. Comma separated list of sso_ids.
user_email: Optional[str]
Filter users by partial email match
search: Optional[str]
Combined search: matches users whose user_id or user_email contains the value (case-insensitive)
team: Optional[str]
Filter users by team id. Will match if user has this team in their teams array.
page: int
@ -2197,7 +2221,11 @@ async def get_users(
where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_id_list}}}
## Filter any none fastapi.Query params - e.g. where_conditions: {'user_email': {'contains': Query(None), 'mode': 'insensitive'}, 'teams': {'has': Query(None)}}
where_conditions = {k: v for k, v in where_conditions.items() if v is not None}
where: Final[Mapping[str, object]] = {
key: value
for key, value in (*where_conditions.items(), *_user_search_where(search).items())
if value is not None
}
# Build order_by conditions
@ -2206,14 +2234,14 @@ async def get_users(
)
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
where=where_conditions,
where=where,
skip=skip,
take=page_size,
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
)
# Get total count of user rows
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where_conditions)
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)
# Get key count for each user
user_key_counts: Final = await get_user_key_counts(prisma_client, [user.user_id for user in users])

198
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -13,10 +13,11 @@ model/{model_id}/update - PATCH endpoint for model update.
import asyncio
import datetime
import json
from collections.abc import Awaitable, Mapping, Sequence
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from json import JSONDecodeError
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
@ -49,6 +50,7 @@ from litellm.proxy._types import (
TeamModelDeleteRequest,
UserAPIKeyAuth,
)
from litellm.proxy.auth.litellm_license import HEURISTIC_V2_LICENSE_REMEDY
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
@ -96,6 +98,9 @@ from litellm.router_strategy.complexity_router import (
from litellm.router_utils.auto_router_model_naming import (
STRATEGY_ROUTER_PARAM_FIELDS,
carries_complexity_router_settings,
count_heuristic_v2_routers,
heuristic_v2_limit_violation,
uses_heuristic_v2_classifier,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
@ -153,6 +158,8 @@ class _ProxyModelTable(Protocol):
def find_many(self, *, where: Mapping[str, object]) -> Awaitable[Sequence[_ProxyModelRow]]: ...
def create(self, *, data: Mapping[str, object]) -> Awaitable[_ProxyModelRow]: ...
def update(
self, *, where: Mapping[str, object], data: Mapping[str, object]
) -> Awaitable[_ProxyModelRow | None]: ...
@ -166,6 +173,9 @@ class _TxModelTables(Protocol):
litellm_proxymodeltable: _ProxyModelTable
_RowT = TypeVar("_RowT")
class _ExistingModelRow(Protocol):
@property
def litellm_params(self) -> Mapping[str, object]: ...
@ -269,6 +279,66 @@ def _raise_on_strategy_router_write_violation(
)
HEURISTIC_V2_SLOT_LOCK_KEY: Final = 5_872_301
_HEURISTIC_V2_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
_HEURISTIC_V2_DB_ROWS_SQL: Final = """
SELECT count(*)::int AS held FROM "LiteLLM_ProxyModelTable"
WHERE model_id <> $1
AND (CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)
-> 'complexity_router_config' ->> 'classifier_type' = 'heuristic_v2'
"""
def _effective_complexity_router_config(
incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
) -> object:
"""The complexity config a write leaves on the row: the incoming one when the write carries it, else the stored one."""
incoming: Final = None if incoming_params is None else incoming_params.complexity_router_config
if incoming is not None or existing_params is None:
return incoming
return existing_params.complexity_router_config
@asynccontextmanager
async def _heuristic_v2_slot(
prisma_client: PrismaClient, *, effective_config: object, model_id: str | None
) -> AsyncGenerator[_ProxyModelTable, None]:
"""Hand out the model table to write through while the row's claim on a heuristic_v2 slot is settled.
A write that leaves the row on classifier_type heuristic_v2 under a limited license runs
inside one transaction that takes an advisory lock in its own statement before counting
(a statement's snapshot predates anything it locks), so pods cannot both pass the count:
the DB rows (any pod, either JSON shape) plus this proxy's config.yaml routers are judged
against the license limit and the write is refused with a 403 before it happens. The row
being edited keeps its own slot through ``model_id``. Every other write, and every write on
an unlimited license, goes through the repository table with no lock. Only the row write
itself may run inside: anything that needs a second connection (the team model bookkeeping)
must wait until the transaction has committed and the lock is released. The transaction
writes bypass the repository's publish-on-write, so the config change is published once
after commit, the way delete_team_models does.
"""
from litellm.proxy.proxy_server import _license_check, llm_router
limit: Final = _license_check.heuristic_v2_router_limit()
if limit is None or not uses_heuristic_v2_classifier(effective_config):
yield _proxy_model_table(prisma_client)
return
async with prisma_client.db.tx() as tx_ctx:
tables: Final[_TxModelTables] = tx_ctx
await tx_ctx.query_raw(_HEURISTIC_V2_LOCK_SQL, HEURISTIC_V2_SLOT_LOCK_KEY)
rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(_HEURISTIC_V2_DB_ROWS_SQL, model_id or "")
db_held: Final = rows[0].get("held") if rows else 0
config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
held: Final = (db_held if isinstance(db_held, int) else 0) + count_heuristic_v2_routers(config_rows)
violation: Final = heuristic_v2_limit_violation(held=held + 1, limit=limit)
if violation is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {HEURISTIC_V2_LICENSE_REMEDY}"
)
yield tables.litellm_proxymodeltable
await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"
_REQUIRED_RATE_LIMIT_FIELDS: Final = ("rpm", "tpm")
@ -720,22 +790,29 @@ async def patch_model(
)
requested_model_name: Final = patch_data.model_name
stored_model_name: str | None = None
async def write_row(update_data: PrismaCompatibleUpdateDBModel) -> _ProxyModelRow | None:
nonlocal stored_model_name
stored_model_name = update_data.get("model_name")
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
update_data["updated_at"] = cast(str, get_utc_datetime())
async with _heuristic_v2_slot(
prisma_client,
effective_config=_effective_complexity_router_config(
patch_data.litellm_params, db_model.litellm_params
),
model_id=model_id,
) as table:
return await table.update(where={"model_id": model_id}, data=update_data)
# Handle team model updates with proper alias management
update_data: Final = await _update_team_model_in_db(
updated_model: Final = await _update_team_model_in_db(
db_model=db_model,
patch_data=patch_data,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
# Add metadata about update
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
update_data["updated_at"] = cast(str, get_utc_datetime())
# Perform partial update
updated_model: Final = await _proxy_model_table(prisma_client).update(
where={"model_id": model_id},
data=update_data,
write_row=write_row,
)
if updated_model is None:
@ -746,7 +823,6 @@ async def patch_model(
param=None,
)
stored_model_name: Final = update_data.get("model_name")
if (
stored_model_name is not None
and stored_model_name == requested_model_name
@ -980,7 +1056,8 @@ async def _add_model_to_db(
prisma_client: PrismaClient,
new_encryption_key: str | None = None,
should_create_model_in_db: bool = True,
) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
# encrypt litellm params #
_litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
_original_litellm_model_name: Final = model_params.litellm_params.model
@ -998,18 +1075,20 @@ async def _add_model_to_db(
if model_params.model_info.id is not None:
_data["model_id"] = model_params.model_info.id
_create_data: Final = cast("Mapping[str, object]", _data) # cast-ok: str-keyed json payload built just above
if should_create_model_in_db:
model_response = await ModelRepository(prisma_client).table.create(data=_create_data)
else:
model_response = LiteLLM_ProxyModelTable(**_data)
return model_response
if not should_create_model_in_db:
return LiteLLM_ProxyModelTable(**_data)
if slot is None:
return await _proxy_model_table(prisma_client).create(data=_create_data)
async with slot as table:
return await table.create(data=_create_data)
async def _add_team_model_to_db(
model_params: Deployment,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
"""
If 'team_id' is provided,
@ -1040,6 +1119,7 @@ async def _add_team_model_to_db(
model_params=model_params,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
slot=slot,
)
if original_model_name:
@ -1060,7 +1140,8 @@ async def _update_team_model_in_db(
patch_data: updateDeployment,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
) -> PrismaCompatibleUpdateDBModel:
write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]],
) -> _RowT:
"""
Handle team model updates with proper alias management.
@ -1068,6 +1149,9 @@ async def _update_team_model_in_db(
- Creates unique internal model_name and team alias
- Adds model to team object
- Preserves team_public_model_name for external reference
The row is written through ``write_row`` before the team's model list is touched, so a
refused or failed write leaves the team as it was (the create path orders itself the same way).
"""
# Validate team_id if present in patch_data
from litellm.proxy.proxy_server import premium_user
@ -1079,9 +1163,7 @@ async def _update_team_model_in_db(
premium_user=premium_user,
)
# Validated before any write, beside the premium check the create path already runs
# here. The team ACL is updated below and autocommits, so a validator that raises
# further down would leave the team mutated and the deployment row never written.
# Validated before the row write, beside the premium check the create path already runs here.
#
# The merged view is what gets stored, so that is what has to satisfy the invariants.
# Validating the patch alone rejected a partial edit of an already valid deployment:
@ -1101,7 +1183,7 @@ async def _update_team_model_in_db(
# No team_id in patch, proceed with standard update
if patch_team_id is None:
return update_db_model(db_model=db_model, updated_patch=patch_data)
return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
# Determine public model name
public_model_name: Final = _get_public_model_name(
@ -1120,11 +1202,14 @@ async def _update_team_model_in_db(
db_team_id: Final = db_model.model_info.team_id if db_model.model_info else None
is_new_team_assignment: Final = db_team_id != patch_team_id
# Team rows keep their internal UUID-based model_name; the public name lives in model_info
patch_data.model_name = f"model_name_{patch_team_id}_{uuid.uuid4()}" if is_new_team_assignment else None
row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
if is_new_team_assignment:
await _setup_new_team_model_assignment(
team_id=patch_team_id,
public_model_name=public_model_name,
patch_data=patch_data,
user_api_key_dict=user_api_key_dict,
)
else:
@ -1132,12 +1217,11 @@ async def _update_team_model_in_db(
team_id=patch_team_id,
public_model_name=public_model_name,
db_model=db_model,
patch_data=patch_data,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
return update_db_model(db_model=db_model, updated_patch=patch_data)
return row
def _get_public_model_name(
@ -1189,13 +1273,9 @@ def _get_public_model_name(
async def _setup_new_team_model_assignment(
team_id: str,
public_model_name: str,
patch_data: updateDeployment,
user_api_key_dict: UserAPIKeyAuth,
) -> None:
"""Set up a new team model with unique name and team membership."""
unique_model_name: Final = f"model_name_{team_id}_{uuid.uuid4()}"
patch_data.model_name = unique_model_name
"""Register a newly team-assigned model's public name on the team."""
await team_model_add(
data=TeamModelAddRequest(
team_id=team_id,
@ -1385,7 +1465,6 @@ async def _update_existing_team_model_assignment(
team_id: str,
public_model_name: str,
db_model: Deployment,
patch_data: updateDeployment,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient | None,
) -> None:
@ -1409,9 +1488,6 @@ async def _update_existing_team_model_assignment(
old_public_name: Final = db_model.model_info.team_public_model_name if db_model.model_info else None
if old_public_name and public_model_name != old_public_name:
# Clear user-supplied public name from patch before any early return so the
# caller does not overwrite the internal UUID-based model_name in the DB.
patch_data.model_name = None
if prisma_client is None:
verbose_proxy_logger.warning(
"prisma_client not initialized; skipping public name update entirely to avoid orphaned entries"
@ -1459,10 +1535,6 @@ async def _update_existing_team_model_assignment(
# else: old_public_name == public_model_name (no rename needed)
# No team_model_add/delete calls required; public name is already registered
# Always clear patch_data.model_name to prevent caller from overwriting
# the internal UUID-based model_name in the DB with the user-supplied public name
patch_data.model_name = None
class ModelManagementAuthChecks:
"""
@ -1878,18 +1950,19 @@ async def add_new_model(
reload_outcome: ReconcileOutcome = ReconcileOutcome(still_desired=None, live_after=None)
try:
_original_litellm_model_name: Final = model_params.model_name
if model_params.model_info.team_id is None:
model_response = await _add_model_to_db(
model_params=priced_model_params,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
else:
model_response = await _add_team_model_to_db(
model_params=priced_model_params,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
add_model: Final = (
_add_model_to_db if model_params.model_info.team_id is None else _add_team_model_to_db
)
model_response = await add_model(
model_params=priced_model_params,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
slot=_heuristic_v2_slot(
prisma_client,
effective_config=priced_model_params.litellm_params.complexity_router_config,
model_id=priced_model_params.model_info.id,
),
)
reload_outcome = await proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
@ -1903,6 +1976,8 @@ async def add_new_model(
passed_model_info=priced_model_params.model_info,
)
except Exception as e:
if isinstance(e, HTTPException):
raise
verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
else:
@ -2070,10 +2145,17 @@ async def update_model(
"updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
**({} if renamed_to is None else {"model_name": renamed_to}),
}
model_response: Final = await _proxy_model_table(prisma_client).update(
where={"model_id": _model_id},
data=_data,
)
async with _heuristic_v2_slot(
prisma_client,
effective_config=_effective_complexity_router_config(
model_params.litellm_params, deployment.litellm_params
),
model_id=_model_id,
) as table:
model_response: Final = await table.update(
where={"model_id": _model_id},
data=_data,
)
if renamed_to is not None:
await sync_access_groups_for_renamed_model(
prisma_client=prisma_client,

23
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -10,6 +10,7 @@ All /team management endpoints
"""
import asyncio
import copy
import json
import math
import traceback
@ -2189,6 +2190,26 @@ async def update_team(
if field in updated_kv
}
_writes_metadata_backed_field: Final = any(
field in updated_kv
for field in (
*LiteLLM_ManagementEndpoint_MetadataFields,
*LiteLLM_ManagementEndpoint_MetadataFields_Premium,
)
)
if isinstance(existing_team_row.metadata, dict):
if "metadata" not in updated_kv and (_team_member_fields_in_request or _writes_metadata_backed_field):
updated_kv["metadata"] = copy.deepcopy(existing_team_row.metadata)
elif isinstance(updated_kv.get("metadata"), dict):
updated_kv["metadata"] = {
**updated_kv["metadata"],
**{
key: existing_team_row.metadata[key]
for key in TeamMemberBudgetHandler.SYSTEM_MANAGED_METADATA_KEYS
if key in existing_team_row.metadata
},
}
if _team_member_fields_in_request and TeamMemberBudgetHandler.should_create_budget(
team_member_budget=data.team_member_budget,
team_member_rpm_limit=data.team_member_rpm_limit,
@ -4318,6 +4339,8 @@ async def _resolve_team_access_group_resources(
access_group_id=group.access_group_id,
access_group_name=group.access_group_name,
models=tuple(group.access_model_names or ()),
mcp_server_ids=tuple(group.access_mcp_server_ids or ()),
agent_ids=tuple(group.access_agent_ids or ()),
)
for group in resolved_groups
),

12
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -1730,6 +1730,16 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict:
return headers
def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool:
return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens"
def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]:
if not _is_vertex_anthropic_count_tokens_route(endpoint):
return headers
return MappingProxyType({name: value for name, value in headers.items() if name.lower() != "anthropic-beta"})
def get_vertex_pass_through_handler(
call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here
) -> BaseVertexAIPassThroughHandler:
@ -2128,7 +2138,7 @@ async def _base_vertex_proxy_route(
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=target,
custom_headers=headers,
custom_headers=_upstream_headers_for_vertex_route(endpoint, headers),
is_streaming_request=is_streaming_request,
) # dynamically construct pass-through endpoint based on incoming path

27
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -107,6 +107,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
response_id=optional_str(response_body.get("id")),
)
return {
@ -148,8 +149,9 @@ class AnthropicPassthroughLoggingHandler:
return model
@staticmethod
def _extract_model_from_anthropic_chunks(
def _extract_message_start_field(
all_chunks: Sequence[str | bytes],
field: str,
) -> str | None:
for raw in all_chunks:
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
@ -163,11 +165,23 @@ class AnthropicPassthroughLoggingHandler:
if not isinstance(data, dict):
continue
if data.get("type") == "message_start":
model = (data.get("message") or {}).get("model")
if model:
return model
value = (data.get("message") or {}).get(field)
if isinstance(value, str) and value:
return value
return None
@staticmethod
def _extract_model_from_anthropic_chunks(
all_chunks: Sequence[str | bytes],
) -> str | None:
return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "model")
@staticmethod
def _extract_response_id_from_anthropic_chunks(
all_chunks: Sequence[str | bytes],
) -> str | None:
return AnthropicPassthroughLoggingHandler._extract_message_start_field(all_chunks, "id")
@staticmethod
def _stream_was_interrupted(
all_chunks: Sequence[str | bytes],
@ -251,6 +265,7 @@ class AnthropicPassthroughLoggingHandler:
start_time: datetime,
end_time: datetime,
logging_obj: LiteLLMLoggingObj,
response_id: str | None = None,
):
"""
Create the standard logging object for Anthropic passthrough
@ -312,8 +327,7 @@ class AnthropicPassthroughLoggingHandler:
json.dumps(kwargs, indent=4, default=str),
)
# set litellm_call_id to logging response object
litellm_model_response.id = logging_obj.litellm_call_id
litellm_model_response.id = response_id or logging_obj.litellm_call_id
litellm_model_response.model = model
logging_obj.model_call_details["model"] = model
if not logging_obj.model_call_details.get("custom_llm_provider"):
@ -413,6 +427,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=litellm_logging_obj,
response_id=AnthropicPassthroughLoggingHandler._extract_response_id_from_anthropic_chunks(all_chunks),
)
return {

241
litellm/proxy/prompts/prompt_endpoints.py
View file

@ -29,6 +29,12 @@ from litellm.proxy._types import (
from litellm.proxy.auth.auth_utils import is_request_body_safe
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.path_utils import safe_filename
from litellm.proxy.prompts.prompt_registry import (
DEFAULT_PROMPT_ENVIRONMENT,
get_base_prompt_id,
get_version_number,
prompt_environment_or_default,
)
from litellm.repositories.table_repositories import PromptRepository
from litellm.types.prompts.init_prompts import (
ListPromptsResponse,
@ -102,165 +108,20 @@ def _prompt_table(prisma_client: "PrismaClient") -> _PromptTableActions:
return PromptRepository(prisma_client).table
def get_base_prompt_id(prompt_id: str) -> str:
"""
Extract the base prompt ID by stripping the version suffix if present.
Args:
prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v1" or "jack_success_v1")
Returns:
Base prompt ID without version suffix (e.g., "jack_success")
Examples:
>>> get_base_prompt_id("jack_success.v1")
"jack_success"
>>> get_base_prompt_id("jack_success_v1")
"jack_success"
>>> get_base_prompt_id("jack_success")
"jack_success"
"""
# Try dot separator first (.v)
if ".v" in prompt_id:
return prompt_id.split(".v")[0]
# Try underscore separator (_v)
if "_v" in prompt_id:
return prompt_id.split("_v")[0]
return prompt_id
def get_version_number(prompt_id: str) -> int:
"""
Extract the version number from a versioned prompt ID.
Args:
prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v2" or "jack_success_v2")
Returns:
Version number (defaults to 1 if no version suffix or invalid format)
Examples:
>>> get_version_number("jack_success.v2")
2
>>> get_version_number("jack_success_v2")
2
>>> get_version_number("jack_success")
1
"""
# Try dot separator first (.v)
if ".v" in prompt_id:
version_str = prompt_id.split(".v")[1]
try:
return int(version_str)
except ValueError:
pass
# Try underscore separator (_v)
if "_v" in prompt_id:
version_str = prompt_id.split("_v")[1]
try:
return int(version_str)
except ValueError:
pass
return 1
def construct_versioned_prompt_id(prompt_id: str, version: int | None = None) -> str:
"""
Construct a versioned prompt ID from a base prompt_id and version number.
Args:
prompt_id: Base prompt ID (e.g., "jack_success")
version: Version number (if None, returns the base prompt_id unchanged)
Returns:
Versioned prompt ID (e.g., "jack_success.v4")
Examples:
>>> construct_versioned_prompt_id("jack_success", 4)
"jack_success.v4"
>>> construct_versioned_prompt_id("jack_success", None)
"jack_success"
>>> construct_versioned_prompt_id("jack_success.v2", 4)
"jack_success.v4"
"""
if version is None:
return prompt_id
# Strip any existing version suffix first
base_id: Final = get_base_prompt_id(prompt_id)
return f"{base_id}.v{version}"
def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: Mapping[str, object]) -> str:
"""
Find the latest version of a prompt from available prompt IDs.
Args:
prompt_id: Base prompt ID or versioned prompt ID (e.g., "jack_success" or "jack_success.v2")
all_prompt_ids: Dictionary of all available prompt IDs (keys are prompt IDs)
Returns:
The prompt ID with the highest version number, or the original prompt_id if no versions exist
Examples:
>>> all_ids = {"jack.v1": {}, "jack.v2": {}, "jack.v3": {}}
>>> get_latest_version_prompt_id("jack", all_ids)
"jack.v3"
>>> get_latest_version_prompt_id("jack.v1", all_ids)
"jack.v3"
>>> all_ids = {"simple": {}}
>>> get_latest_version_prompt_id("simple", all_ids)
"simple"
"""
base_id: Final = get_base_prompt_id(prompt_id=prompt_id)
# Find all versions of this prompt
matching_versions: Final = []
for stored_prompt_id in all_prompt_ids:
if get_base_prompt_id(prompt_id=stored_prompt_id) == base_id:
version_num = get_version_number(prompt_id=stored_prompt_id)
matching_versions.append((version_num, stored_prompt_id))
# Use the highest version number
if matching_versions:
matching_versions.sort(reverse=True)
return matching_versions[0][1]
else:
# No versioned prompts found, use the base ID as-is
return prompt_id
def get_latest_prompt_versions(prompts: list[PromptSpec]) -> list[PromptSpec]:
"""
Filter a list of prompts to return only the latest version of each unique prompt.
Args:
prompts: List of PromptSpec objects
Returns:
List of PromptSpec objects with only the latest version of each prompt
Filter prompts down to the latest version per (base prompt id, environment).
"""
latest_prompts: Final[dict[str, PromptSpec]] = {}
for prompt in prompts:
base_id = get_base_prompt_id(prompt_id=prompt.prompt_id)
version = get_version_number(prompt_id=prompt.prompt_id)
# Keep the prompt with the highest version number
if base_id not in latest_prompts:
latest_prompts[base_id] = prompt
else:
existing_version = get_version_number(prompt_id=latest_prompts[base_id].prompt_id)
if version > existing_version:
latest_prompts[base_id] = prompt
sorted_prompts: Final = sorted(prompts, key=lambda prompt: get_version_number(prompt_id=prompt.prompt_id))
latest_prompts: Final = {
(get_base_prompt_id(prompt_id=prompt.prompt_id), prompt_environment_or_default(prompt.environment)): prompt
for prompt in sorted_prompts
}
return list(latest_prompts.values())
async def get_next_version_for_prompt(
prisma_client: "PrismaClient", prompt_id: str, environment: str = "development"
prisma_client: "PrismaClient", prompt_id: str, environment: str = DEFAULT_PROMPT_ENVIRONMENT
) -> int:
"""
Get the next version number for a prompt in a specific environment.
@ -403,11 +264,14 @@ async def list_prompts(
if key_metadata is not None:
prompts: Final = cast(list[str] | None, key_metadata.get("prompts", None))
if prompts is not None:
all_prompts = [
IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id]
for prompt_id in prompts
if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS
allowed_prompt_ids: Final = frozenset(prompts)
allowed_prompts: Final = [
spec
for spec in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()
if spec.prompt_id in allowed_prompt_ids
or get_base_prompt_id(prompt_id=spec.prompt_id) in allowed_prompt_ids
]
all_prompts = get_latest_prompt_versions(prompts=allowed_prompts)
if environment:
all_prompts = [p for p in all_prompts if p.environment == environment]
prompt_list: Final = []
@ -576,7 +440,7 @@ def _get_prompt_template(prompt_spec: PromptSpec, base_prompt_id: str) -> Prompt
metadata=parsed.get("metadata"),
)
else:
prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(prompt_spec.prompt_id)
prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec)
if prompt_callback is not None:
integration_name: Final = prompt_callback.integration_name
if integration_name == "dotprompt":
@ -690,15 +554,10 @@ async def get_prompt_info(
if env_prompts:
prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0])
# Fallback: use in-memory registry (no environment filter)
if prompt_spec is None and environment is None:
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
if prompt_spec is None:
latest_prompt_id: Final = get_latest_version_prompt_id(
prompt_id=prompt_id,
all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
)
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
if prompt_spec is None:
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(
prompt_id, version=requested_version, environment=environment
)
if prompt_spec is None:
raise HTTPException(
@ -785,7 +644,7 @@ async def create_prompt(
environment: Final = (
request.prompt_info.environment
if request.prompt_info and request.prompt_info.environment
else "development"
else DEFAULT_PROMPT_ENVIRONMENT
)
# Get next version number
@ -885,7 +744,7 @@ async def update_prompt(
environment: Final = (
request.prompt_info.environment
if request.prompt_info and request.prompt_info.environment
else "development"
else DEFAULT_PROMPT_ENVIRONMENT
)
# Check if any version of this prompt exists (in any environment)
@ -897,9 +756,7 @@ async def update_prompt(
detail=f"Prompt with ID {base_prompt_id} not found",
)
# Check if it's a config prompt
existing_in_memory: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
if existing_in_memory and existing_in_memory.prompt_info.prompt_type == "config":
if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
raise HTTPException(
status_code=400,
detail="Cannot update config prompts.",
@ -988,40 +845,26 @@ async def delete_prompt(
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
try:
# Try to get prompt directly first
existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
# If not found, try to find the latest version
if existing_prompt is None:
latest_prompt_id: Final = get_latest_version_prompt_id(
prompt_id=prompt_id,
all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
)
existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
# Use the resolved prompt_id for deletion
prompt_id = latest_prompt_id
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, environment=environment)
if existing_prompt is None:
raise HTTPException(status_code=404, detail=f"Prompt with ID {prompt_id} not found")
if existing_prompt.prompt_info.prompt_type == "config":
if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
raise HTTPException(
status_code=400,
detail="Cannot delete config prompts.",
)
# Get the base prompt ID (without version suffix) for database deletion
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
# Build delete filter; scope to environment if provided
delete_where: Final[dict[str, str]] = {"prompt_id": base_prompt_id}
if environment:
delete_where["environment"] = environment
# Delete versions from the database (scoped to environment if provided)
delete_where: Final[dict[str, str]] = {
"prompt_id": base_prompt_id,
**({"environment": environment} if environment else {}),
}
await _prompt_table(prisma_client).delete_many(where=delete_where)
IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id, environment=environment or None)
IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(
base_prompt_id=base_prompt_id, environment=environment or None
)
env_msg: Final = f" from {environment}" if environment else ""
return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"}
@ -1093,7 +936,7 @@ async def patch_prompt(
try:
# Resolve the target row: find the latest version in the given environment
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
env: Final = environment or "development"
env: Final = prompt_environment_or_default(environment)
requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None
# Build query to find the exact row by composite unique key
@ -1117,11 +960,7 @@ async def patch_prompt(
target_row: Final = db_rows[0]
# Check if prompt exists in memory
versioned_id: Final = f"{base_prompt_id}.v{target_row.version}"
existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id)
if existing_prompt and existing_prompt.prompt_info.prompt_type == "config":
if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id):
raise HTTPException(
status_code=400,
detail="Cannot update config prompts.",

189
litellm/proxy/prompts/prompt_registry.py
View file

@ -1,6 +1,6 @@
import importlib
import os
from collections.abc import Callable
from collections.abc import Callable, Sequence
from pathlib import Path
from typing import Final
@ -14,6 +14,87 @@ from litellm.types.prompts.init_prompts import (
prompt_initializer_registry = {}
DEFAULT_PROMPT_ENVIRONMENT: Final = "development"
PROMPT_ENVIRONMENT_SERVE_PRECEDENCE: Final = ("production", "staging", "development")
def get_base_prompt_id(prompt_id: str) -> str:
"""
Extract the base prompt ID by stripping the version suffix if present.
Examples:
>>> get_base_prompt_id("jack_success.v1")
"jack_success"
>>> get_base_prompt_id("jack_success_v1")
"jack_success"
>>> get_base_prompt_id("jack_success")
"jack_success"
"""
if ".v" in prompt_id:
return prompt_id.split(".v")[0]
if "_v" in prompt_id:
return prompt_id.split("_v")[0]
return prompt_id
def get_version_number(prompt_id: str) -> int:
"""
Extract the version number from a versioned prompt ID (defaults to 1).
Examples:
>>> get_version_number("jack_success.v2")
2
>>> get_version_number("jack_success_v2")
2
>>> get_version_number("jack_success")
1
"""
if ".v" in prompt_id:
version_str = prompt_id.split(".v")[1]
try:
return int(version_str)
except ValueError:
pass
if "_v" in prompt_id:
version_str = prompt_id.split("_v")[1]
try:
return int(version_str)
except ValueError:
pass
return 1
def prompt_environment_or_default(environment: str | None) -> str:
return environment or DEFAULT_PROMPT_ENVIRONMENT
def registry_key_for_prompt(prompt: PromptSpec) -> str:
return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}"
def parse_prompt_version(raw_version: object) -> int | None:
if isinstance(raw_version, bool):
return None
if isinstance(raw_version, int):
return raw_version
if isinstance(raw_version, str) and raw_version.isdigit():
return int(raw_version)
return None
def _spec_version(prompt: PromptSpec) -> int:
return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id)
def _default_serve_environment(prompts: Sequence[PromptSpec]) -> str:
present: Final = frozenset(prompt_environment_or_default(prompt.environment) for prompt in prompts)
ladder_pick: Final = next((env for env in PROMPT_ENVIRONMENT_SERVE_PRECEDENCE if env in present), None)
if ladder_pick is not None:
return ladder_pick
return min(present) if present else DEFAULT_PROMPT_ENVIRONMENT
def get_prompt_initializer_from_integrations():
"""
@ -113,17 +194,16 @@ class InMemoryPromptRegistry:
"""
import litellm
prompt_id: Final = prompt.prompt_id
if prompt_id in self.IN_MEMORY_PROMPTS:
verbose_proxy_logger.debug("prompt_id already exists in IN_MEMORY_PROMPTS")
return self.IN_MEMORY_PROMPTS[prompt_id]
registry_key: Final = registry_key_for_prompt(prompt)
if registry_key in self.IN_MEMORY_PROMPTS:
verbose_proxy_logger.debug("prompt already exists in IN_MEMORY_PROMPTS")
return self.IN_MEMORY_PROMPTS[registry_key]
parsed_prompt, custom_prompt_callback = self._build_prompt_callback(prompt=prompt)
litellm.logging_callback_manager.add_litellm_callback(custom_prompt_callback)
# store references to the prompt in memory
self.IN_MEMORY_PROMPTS[prompt_id] = parsed_prompt
self.prompt_id_to_custom_prompt[prompt_id] = custom_prompt_callback
self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt
self.prompt_id_to_custom_prompt[registry_key] = custom_prompt_callback
return parsed_prompt
@ -166,68 +246,93 @@ class InMemoryPromptRegistry:
import litellm
parsed_prompt, new_callback = self._build_prompt_callback(prompt=prompt)
stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt.prompt_id, None)
self.IN_MEMORY_PROMPTS.pop(prompt.prompt_id, None)
registry_key: Final = registry_key_for_prompt(parsed_prompt)
stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None)
self.IN_MEMORY_PROMPTS.pop(registry_key, None)
if stale_callback is not None:
litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback)
litellm.logging_callback_manager.add_litellm_callback(new_callback)
self.IN_MEMORY_PROMPTS[prompt.prompt_id] = parsed_prompt
self.prompt_id_to_custom_prompt[prompt.prompt_id] = new_callback
self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt
self.prompt_id_to_custom_prompt[registry_key] = new_callback
return parsed_prompt
def sync_prompt_from_db(self, prompt: PromptSpec) -> PromptSpec | None:
existing: Final = self.IN_MEMORY_PROMPTS.get(prompt.prompt_id)
existing: Final = self.IN_MEMORY_PROMPTS.get(registry_key_for_prompt(prompt))
if existing is None:
return self.initialize_prompt(prompt=prompt)
if existing.litellm_params == prompt.litellm_params and existing.prompt_info == prompt.prompt_info:
return existing
return self.reload_prompt(prompt=prompt)
def get_prompt_by_id(self, prompt_id: str) -> PromptSpec | None:
def resolve_prompt_spec(
self,
prompt_id: str,
version: int | None = None,
environment: str | None = None,
) -> PromptSpec | None:
"""
Get a prompt by its ID from memory
"""
return self.IN_MEMORY_PROMPTS.get(prompt_id)
Resolve a prompt spec by base prompt id, optional version, and optional environment.
def get_prompt_callback_by_id(self, prompt_id: str) -> CustomPromptManagement | None:
With no environment, resolves within the default serve environment
(production > staging > development > alphabetical first present).
With no version, resolves to the highest version in the chosen environment.
"""
Get a prompt callback by its ID from memory
"""
return self.prompt_id_to_custom_prompt.get(prompt_id)
base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id)
base_matches: Final = tuple(
spec
for spec in self.IN_MEMORY_PROMPTS.values()
if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
)
if not base_matches:
return None
resolved_environment: Final = (
environment if environment is not None else _default_serve_environment(base_matches)
)
env_matches: Final = tuple(
spec for spec in base_matches if prompt_environment_or_default(spec.environment) == resolved_environment
)
if not env_matches:
return None
if version is not None:
return next((spec for spec in env_matches if _spec_version(spec) == version), None)
return max(env_matches, key=_spec_version)
def remove_prompt(self, prompt_id: str) -> None:
def get_prompt_callback_for_prompt(self, prompt: PromptSpec) -> CustomPromptManagement | None:
return self.prompt_id_to_custom_prompt.get(registry_key_for_prompt(prompt))
def has_config_prompt(self, base_prompt_id: str) -> bool:
return any(
spec.prompt_info.prompt_type == "config"
for spec in self.IN_MEMORY_PROMPTS.values()
if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
)
def remove_prompt(self, registry_key: str) -> None:
import litellm
self.IN_MEMORY_PROMPTS.pop(prompt_id, None)
stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt_id, None)
self.IN_MEMORY_PROMPTS.pop(registry_key, None)
stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None)
if stale_callback is not None:
litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback)
def delete_prompts_by_base_id(self, base_prompt_id: str, environment: str | None = None) -> list[str]:
"""
Delete all prompts matching the given base prompt ID from memory, along with their
registered callbacks; scoped to one environment when given.
Delete matching prompts from memory, along with their registered callbacks,
scoped to one environment when given.
Args:
base_prompt_id: The base prompt ID (without version suffix)
environment: When set, only delete prompts deployed to this environment
Returns:
List of prompt IDs that were deleted
Returns the registry keys that were deleted.
"""
from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id
prompts_to_delete: Final = [
pid
for pid, prompt in self.IN_MEMORY_PROMPTS.items()
if get_base_prompt_id(prompt_id=pid) == base_prompt_id
and (environment is None or prompt.environment == environment)
keys_to_delete: Final = [
key
for key, spec in self.IN_MEMORY_PROMPTS.items()
if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id
and (environment is None or prompt_environment_or_default(spec.environment) == environment)
]
for pid in prompts_to_delete:
self.remove_prompt(prompt_id=pid)
for key in keys_to_delete:
self.remove_prompt(registry_key=key)
return prompts_to_delete
return keys_to_delete
IN_MEMORY_PROMPT_REGISTRY: Final = InMemoryPromptRegistry()

50
litellm/proxy/proxy_server.py
View file

@ -120,6 +120,8 @@ from litellm.router_utils.add_retry_fallback_headers import (
from litellm.router_utils.auto_router_model_naming import (
STRATEGY_ROUTER_PARAM_FIELDS,
carries_complexity_router_settings,
count_heuristic_v2_routers,
heuristic_v2_limit_violation,
validate_complexity_router_config_placement,
)
from litellm.types.utils import (
@ -301,7 +303,7 @@ from litellm.proxy.auth.auth_utils import (
)
from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.litellm_license import LicenseCheck
from litellm.proxy.auth.litellm_license import HEURISTIC_V2_LICENSE_REMEDY, LicenseCheck
from litellm.proxy.auth.model_checks import (
expand_wildcard_deployments_for_model_info,
get_all_fallbacks,
@ -4316,6 +4318,19 @@ def validate_deployment_complexity_router_placement(model: Mapping[str, object])
raise ValueError(f"model {model.get('model_name', '')!r}: {violation}")
def validate_heuristic_v2_router_limit(model_list: Sequence[Mapping[str, object]], *, limit: int | None) -> None:
"""
Refuse to start when config.yaml defines more heuristic_v2 auto-routers than the license allows.
Checked here rather than left to router registration for the same reason as the two
validators above: the proxy builds its router with `ignore_invalid_deployments=True`, so
the router's own refusal would turn the extra router into a silently missing model.
"""
violation: Final = heuristic_v2_limit_violation(held=count_heuristic_v2_routers(model_list), limit=limit)
if violation is not None:
raise ValueError(f"config.yaml model_list: {violation} {HEURISTIC_V2_LICENSE_REMEDY}")
def pin_complexity_router_model_id(model: dict) -> None: # mutable-ok: out-param, model_info is stamped in place
"""
Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps
@ -5721,6 +5736,7 @@ class ProxyConfig:
model_list: Final = config.get("model_list", None)
if model_list:
router_params["model_list"] = model_list
validate_heuristic_v2_router_limit(model_list, limit=_license_check.heuristic_v2_router_limit())
print( # noqa: T201
"\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m"
)
@ -5810,6 +5826,7 @@ class ProxyConfig:
),
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
fallback_access_check=router_fallback_access_check,
heuristic_v2_router_limit=_license_check.heuristic_v2_router_limit,
)
if redis_usage_cache is not None and router.cache.redis_cache is None:
@ -6270,6 +6287,7 @@ class ProxyConfig:
search_tools=search_tools,
ignore_invalid_deployments=True,
fallback_access_check=router_fallback_access_check,
heuristic_v2_router_limit=_license_check.heuristic_v2_router_limit,
)
verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
else:
@ -7571,7 +7589,7 @@ class ProxyConfig:
return create_versioned_prompt_spec(db_prompt=db_prompt)
async def _init_prompts_in_db(self, prisma_client: PrismaClient):
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY, registry_key_for_prompt
from litellm.types.prompts.init_prompts import PromptSpec
def parse_row(db_prompt: object) -> PromptSpec | None:
@ -7586,21 +7604,12 @@ class ProxyConfig:
return None
try:
prompt_ids_loaded_before_db_read: Final = frozenset(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS)
registry_keys_loaded_before_db_read: Final = frozenset(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS)
prompts_in_db: Final[Sequence[object]] = await PromptRepository(prisma_client).table.find_many()
parsed_specs: Final[tuple[PromptSpec, ...]] = tuple(
spec for row in prompts_in_db if (spec := parse_row(row)) is not None
)
newest_spec_per_id: Final[Mapping[str, PromptSpec]] = MappingProxyType(
{
spec.prompt_id: spec
for spec in sorted(
parsed_specs,
key=lambda s: s.updated_at.timestamp() if s.updated_at else float("-inf"),
)
}
)
for prompt_spec in newest_spec_per_id.values():
for prompt_spec in parsed_specs:
try:
IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec)
except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the remaining prompts
@ -7612,15 +7621,16 @@ class ProxyConfig:
# An unparsable row still exists in the DB, so skip the sweep rather than unload its in-memory copy
every_row_parsed: Final = len(parsed_specs) == len(prompts_in_db)
if every_row_parsed:
deleted_db_prompt_ids: Final = tuple(
prompt_id
for prompt_id in prompt_ids_loaded_before_db_read
if (loaded_spec := IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.get(prompt_id)) is not None
db_registry_keys: Final = frozenset(registry_key_for_prompt(spec) for spec in parsed_specs)
deleted_db_registry_keys: Final = tuple(
registry_key
for registry_key in registry_keys_loaded_before_db_read
if (loaded_spec := IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.get(registry_key)) is not None
and loaded_spec.prompt_info.prompt_type == "db"
and prompt_id not in newest_spec_per_id
and registry_key not in db_registry_keys
)
for deleted_prompt_id in deleted_db_prompt_ids:
IN_MEMORY_PROMPT_REGISTRY.remove_prompt(prompt_id=deleted_prompt_id)
for deleted_registry_key in deleted_db_registry_keys:
IN_MEMORY_PROMPT_REGISTRY.remove_prompt(registry_key=deleted_registry_key)
except Exception as e:
verbose_proxy_logger.debug("litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - %s", e)

1
litellm/proxy/rag_endpoints/endpoints.py
View file

@ -761,6 +761,7 @@ async def rag_query(
model=model,
messages=messages,
retrieval_config=merged_retrieval_config,
vector_store_params=store_data,
rerank=rerank,
stream=stream,
router=llm_router,

21
litellm/proxy/spend_tracking/spend_management_endpoints.py
View file

@ -173,6 +173,9 @@ class _SessionSpendRow(TypedDict):
session_cache_hit_count: ReadOnly[int]
session_llm_count: ReadOnly[int]
session_agent_count: ReadOnly[int]
session_total_prompt_tokens: ReadOnly[int]
session_total_completion_tokens: ReadOnly[int]
session_total_tokens: ReadOnly[int]
session_models: ReadOnly[Sequence[str]]
@ -188,6 +191,9 @@ class _SessionSpendStats(NamedTuple):
session_cache_hit_count: int
session_llm_count: int
session_agent_count: int
session_total_prompt_tokens: int
session_total_completion_tokens: int
session_total_tokens: int
session_models: Sequence[str]
session_models_truncated: bool
@ -4287,8 +4293,8 @@ async def _build_ui_spend_logs_response(
Build the paginated response for the UI spend-logs endpoint.
When ``enrich_session_counts`` is ``True`` (the default for the v1/UI
endpoint), each row is enriched with ``session_total_count`` plus spend
and call-type aggregates so the frontend knows which sessions are
endpoint), each row is enriched with ``session_total_count`` plus spend,
token and call-type aggregates so the frontend knows which sessions are
expandable (multi-call sessions). One ``GROUP BY (session_id, api_key)``
query serves every referenced session, keyed per api key so two callers
reusing a session id never see each other's totals. Rows without a
@ -4356,7 +4362,10 @@ async def _build_ui_spend_logs_response(
COUNT(*) FILTER (
WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL}
)::int AS session_llm_count,
COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count
COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens,
COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens,
COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens
FROM "LiteLLM_SpendLogs"
WHERE session_id = ANY($1::text[])
AND api_key = ANY($2::text[])
@ -4389,6 +4398,9 @@ async def _build_ui_spend_logs_response(
session_cache_hit_count=int(row.get("session_cache_hit_count") or 0),
session_llm_count=int(row.get("session_llm_count") or 0),
session_agent_count=int(row.get("session_agent_count") or 0),
session_total_prompt_tokens=int(row.get("session_total_prompt_tokens") or 0),
session_total_completion_tokens=int(row.get("session_total_completion_tokens") or 0),
session_total_tokens=int(row.get("session_total_tokens") or 0),
session_models=models[:_SESSION_MODELS_LIMIT],
session_models_truncated=len(models) > _SESSION_MODELS_LIMIT,
)
@ -4418,6 +4430,9 @@ async def _build_ui_spend_logs_response(
row_dict["session_cache_hit_count"] = session_stats.session_cache_hit_count
row_dict["session_llm_count"] = session_stats.session_llm_count
row_dict["session_agent_count"] = session_stats.session_agent_count
row_dict["session_total_prompt_tokens"] = session_stats.session_total_prompt_tokens
row_dict["session_total_completion_tokens"] = session_stats.session_total_completion_tokens
row_dict["session_total_tokens"] = session_stats.session_total_tokens
row_dict["session_models"] = session_stats.session_models
row_dict["session_models_truncated"] = session_stats.session_models_truncated
enriched.append(row_dict)

36
litellm/proxy/utils.py
View file

@ -1478,28 +1478,27 @@ class ProxyLogging:
) -> None:
"""Process prompt template if applicable."""
from litellm.proxy.prompts.prompt_endpoints import (
construct_versioned_prompt_id,
get_latest_version_prompt_id,
)
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.utils import get_non_default_completion_params
if prompt_version is None:
lookup_prompt_id = get_latest_version_prompt_id(
prompt_id=prompt_id,
all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
)
else:
lookup_prompt_id = construct_versioned_prompt_id(prompt_id=prompt_id, version=prompt_version)
custom_logger: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(lookup_prompt_id)
prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(lookup_prompt_id)
raw_prompt_environment: Final = data.get("prompt_environment", None)
prompt_environment: Final = raw_prompt_environment if isinstance(raw_prompt_environment, str) else None
prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(
prompt_id,
version=prompt_version,
environment=prompt_environment,
)
custom_logger: Final = (
IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec)
if prompt_spec is not None
else None
)
litellm_prompt_id: str | None = None
if prompt_spec is not None:
litellm_prompt_id = prompt_spec.litellm_params.prompt_id
data.pop("prompt_id", None)
data.pop("prompt_environment", None)
if custom_logger and prompt_spec is not None:
is_responses_call: Final = call_type == "aresponses"
@ -1542,6 +1541,7 @@ class ProxyLogging:
data.pop("prompt_variables", None)
data.pop("prompt_label", None)
data.pop("prompt_version", None)
data.pop("prompt_environment", None)
def _process_guardrail_metadata(self, data: dict) -> None:
"""Process guardrails from metadata and add to applied_guardrails."""
@ -1750,7 +1750,6 @@ class ProxyLogging:
litellm_logging_obj: Final = cast(Optional["LiteLLMLoggingObj"], data.get("litellm_logging_obj", None))
prompt_id: Final[str | None] = data.get("prompt_id", None)
prompt_version: Final[int | None] = data.get("prompt_version", None)
## PROMPT TEMPLATE CHECK ##
@ -1760,11 +1759,13 @@ class ProxyLogging:
and prompt_id is not None
and (call_type == "completion" or call_type == "acompletion" or call_type == "aresponses")
):
from litellm.proxy.prompts.prompt_registry import parse_prompt_version
await self._process_prompt_template(
data=data,
litellm_logging_obj=litellm_logging_obj,
prompt_id=prompt_id,
prompt_version=prompt_version,
prompt_version=parse_prompt_version(data.get("prompt_version", None)),
call_type=call_type,
)
@ -7531,6 +7532,9 @@ def create_model_info_response(
max_input_tokens = configured_input
if configured_output is not None:
max_output_tokens = configured_output
configured_mode: Final = llm_router.get_configured_mode(model_id)
if isinstance(configured_mode, str):
base["mode"] = configured_mode
if max_input_tokens is not None:
base["max_input_tokens"] = max_input_tokens

14
litellm/proxy/vector_store_endpoints/management_endpoints.py
View file

@ -30,7 +30,10 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store
from litellm.proxy.vector_store_endpoints.utils import (
can_user_access_vector_store,
filter_listable_vector_stores,
)
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import ManagedVectorStoresRepository
from litellm.types.vector_stores import (
@ -390,11 +393,10 @@ async def list_vector_stores(
# Filter vector stores based on access control
accessible_vector_stores: Final = []
for vs in vector_store_map.values():
if await _check_vector_store_access(vs, user_api_key_dict):
redacted = LiteLLM_ManagedVectorStore(**vs)
redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params"))
accessible_vector_stores.append(redacted)
for vs in await filter_listable_vector_stores(vector_store_map.values(), user_api_key_dict):
redacted = LiteLLM_ManagedVectorStore(**vs)
redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params"))
accessible_vector_stores.append(redacted)
total_count: Final = len(accessible_vector_stores)
total_pages: Final = (total_count + page_size - 1) // page_size

78
litellm/proxy/vector_store_endpoints/utils.py
View file

@ -1,11 +1,17 @@
import json
import re
from collections.abc import Iterable
from types import MappingProxyType
from typing import Any, Final, Literal
from fastapi import HTTPException, Request
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._experimental.mcp_server.ui_session_utils import (
is_ui_session_credential,
resolve_ui_session_team_ids,
)
from litellm.proxy._types import (
LiteLLM_ObjectPermissionTable,
LitellmUserRoles,
@ -160,10 +166,16 @@ async def can_user_access_vector_store(
if _is_proxy_admin(user_api_key_dict):
return True
vector_store_team_id: Final = vector_store.get("team_id")
if vector_store_team_id is None:
if vector_store.get("team_id") is None:
return True
return await _is_vector_store_granted(vector_store, user_api_key_dict)
async def _is_vector_store_granted(
vector_store: LiteLLM_ManagedVectorStore,
user_api_key_dict: UserAPIKeyAuth,
) -> bool:
vector_store_id: Final = vector_store.get("vector_store_id") or ""
key_object_permission = user_api_key_dict.object_permission
@ -178,12 +190,70 @@ async def can_user_access_vector_store(
if _object_permission_allows_vector_store(team_object_permission, vector_store_id):
return True
if user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store_team_id:
return True
return user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store.get("team_id")
async def _team_auth_context(team_id: str, user_api_key_dict: UserAPIKeyAuth) -> UserAPIKeyAuth:
from litellm.proxy.auth.auth_checks import get_team_object
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
team: Final = await get_team_object(
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=user_api_key_dict.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
return user_api_key_dict.model_copy(
update=MappingProxyType(
{
"team_id": team_id,
"team_object_permission": team.object_permission,
"team_object_permission_id": team.object_permission_id,
}
)
)
async def _vector_store_listing_auth_contexts(
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[UserAPIKeyAuth, ...]:
if not is_ui_session_credential(user_api_key_dict):
return (user_api_key_dict,)
session_key_context: Final = user_api_key_dict.model_copy(
update=MappingProxyType({"team_id": None, "team_object_permission": None, "team_object_permission_id": None})
)
team_ids: Final = await resolve_ui_session_team_ids(user_api_key_dict)
team_contexts: Final = tuple([await _team_auth_context(team_id, user_api_key_dict) for team_id in team_ids])
return (session_key_context, *team_contexts)
async def _is_vector_store_granted_to_any(
vector_store: LiteLLM_ManagedVectorStore,
auth_contexts: tuple[UserAPIKeyAuth, ...],
) -> bool:
for auth_context in auth_contexts:
if await _is_vector_store_granted(vector_store, auth_context):
return True
return False
async def filter_listable_vector_stores(
vector_stores: Iterable[LiteLLM_ManagedVectorStore],
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[LiteLLM_ManagedVectorStore, ...]:
"""Non-admins only see stores their key, one of their teams' object_permission, or team ownership grants."""
if _is_proxy_admin(user_api_key_dict):
return tuple(vector_stores)
auth_contexts: Final = await _vector_store_listing_auth_contexts(user_api_key_dict)
return tuple([vs for vs in vector_stores if await _is_vector_store_granted_to_any(vs, auth_contexts)])
async def get_litellm_managed_vector_store(
vector_store_id: str,
) -> LiteLLM_ManagedVectorStore | None:

24
litellm/rag/main.py
View file

@ -11,7 +11,7 @@ __all__ = ["aingest", "aquery", "ingest", "query"]
import asyncio
import contextvars
from collections.abc import Coroutine, Iterator
from collections.abc import Coroutine, Iterator, Mapping
from contextlib import contextmanager
from functools import partial
from types import MappingProxyType
@ -66,6 +66,10 @@ _FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset(
}
)
_SEARCH_ARGS_SET_BY_PIPELINE: Final = frozenset(
{"vector_store_id", "query", "max_num_results", "custom_llm_provider", "router"}
)
def get_ingestion_class(provider: str) -> type[BaseRAGIngestion]:
"""
@ -225,6 +229,7 @@ async def _execute_query_pipeline(
retrieval_config: dict[str, Any],
rerank: dict[str, Any] | None = None,
stream: bool = False,
vector_store_params: Mapping[str, object] | None = None,
**kwargs,
) -> ModelResponse:
"""
@ -241,11 +246,19 @@ async def _execute_query_pipeline(
# 2. Search vector store
# Forward allowlisted provider retrieval_config extras (region, embedding
# model, bucket, credential refs) to the search call; kwargs win on conflict.
# model, bucket, credential refs) to the search call; the managed store's
# params win on conflict.
provider_search_params: Final = MappingProxyType(
{k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS}
)
forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs})
store_search_params: Final = MappingProxyType(
{
k: v
for k, v in (vector_store_params.items() if vector_store_params else ())
if k not in _SEARCH_ARGS_SET_BY_PIPELINE
}
)
forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs, **store_search_params})
with _suppressed_sub_call_billing():
search_response: Final = await litellm.vector_stores.asearch(
vector_store_id=retrieval_config["vector_store_id"],
@ -339,6 +352,7 @@ async def aquery(
retrieval_config: dict[str, Any],
rerank: dict[str, Any] | None = None,
stream: bool = False,
vector_store_params: Mapping[str, object] | None = None,
**kwargs,
) -> ModelResponse:
"""
@ -356,6 +370,7 @@ async def aquery(
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
vector_store_params=vector_store_params,
**kwargs,
)
@ -386,6 +401,7 @@ def query(
retrieval_config: dict[str, Any],
rerank: dict[str, Any] | None = None,
stream: bool = False,
vector_store_params: Mapping[str, object] | None = None,
**kwargs,
) -> ModelResponse | Coroutine[None, None, ModelResponse]:
"""
@ -402,6 +418,7 @@ def query(
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
vector_store_params=vector_store_params,
**kwargs,
)
else:
@ -412,6 +429,7 @@ def query(
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
vector_store_params=vector_store_params,
**kwargs,
)
)

3
litellm/realtime_api/main.py
View file

@ -8,6 +8,7 @@ from typing import Any, Final, Literal, cast
import litellm
from litellm.constants import (
AZURE_OPENAI_AUDIO_PROVIDERS,
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
request_timeout,
@ -400,7 +401,7 @@ async def _arealtime(
litellm_metadata=_build_litellm_metadata(kwargs),
query_params=query_params,
)
elif _custom_llm_provider == "azure":
elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
# set API KEY
api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_API_KEY")

6
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -562,6 +562,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
hidden_params: Final = getattr(chunk, "_hidden_params", None)
if hidden_params is not None:
chunk_dict["_hidden_params"] = dict(hidden_params) if isinstance(hidden_params, dict) else hidden_params
if (
chunk_dict.get("usage") is None
and isinstance(hidden_params, dict)
and hidden_params.get("usage") is not None
):
chunk_dict["usage"] = hidden_params["usage"]
return chunk_dict
def create_reasoning_summary_text_done_event(

116
litellm/router.py
View file

@ -21,7 +21,7 @@ import time
import traceback
import weakref
from collections import defaultdict
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Iterator, Mapping, Sequence
from functools import lru_cache, partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
@ -117,6 +117,9 @@ from litellm.router_utils.add_retry_fallback_headers import (
from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
classify_strategy_router_model,
count_heuristic_v2_routers,
heuristic_v2_limit_violation,
uses_heuristic_v2_classifier,
)
from litellm.router_utils.batch_utils import (
_get_router_metadata_variable_name,
@ -211,6 +214,7 @@ from litellm.types.router import (
DeploymentTypedDict,
FallbackAccessCheck,
GuardrailTypedDict,
HeuristicV2RouterLimit,
LiteLLM_Params,
MockRouterTestingParams,
ModelGroupInfo,
@ -590,16 +594,20 @@ set_live_deployment_replay(_replay_live_router_model_cost)
# Kwargs that carry no signal about the failed attempt, so log_retry drops them from a
# breadcrumb entirely: the request payload and the router-internal walk state. Credentials are
# handled separately by mask_credentials_in_payload, which scrubs credential-named values from
# whatever kwargs remain rather than trying to enumerate every credential-bearing key here.
# breadcrumb entirely: the request payload, the proxy's snapshot of the inbound request (its body
# aliases the live request metadata, earlier breadcrumbs included, so copying it would nest every
# breadcrumb inside the next one), and the router-internal walk state. Credentials are handled
# separately by mask_credentials_in_payload, which scrubs credential-named values from whatever
# kwargs remain rather than trying to enumerate every credential-bearing key here.
RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(
(
"messages",
"original_function",
"attempted_targets",
"proxy_server_request",
)
)
RETRY_BREADCRUMB_LIMIT: Final = 4
class Router:
@ -683,6 +691,7 @@ class Router:
background_health_check_model_groups: Sequence[str] | None = None,
enable_weighted_failover: bool = False,
fallback_access_check: FallbackAccessCheck | None = None,
heuristic_v2_router_limit: HeuristicV2RouterLimit | None = None,
) -> None:
"""
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
@ -759,6 +768,7 @@ class Router:
self.set_verbose = set_verbose
self.ignore_invalid_deployments = ignore_invalid_deployments
self.heuristic_v2_router_limit = heuristic_v2_router_limit
self.fallback_access_check: Final = fallback_access_check
self.debug_level = debug_level
self.enable_pre_call_checks = enable_pre_call_checks
@ -958,7 +968,6 @@ class Router:
self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model
self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model
self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model
self.previous_models: list = [] # list to store failed calls (passed in as metadata to next call)
# make Router.chat.completions.create compatible for openai.chat.completions.create
default_litellm_params = default_litellm_params or {}
@ -8137,35 +8146,31 @@ class Router:
"""
When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
"""
try:
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
# Log failed model as the previous model
previous_model: Final = {
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var]
attempt_kwargs: Final = MappingProxyType(
{k: v for k, v in kwargs.items() if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS}
)
attempt_metadata: Final = MappingProxyType(
{k: v for k, v in request_metadata.items() if k != "previous_models"}
)
previous_model: Final = MappingProxyType(
{
"exception_type": type(e).__name__,
"exception_string": str(e),
**attempt_kwargs,
_metadata_var: attempt_metadata,
}
for (
k,
v,
) in kwargs.items(): # log everything in kwargs except the old previous_models value - prevent nesting
if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS:
previous_model[k] = v
elif k == _metadata_var and isinstance(v, dict):
previous_model[_metadata_var] = {}
for metadata_k, metadata_v in kwargs[_metadata_var].items():
if metadata_k != "previous_models":
previous_model[k][metadata_k] = metadata_v
# check current size of self.previous_models, if it's larger than 3, remove the first element
if len(self.previous_models) > 3:
self.previous_models.pop(0)
scrubbed_previous_model: Final = mask_credentials_in_payload(previous_model)
self.previous_models.append(scrubbed_previous_model)
kwargs[_metadata_var]["previous_models"] = self.previous_models
return kwargs
except Exception as e:
raise e
)
earlier_breadcrumbs: Final = request_metadata.get("previous_models")
kept_breadcrumbs: Final[tuple[object, ...]] = (
tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :]
if isinstance(earlier_breadcrumbs, (list, tuple))
else ()
)
breadcrumbs: Final = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model))
kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict
return kwargs
def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int:
"""
@ -8796,6 +8801,30 @@ class Router:
"""
return classify_strategy_router_model(litellm_params.model) == "complexity"
def config_deployments(self) -> Iterator[Mapping[str, object]]:
"""The model_list rows that came from config.yaml rather than the DB (``model_info.db_model`` unset)."""
for deployment in self.model_list:
if not isinstance(deployment, Mapping):
continue
model_info = deployment.get("model_info")
if not (isinstance(model_info, Mapping) and model_info.get("db_model")):
yield deployment
def heuristic_v2_router_limit_violation(self) -> str | None:
"""
Why one more heuristic_v2 router cannot join this router, or None when it can.
Judged against every deployment currently on the model_list; an upsert pops the row being
edited first, so an edit of an existing heuristic_v2 router keeps its own slot. The limit is
resolved on every call through ``heuristic_v2_router_limit``; unset means unlimited, which
is the SDK default, and the proxy injects a resolver backed by its license.
"""
limit: Final = self.heuristic_v2_router_limit() if self.heuristic_v2_router_limit is not None else None
others: Final = count_heuristic_v2_routers(
deployment for deployment in self.model_list if isinstance(deployment, Mapping)
)
return heuristic_v2_limit_violation(held=others + 1, limit=limit)
def init_complexity_router_deployment(self, deployment: Deployment):
"""
Initialize the complexity-router deployment.
@ -8813,6 +8842,10 @@ class Router:
)
complexity_router_config: Final[dict | None] = deployment.litellm_params.complexity_router_config
if uses_heuristic_v2_classifier(complexity_router_config):
limit_violation: Final = self.heuristic_v2_router_limit_violation()
if limit_violation is not None:
raise ValueError(limit_violation)
default_model: str | None = deployment.litellm_params.complexity_router_default_model
@ -9636,8 +9669,16 @@ class Router:
raise e
def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None:
"""Put a deployment back the way it was before a failed upsert popped it.
A rollback re-admits state that was already serving, so it does not go through the
heuristic_v2 ceiling a newcomer gets: with the ceiling tightened since the deployment first
registered, judging the rollback would drop a serving router over an unrelated failed edit.
"""
if previous_deployment is None or self.has_model_id(model_id):
return
limit_resolver: Final = self.heuristic_v2_router_limit
self.heuristic_v2_router_limit = None
try:
self.add_deployment(deployment=previous_deployment)
verbose_router_logger.info(
@ -9652,6 +9693,8 @@ class Router:
model_id,
restore_error,
)
finally:
self.heuristic_v2_router_limit = limit_resolver
@staticmethod
def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]:
@ -9981,6 +10024,17 @@ class Router:
coerce_token_limit(model_info.get("max_output_tokens")),
)
def get_configured_mode(self, model_name: str) -> "str | None":
"""Return the mode explicitly configured for a concrete deployment."""
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
if deployment is None:
return None
mode: Final = deployment.model_info.get("mode")
if isinstance(mode, str) and mode.strip():
return mode
return None
def get_configured_display_name(self, model_name: str) -> "str | None":
"""
Return the display_name explicitly configured in a concrete deployment's

59
litellm/router_strategy/auto_router/auto_router.py
View file

@ -2,23 +2,41 @@
Auto-Routing Strategy that works with a Semantic Router Config
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Optional
from pydantic import BaseModel, ConfigDict
from litellm._logging import verbose_router_logger
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.internal_call_metadata import (
effective_turn_off_message_logging,
forwarded_internal_call_metadata,
parent_session_kwargs,
)
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
from semantic_router.routers.base import Route
from litellm.router import Router
from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
from litellm.types.router import PreRoutingHookResponse
else:
Router = Any
PreRoutingHookResponse = Any
Route = Any
SemanticRouter = Any
LiteLLMRouterEncoder = Any
class _CallerMetadata(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
metadata: Mapping[str, object] | None = None
litellm_metadata: Mapping[str, object] | None = None
class AutoRouter(CustomLogger):
@ -50,6 +68,8 @@ class AutoRouter(CustomLogger):
"""
from semantic_router.routers import SemanticRouter
from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
self.auto_router_config_path: str | None = auto_router_config_path
self.auto_router_config: str | None = auto_router_config
self.auto_sync_value = self.DEFAULT_AUTO_SYNC_VALUE
@ -59,6 +79,11 @@ class AutoRouter(CustomLogger):
self.embedding_model: str = embedding_model
self.max_input_chars: int = max_input_chars
self.litellm_router_instance: Router = litellm_router_instance
self.encoder: LiteLLMRouterEncoder = LiteLLMRouterEncoder(
litellm_router_instance=litellm_router_instance,
model_name=embedding_model,
max_input_chars=max_input_chars,
)
def _load_semantic_routing_routes(self) -> list[Route]:
from semantic_router.routers import SemanticRouter
@ -129,9 +154,6 @@ class AutoRouter(CustomLogger):
from semantic_router.routers import SemanticRouter
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
from litellm.router_strategy.auto_router.litellm_encoder import (
LiteLLMRouterEncoder,
)
from litellm.types.router import PreRoutingHookResponse
resolved_messages: Final = (
@ -149,34 +171,47 @@ class AutoRouter(CustomLogger):
#######################
routelayer = SemanticRouter(
routes=self.loaded_routes,
encoder=LiteLLMRouterEncoder(
litellm_router_instance=self.litellm_router_instance,
model_name=self.embedding_model,
max_input_chars=self.max_input_chars,
),
encoder=self.encoder,
auto_sync=self.auto_sync_value,
)
self.routelayer = routelayer
message_content: Final = self._extract_text_from_messages(resolved_messages)
route_name: Final = self._matched_route_name(routelayer, message_content)
route_name: Final = await self._matched_route_name(routelayer, message_content, request_kwargs)
return PreRoutingHookResponse(
model=route_name or self.default_model,
messages=messages,
)
def _matched_route_name(self, routelayer: "SemanticRouter", text: str) -> str | None:
async def _matched_route_name(
self, routelayer: "SemanticRouter", text: str, request_kwargs: Mapping[str, object]
) -> str | None:
"""Name of the route `text` matches, or None when nothing matched or the match failed.
The route layer embeds `text` to compare it against the routes, and that embedding call can
`text` is embedded here rather than by `routelayer(text=...)` so the caller's metadata reaches
`aembedding()` and the embedding's spend lands on the key/team that sent the request;
SemanticRouter has no way to pass kwargs through to its encoder. That embedding call can
fail (context limit, timeout, provider error). Choosing a model is a routing decision, so a
failure here falls back to the default model rather than failing the user's request.
"""
from semantic_router.schema import RouteChoice
try:
route_choice: Final = routelayer(text=text)
caller: Final = _CallerMetadata.model_validate(request_kwargs)
query_vector: Final = (
await self.encoder.aencode_queries(
[text],
metadata=forwarded_internal_call_metadata(caller.metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN),
litellm_metadata=forwarded_internal_call_metadata(
caller.litellm_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN
),
proxy_server_request={"body": {"model": self.embedding_model, "input": [text]}},
turn_off_message_logging=effective_turn_off_message_logging(request_kwargs),
**parent_session_kwargs(request_kwargs),
)
)[0]
route_choice: Final = await routelayer.acall(vector=query_vector)
except Exception as e: # noqa: BLE001 -- the embedding call behind the route layer can fail many ways (context limit, timeout, provider/network error); none of them may fail the request
verbose_router_logger.warning(
"AutoRouter: semantic routing failed (%s), falling back to default model %s", e, self.default_model

5
litellm/router_strategy/lowest_tpm_rpm_v2.py
View file

@ -1,6 +1,7 @@
#### What this does ####
# identifies lowest tpm deployment
import random
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -350,9 +351,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
model_group: str,
healthy_deployments: list,
tpm_keys: list,
tpm_values: list | None,
tpm_values: Sequence | None,
rpm_keys: list,
rpm_values: list | None,
rpm_values: Sequence | None,
messages: list[dict[str, str]] | None = None,
input: str | list | None = None,
) -> dict | None:

34
litellm/router_utils/auto_router_model_naming.py
View file

@ -10,7 +10,7 @@ the router silently dropping the deployment at load time under
``ignore_invalid_deployments``.
"""
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
@ -163,6 +163,38 @@ def strategy_router_dependencies(
)
def uses_heuristic_v2_classifier(complexity_router_config: object) -> bool:
"""Whether this complexity config classifies with the bundled heuristic_v2 model."""
return _mapping(complexity_router_config).get("classifier_type") == "heuristic_v2"
def is_heuristic_v2_router(litellm_params: Mapping[str, object]) -> bool:
"""Whether this deployment is a complexity router that classifies with heuristic_v2."""
return classify_strategy_router_model(str(litellm_params.get("model") or "")) == "complexity" and (
uses_heuristic_v2_classifier(litellm_params.get("complexity_router_config"))
)
def count_heuristic_v2_routers(deployments: Iterable[Mapping[str, object]]) -> int:
"""How many of ``deployments`` (router model_list entries or config.yaml rows) are heuristic_v2 routers."""
return sum(1 for deployment in deployments if is_heuristic_v2_router(_mapping(deployment.get("litellm_params"))))
def heuristic_v2_limit_violation(*, held: int, limit: int | None) -> str | None:
"""Why holding ``held`` heuristic_v2 routers exceeds ``limit``, or None when it fits.
``limit`` None means unlimited. The message is shared by every enforcement point (config
load, model writes, router registration) and stays SDK-neutral: it names the cap and what
the caller can change; the proxy appends how its license lifts the cap.
"""
if limit is None or held <= limit:
return None
return (
f"At most {limit} auto-router(s) with classifier_type 'heuristic_v2' can be registered but this would make "
f"{held}. Use classifier_type 'heuristic' for this router or remove an existing heuristic_v2 router."
)
def validate_complexity_router_config_write(complexity_router_config: Mapping[str, object] | None) -> str | None:
"""Reject a complexity config the router would refuse to build a deployment from.

42
litellm/router_utils/pre_call_checks/deployment_affinity_check.py
View file

@ -82,6 +82,7 @@ class DeploymentAffinityCheck(CustomLogger):
"""
CACHE_KEY_PREFIX = "deployment_affinity:v1"
USER_ID_AFFINITY_PREFIX: Final = "user_id:"
def __init__(
self,
@ -253,15 +254,6 @@ class DeploymentAffinityCheck(CustomLogger):
hashed_user_key: Final = cls._hash_user_key(user_key) if user_key is not None else "unscoped"
return f"{cls.CACHE_KEY_PREFIX}:session:{model_group}:{hashed_user_key}:{session_id}"
@staticmethod
def _get_user_key_from_metadata_dict(metadata: dict) -> str | None:
# NOTE: affinity is keyed on the *API key hash* provided by the proxy (not the
# OpenAI `user` parameter, which is an end-user identifier).
user_key: Final = metadata.get("user_api_key_hash")
if user_key is None:
return None
return str(user_key)
@staticmethod
def _get_session_id_from_metadata_dict(metadata: dict) -> str | None:
session_id: Final = metadata.get("session_id")
@ -285,22 +277,30 @@ class DeploymentAffinityCheck(CustomLogger):
return metadata_dicts
@staticmethod
def _get_user_key_from_request_kwargs(request_kwargs: dict) -> str | None:
def _first_metadata_value(metadata_dicts: Sequence[dict], key: str) -> str | None:
value: Final = next((metadata[key] for metadata in metadata_dicts if metadata.get(key) is not None), None)
return None if value is None else str(value)
@classmethod
def _get_user_key_from_request_kwargs(cls, request_kwargs: dict) -> str | None:
"""
Extract a stable affinity key from request kwargs.
Source (proxy): `metadata.user_api_key_hash`
Source (proxy): `metadata.user_api_key_hash` for virtual-key callers. JWT-authenticated
callers carry no key hash, so their `metadata.user_api_key_user_id` stands in for it,
namespaced under `USER_ID_AFFINITY_PREFIX` so a user id can never alias a key hash.
Note: the OpenAI `user` parameter is an end-user identifier and is intentionally
not used for deployment affinity.
"""
# Check metadata dicts (Proxy usage)
for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs):
user_key = DeploymentAffinityCheck._get_user_key_from_metadata_dict(metadata=metadata)
if user_key is not None:
return user_key
return None
metadata_dicts: Final = cls._iter_metadata_dicts(request_kwargs)
user_api_key_hash: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_hash")
if user_api_key_hash is not None:
return user_api_key_hash
user_id: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_user_id")
if user_id is None:
return None
return f"{cls.USER_ID_AFFINITY_PREFIX}{user_id}"
@staticmethod
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
@ -533,9 +533,9 @@ class DeploymentAffinityCheck(CustomLogger):
return typed_healthy_deployments
verbose_router_logger.debug(
"DeploymentAffinityCheck: api-key affinity hit -> deployment=%s user_key=%s",
"DeploymentAffinityCheck: caller affinity hit -> deployment=%s user_key=%s",
model_id,
self._shorten_for_logs(user_key),
self._shorten_for_logs(self._hash_user_key(user_key)),
)
return [deployment]
@ -626,7 +626,7 @@ class DeploymentAffinityCheck(CustomLogger):
deployment_model_name,
model_id,
self.ttl_seconds,
self._shorten_for_logs(user_key),
self._shorten_for_logs(self._hash_user_key(user_key)),
)
else:
verbose_router_logger.debug(

8
litellm/types/integrations/custom_logger.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final
from pydantic import BaseModel, Field
@ -29,6 +30,13 @@ def is_interception_internal_key(
return any(key.startswith(prefix) for prefix in prefixes)
CONVERTED_STREAM_KEYS: Final = frozenset(f"{prefix}_converted_stream" for prefix in INTERCEPTION_INTERNAL_PREFIXES)
def converted_stream_requested(params: Mapping[str, object]) -> bool:
return any(bool(params.get(key)) for key in CONVERTED_STREAM_KEYS)
class AgenticLoopSafetyError(ValueError):
"""
Raised when an agentic-loop safety rail refuses a rerun.

Some files were not shown because too many files have changed in this diff Show more