scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

chore: merge litellm_internal_staging into litellm_fix_v2_resolver_migration_deadlock

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-02 19:53:03 +00:00
commit aaf59abb1f
256 changed files with 16410 additions and 1368 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/image-scan.yml vendored
View file

@ -80,7 +80,7 @@ jobs:
LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
# Scans the whole shipped artifact: OS/apk plus every language package
# baked into the image, including ones no lockfile declares (e.g. prisma's
@ -124,7 +124,7 @@ jobs:
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
migrations-image:
name: migrations-image
@ -185,7 +185,7 @@ jobs:
LITELLM_COMPONENT_PORT: "4000"
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
ui-image:
name: ui-image

8
Dockerfile
View file

@ -66,6 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
# Copy full source tree
@ -87,6 +88,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@ -101,6 +103,12 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# The base image only configures Chainguard's authenticated apk repo, which
# requires an enterprise subscription. Add the public Wolfi repo so `apk add`
# also works for anyone installing extra packages into a running container.
# https://github.com/BerriAI/litellm/issues/33518
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile

2
backend/Dockerfile
View file

@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
@ -57,6 +58,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

2
basedpyright-code-budget.json
View file

@ -117,7 +117,7 @@
"limit": 111
},
"reportUnnecessaryComparison": {
"limit": 695
"limit": 692
},
"reportUnnecessaryContains": {
"limit": 5

2
docker/Dockerfile.database
View file

@ -64,6 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
# Copy full source tree
@ -85,6 +86,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

3
docker/Dockerfile.non_root
View file

@ -70,6 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
# Copy full source tree
@ -97,6 +98,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13 \
--no-sources-package litellm-proxy-extras; \
else \
@ -106,6 +108,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13; \
fi

2
helm/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.1.2
version: 1.1.3
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

4
helm/litellm-helm/README.md
View file

@ -26,7 +26,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
@ -212,6 +212,8 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
The key is generated once on the first install; later `helm upgrade` runs reuse the
value already in that Secret, so upgrading never rotates the master key.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

6
helm/litellm-helm/templates/secret-masterkey.yaml
View file

@ -1,9 +1,11 @@
{{- if not .Values.masterkeySecretName }}
{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }}
{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "litellm.fullname" . }}-masterkey
name: {{ $secretName }}
data:
masterkey: {{ $masterkey | b64enc }}
type: Opaque

42
helm/litellm-helm/tests/hpa_tests.yaml
View file

@ -1,4 +1,4 @@
suite: "hpa with behavior"
suite: "hpa"
templates:
- hpa.yaml
tests:
@ -23,14 +23,44 @@ tests:
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
---
suite: "hpa without behavior"
templates:
- hpa.yaml
tests:
- it: "does not render behavior when not set"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- isNull: { path: spec.behavior }
- it: "scales on cpu at the documented 60 percent by default"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
- it: "does not scale on memory by default"
set:
autoscaling.enabled: true
asserts:
- lengthEqual: { path: spec.metrics, count: 1 }
- it: "honours an explicit cpu target override"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: 75
asserts:
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
- it: "renders a memory metric only when a memory target is set"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
- it: "renders no hpa when autoscaling is disabled"
asserts:
- hasDocuments: { count: 0 }

47
helm/litellm-helm/tests/masterkey-secret_tests.yaml
View file

@ -15,6 +15,53 @@ tests:
# Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
# but stored as base64 encoded in Kubernetes secret (requirement).
# "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
kubernetesProvider:
scheme:
"v1/Secret":
gvr:
version: "v1"
resource: "secrets"
namespaced: true
objects:
- kind: Secret
apiVersion: v1
metadata:
name: RELEASE-NAME-litellm-masterkey
namespace: NAMESPACE
data:
masterkey: c2stZXhpc3Rpbmcta2V5
asserts:
- equal:
path: data.masterkey
value: c2stZXhpc3Rpbmcta2V5
- it: should let an explicit masterkey value override the one already stored in the cluster
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
masterkey: sk-explicit
kubernetesProvider:
scheme:
"v1/Secret":
gvr:
version: "v1"
resource: "secrets"
namespaced: true
objects:
- kind: Secret
apiVersion: v1
metadata:
name: RELEASE-NAME-litellm-masterkey
namespace: NAMESPACE
data:
masterkey: c2stZXhpc3Rpbmcta2V5
asserts:
- equal:
path: data.masterkey
value: c2stZXhwbGljaXQ=
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:

11
helm/litellm-helm/values.yaml
View file

@ -200,7 +200,16 @@ autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# 60 is the documented recommendation. See "Recommended Machine Specifications"
# in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
# above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
# high enough to trip near saturation adds capacity minutes after it was needed.
targetCPUUtilizationPercentage: 60
# Deliberately left unset rather than given a value. The prisma query engine's
# resident memory is a high-water mark that ratchets to the pod's worst-ever write
# and is never returned, so a memory target reads the largest write a pod ever did
# rather than what it is doing now, and replicas ratchet up without scaling back in.
# Memory is a floor to provision under 'resources', not a signal to scale on.
# targetMemoryUtilizationPercentage: 80
# behavior: {}

30
litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py
View file

@ -14,9 +14,22 @@ then fails on a Node binary that was never written. Deleting a cache directory
that exists without a Node binary is what turns a killed bootstrap back into a
recoverable one.
Both budgets are overridable so an operator can widen them without a release:
``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and
``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command.
``prisma migrate deploy`` is the other command whose runtime is not a
constant: it grows with the number of pending migrations, so a fresh database
that has to replay every migration this package ships overruns a per-command
budget sized for the short bookkeeping commands, on a laptop as much as on a
slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
as separate children, so killing the wrapper on timeout leaves them running:
the retry then contends with that orphan for Prisma's advisory lock and cannot
finish any sooner. Migrate deploy therefore runs under its own budget.
All three budgets are overridable so an operator can widen them without a
release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
``LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT`` for ``prisma migrate deploy`` and
``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. The
per-command budget used to bound migrate deploy as well, so a deployment that
raised it above the deploy default keeps that larger budget for deploy unless
the deploy override says otherwise.
"""
import math
@ -36,10 +49,12 @@ except ImportError:
PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
BOOTSTRAP_ARG = "--version"
@ -88,6 +103,15 @@ def prisma_bootstrap_timeout() -> float:
)
def prisma_migrate_deploy_timeout() -> float:
"""Seconds one ``prisma migrate deploy`` may run for, however many migrations are pending."""
if os.getenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR) is not None:
return _timeout_from_env(
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT
)
return max(DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, prisma_command_timeout())
def nodeenv_cache_dir() -> Optional[Path]:
"""Where Prisma installs its private Node runtime, or None if unknowable."""
override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR)

25
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -15,8 +15,11 @@ from litellm_proxy_extras.replica_identity import (
apply_replica_identity_full,
)
from litellm_proxy_extras.prisma_toolchain import (
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
ensure_prisma_toolchain,
prisma_command_timeout,
prisma_migrate_deploy_timeout,
)
@ -751,12 +754,13 @@ class ProxyExtrasDBManager:
original_dir = os.getcwd()
os.chdir(migrations_dir)
deploy_timeout = prisma_migrate_deploy_timeout()
try:
for attempt in range(4):
try:
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=prisma_command_timeout(),
timeout=deploy_timeout,
check=True,
capture_output=True,
text=True,
@ -766,8 +770,12 @@ class ProxyExtrasDBManager:
return True
except subprocess.TimeoutExpired:
logger.info(
f"prisma migrate deploy attempt {attempt + 1} timed out, retrying"
logger.warning(
"prisma migrate deploy attempt %s timed out after %ss, retrying. "
"Raise %s if this database needs longer to apply its pending migrations.",
attempt + 1,
deploy_timeout,
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
)
time.sleep(random.randrange(5, 15))
continue
@ -914,7 +922,8 @@ class ProxyExtrasDBManager:
"Database migration failed after 4 attempts (retry loop "
"exhausted by timeouts, deadlock retries, or repeated "
"idempotent-recovery continues). Check database connectivity, "
"load, and _prisma_migrations ledger state."
"load, and _prisma_migrations ledger state, and raise "
f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
)
finally:
os.chdir(original_dir)
@ -999,7 +1008,7 @@ class ProxyExtrasDBManager:
# Set migrations directory for Prisma
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=prisma_command_timeout(),
timeout=prisma_migrate_deploy_timeout(),
check=True,
capture_output=True,
text=True,
@ -1217,7 +1226,11 @@ class ProxyExtrasDBManager:
)
return True
except subprocess.TimeoutExpired:
logger.info(f"Attempt {attempt + 1} timed out")
logger.warning(
"Attempt %s timed out. Raise %s if this database needs longer to apply its schema.",
attempt + 1,
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR,
)
time.sleep(random.randrange(5, 15))
except subprocess.CalledProcessError as e:
attempts_left = 3 - attempt

51
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -10,8 +10,19 @@ from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from litellm._logging import verbose_logger
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
A2A_USER_API_KEY_HASH_PARAM,
)
from litellm.a2a_protocol.utils import (
get_session_id_from_a2a_params,
scope_session_to_principal,
)
from litellm.exceptions import BadRequestError
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33
RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256
# Reserved outbound header names that must never be sourced from per-request
# ``agent_extra_headers`` for AgentCore requests. ``agent_extra_headers`` carries
# values rewritten from the client-controlled ``x-a2a-{agent}-*`` convention, so
@ -19,8 +30,9 @@ from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreCo
# request identity / SigV4 metadata by overwriting headers the proxy sets from
# trusted server-side config.
#
# The runtime headers (session / user id) are derived server-side from
# ``runtimeSessionId`` / ``runtimeUserId`` in the agent's ``litellm_params``;
# The runtime headers (session / user id) are derived server-side from the A2A
# ``message.contextId`` and ``runtimeSessionId`` / ``runtimeUserId`` in the
# agent's ``litellm_params``;
# ``authorization`` is set by the AgentCore signer (JWT or SigV4); ``host`` and
# the ``x-amz-*`` family are owned by SigV4 itself.
_RESERVED_EXACT_HEADERS: Final = frozenset(
@ -66,6 +78,31 @@ def _filter_reserved_headers(
return filtered or None
def _request_scoped_runtime_session_id(
params: Mapping[str, Any],
litellm_params: Mapping[str, Any],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
if RUNTIME_SESSION_ID_MIN_LENGTH <= len(session_id) <= RUNTIME_SESSION_ID_MAX_LENGTH:
return session_id
raise BadRequestError(
message=(
f"Invalid AgentCore runtime session id {session_id!r}: AWS requires "
f"{RUNTIME_SESSION_ID_MIN_LENGTH}-{RUNTIME_SESSION_ID_MAX_LENGTH} characters. It is built from the A2A "
"message.contextId (prefixed with a 16-hex-char hash of the calling key and '-') when set, "
"otherwise from the agent's configured runtimeSessionId."
),
model=model,
llm_provider="bedrock",
)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -100,7 +137,9 @@ class BedrockAgentCoreA2ATransformation:
here to prevent a caller-controlled ``x-a2a-{agent}-*`` header from
spoofing the AgentCore runtime user id or other SigV4 metadata. Use
``api_key`` / ``runtimeUserId`` / ``runtimeSessionId`` in litellm_params
(not ``agent_extra_headers``) to override those values.
(not ``agent_extra_headers``) to override those values. The runtime
session id is taken from ``params["message"]["contextId"]`` (scoped to
the calling key) when present, then ``runtimeSessionId``, else generated.
Returns:
Tuple of (url, signed_headers, signed_body_bytes)
@ -139,7 +178,11 @@ class BedrockAgentCoreA2ATransformation:
# Set required AgentCore session headers (normally set by transform_request,
# which we skip because it also builds {"prompt": "..."})
headers: Final[dict] = {}
session_id: Final = agentcore_config._get_runtime_session_id(optional_params)
session_id: Final = _validate_runtime_session_id(
_request_scoped_runtime_session_id(params, litellm_params)
or agentcore_config._get_runtime_session_id(optional_params),
model=model,
)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id: Final = agentcore_config._get_runtime_user_id(optional_params)
if runtime_user_id:

25
litellm/a2a_protocol/utils.py
View file

@ -2,6 +2,8 @@
Utility functions for A2A protocol.
"""
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import litellm
@ -140,6 +142,29 @@ class A2ARequestUtils:
return prompt_tokens, completion_tokens, total_tokens
def get_session_id_from_a2a_params(params: Mapping[str, Any]) -> str | None:
message: Final = params.get("message", {})
if isinstance(message, dict):
return message.get("contextId")
return getattr(message, "contextId", None)
def scope_session_to_principal(session_id: str, principal: str | None) -> str:
"""
Bind a client-supplied A2A contextId to the authenticated principal.
Without this, two distinct keys authorized for the same agent could set the
same contextId and read/append to each other's backend memory. The
principal is hashed (it is already a hashed token) so the raw value is never
sent to the agent backend, while the original contextId is kept as a suffix
for operator-side correlation.
"""
if not principal:
return session_id
principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16]
return f"{principal_prefix}-{session_id}"
# Backwards compatibility aliases
def extract_text_from_a2a_message(message: Any) -> str:
return A2ARequestUtils.extract_text_from_message(message)

34
litellm/constants.py
View file

@ -9,6 +9,38 @@ DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT"
AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
{
"routing_strategy_args",
"routing_strategy",
"routing_groups",
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"max_retries",
"retry_after",
"fallbacks",
"context_window_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
}
)
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
{
"model_list",
"search_tools",
"assistants_config",
"router_general_settings",
"ignore_invalid_deployments",
"fallback_access_check",
}
)
DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))
@ -136,6 +168,7 @@ MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0"
MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0"))
MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0"))
MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0"))
MCP_TOOL_LISTING_MAX_PAGES: Final = 1000
# Allowlist of commands permitted for MCP stdio transport.
# Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation.
@ -1589,6 +1622,7 @@ KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job"
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job"
PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"

20
litellm/experimental_mcp_client/client.py
View file

@ -7,6 +7,7 @@ import base64
import os
from collections.abc import Awaitable, Callable, Generator
from datetime import timedelta
from functools import partial
from importlib import metadata
from typing import Any, Final, TypeVar
@ -47,7 +48,8 @@ from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from litellm._logging import verbose_logger
from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR
from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR, MCP_TOOL_LISTING_TIMEOUT
from litellm.experimental_mcp_client.tools import list_tools_with_pagination
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.types.llms.custom_http import VerifyTypes
from litellm.types.mcp import (
@ -603,17 +605,19 @@ class MCPClient:
"""
verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio")
async def _list_tools_operation(session: ClientSession):
return await session.list_tools()
try:
result: Final = await self.run_with_session(_list_tools_operation, quiet_on_error=raise_on_error)
tool_count: Final = len(result.tools)
tool_names: Final = [tool.name for tool in result.tools]
# A per-server timeout above the global default extends the whole-walk deadline
listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)
tools: Final = await self.run_with_session(
partial(list_tools_with_pagination, listing_deadline=listing_deadline),
quiet_on_error=raise_on_error,
)
tool_count: Final = len(tools)
tool_names: Final = tuple(tool.name for tool in tools)
verbose_logger.info(
"MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names
)
return result.tools
return tools
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_tools was cancelled")
raise

74
litellm/experimental_mcp_client/tools.py
View file

@ -1,14 +1,22 @@
import json
from typing import Final, Literal
import anyio
from mcp import ClientSession
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import PaginatedRequestParams
from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from litellm._logging import verbose_logger
from litellm.constants import (
MCP_CLIENT_TIMEOUT,
MCP_TOOL_LISTING_MAX_PAGES,
MCP_TOOL_LISTING_TIMEOUT,
)
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
@ -90,6 +98,64 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages
)
async def list_tools_with_pagination(
session: ClientSession, listing_deadline: float | None = None
) -> list[MCPTool]: # mutable-ok: list return contract
"""Collect tools from every tools/list page by following nextCursor.
Stops and returns the tools collected so far when the upstream repeats a
cursor, the page cap is reached, or the whole-walk deadline expires, so a
buggy or slow upstream yields a partial catalog instead of an error.
listing_deadline overrides the default whole-walk deadline; callers with a
per-server timeout above the global default pass it through here.
"""
tools: Final[list[MCPTool]] = [] # mutable-ok: accumulates each page's tools
seen_cursors: Final[set[str]] = set() # mutable-ok: guards against cursor loops
cursor: str | None = None # rebind-ok: advances to each page's nextCursor
# The per-request session read timeout restarts on every page, so a multi-page
# walk needs its own overall deadline. max() keeps the pre-pagination guarantee
# that a single page slower than the listing timeout but within the client
# timeout still succeeds.
effective_deadline: Final = (
listing_deadline if listing_deadline is not None else max(MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT)
)
with anyio.move_on_after(effective_deadline):
for _ in range(MCP_TOOL_LISTING_MAX_PAGES):
result = (
await session.list_tools()
if cursor is None
else await session.list_tools(params=PaginatedRequestParams(cursor=cursor))
)
tools.extend(result.tools)
next_cursor = getattr(result, "nextCursor", None)
if not isinstance(next_cursor, str) or not next_cursor:
return tools
if next_cursor in seen_cursors:
verbose_logger.warning(
"MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far",
len(tools),
)
return tools
seen_cursors.add(next_cursor)
cursor = next_cursor
verbose_logger.warning(
"MCP server tools/list pagination exceeded the maximum of %s pages; returning %s tools collected so far",
MCP_TOOL_LISTING_MAX_PAGES,
len(tools),
)
return tools
verbose_logger.warning(
"MCP server tools/list pagination exceeded the %s second listing deadline; returning %s tools collected so far",
effective_deadline,
len(tools),
)
return tools
async def load_mcp_tools(
session: ClientSession, format: Literal["mcp", "openai"] = "mcp"
) -> list[MCPTool] | list[ChatCompletionToolParam]:
@ -103,10 +169,12 @@ async def load_mcp_tools(
If format is set to "openai", the tools are converted to OpenAI API compatible tools.
"""
tools: Final = await session.list_tools()
tools: Final = await list_tools_with_pagination(session)
if format == "openai":
return [transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools.tools]
return tools.tools
return [ # mutable-ok: public API returns a list
transform_mcp_tool_to_openai_tool(mcp_tool=tool) for tool in tools
]
return tools
########################################################

64
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -68,6 +68,7 @@ from .utils import process_slack_alerting_variables
if TYPE_CHECKING:
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
from litellm.proxy.utils import PrismaClient
from litellm.router import Router as _Router
Router = _Router
@ -1944,6 +1945,69 @@ Model Info:
except Exception as e:
verbose_proxy_logger.exception("Error sending weekly spend report %s", e)
async def send_user_spend_alerts(self, prisma_client: "PrismaClient | None" = None) -> None:
"""Check per-user daily/monthly spend thresholds and spend anomalies, alerting once per user per period."""
if self.alerting is None or "slack" not in self.alerting:
return
thresholds_enabled: Final = AlertType.user_spend_thresholds in self.alert_types
anomalies_enabled: Final = AlertType.user_spend_anomalies in self.alert_types
if not thresholds_enabled and not anomalies_enabled:
return
if prisma_client is None:
from litellm.proxy.proxy_server import prisma_client as global_prisma_client
prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client
if prisma_client is None:
return
from litellm.integrations.SlackAlerting.user_spend_alerts import (
evaluate_user_spend,
fetch_user_spend_rows,
)
try:
today: Final = datetime.datetime.now(datetime.timezone.utc).date()
rows: Final = await fetch_user_spend_rows(
prisma_client=prisma_client,
today=today,
baseline_days=self.alerting_args.spend_anomaly_baseline_days,
)
all_events: Final = tuple(
event
for row in rows
for event in evaluate_user_spend(
row=row,
args=self.alerting_args,
today=today,
thresholds_enabled=thresholds_enabled,
anomalies_enabled=anomalies_enabled,
)
)
cached_flags: Final = await asyncio.gather(
*(self.internal_usage_cache.async_get_cache(key=event.cache_key) for event in all_events)
)
new_events: Final = tuple(event for event, cached in zip(all_events, cached_flags) if not cached)
for alert_type in (AlertType.user_spend_thresholds, AlertType.user_spend_anomalies):
typed_events = tuple(event for event in new_events if event.alert_type == alert_type)
if not typed_events:
continue
await self.send_alert(
message="\n\n".join(event.message for event in typed_events),
level="High",
alert_type=alert_type,
alerting_metadata={}, # mutable-ok: send_alert takes a dict payload
)
for event in typed_events:
await self.internal_usage_cache.async_set_cache(
key=event.cache_key,
value="SENT",
ttl=event.cache_ttl,
)
except Exception as e: # noqa: BLE001 # background job must not crash the scheduler
verbose_proxy_logger.exception("Error sending user spend alerts: %s", e)
async def send_fallback_stats_from_prometheus(self):
"""
Helper to send fallback statistics from prometheus server -> to slack

139
litellm/integrations/SlackAlerting/user_spend_alerts.py Normal file
View file

@ -0,0 +1,139 @@
"""Per-user daily/monthly spend threshold alerts and spend anomaly detection."""
import datetime
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, Literal
from pydantic import TypeAdapter
from litellm.constants import HOURS_IN_A_DAY
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
DAY_SECONDS: Final = HOURS_IN_A_DAY * 60 * 60
MONTHLY_ALERT_TTL_SECONDS: Final = 32 * DAY_SECONDS
USER_SPEND_QUERY: Final = """
SELECT
user_id,
COALESCE(SUM(spend) FILTER (WHERE date = $1), 0)::float AS daily_spend,
COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0)::float AS monthly_spend,
COALESCE(SUM(spend) FILTER (WHERE date >= $3 AND date < $1), 0)::float AS baseline_spend
FROM "LiteLLM_DailyUserSpend"
WHERE date >= LEAST($2, $3) AND user_id IS NOT NULL
GROUP BY user_id
HAVING COALESCE(SUM(spend) FILTER (WHERE date >= $2), 0) > 0
"""
@dataclass(frozen=True, slots=True)
class UserSpendRow:
user_id: str
daily_spend: float
monthly_spend: float
baseline_spend: float
@dataclass(frozen=True, slots=True)
class UserSpendAlertEvent:
kind: Literal["daily_threshold", "monthly_threshold", "anomaly"]
alert_type: AlertType
message: str
cache_key: str
cache_ttl: int
USER_SPEND_ROWS_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
async def fetch_user_spend_rows(
prisma_client: "PrismaClient",
today: datetime.date,
baseline_days: int,
) -> tuple[UserSpendRow, ...]:
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
threshold: Final = args.daily_spend_per_user_threshold
if threshold is None or row.daily_spend < threshold:
return None
return UserSpendAlertEvent(
kind="daily_threshold",
alert_type=AlertType.user_spend_thresholds,
message=(
f"User Daily Spend Threshold Crossed:\n"
f"User: `{row.user_id}`\n"
f"Spend Today: `${row.daily_spend:.2f}`\n"
f"Daily Threshold: `${threshold:.2f}`"
),
cache_key=f"user_spend_alert_daily_{row.user_id}_{today_str}",
cache_ttl=DAY_SECONDS,
)
def _monthly_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, month_str: str) -> UserSpendAlertEvent | None:
threshold: Final = args.monthly_spend_per_user_threshold
if threshold is None or row.monthly_spend < threshold:
return None
return UserSpendAlertEvent(
kind="monthly_threshold",
alert_type=AlertType.user_spend_thresholds,
message=(
f"User Monthly Spend Threshold Crossed:\n"
f"User: `{row.user_id}`\n"
f"Spend This Month: `${row.monthly_spend:.2f}`\n"
f"Monthly Threshold: `${threshold:.2f}`"
),
cache_key=f"user_spend_alert_monthly_{row.user_id}_{month_str}",
cache_ttl=MONTHLY_ALERT_TTL_SECONDS,
)
def _anomaly_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
if row.daily_spend < args.spend_anomaly_min_spend:
return None
baseline_daily_avg: Final = row.baseline_spend / args.spend_anomaly_baseline_days
if row.baseline_spend > 0 and row.daily_spend <= args.spend_anomaly_multiplier * baseline_daily_avg:
return None
return UserSpendAlertEvent(
kind="anomaly",
alert_type=AlertType.user_spend_anomalies,
message=(
f"User Spend Anomaly Detected:\n"
f"User: `{row.user_id}`\n"
f"Spend Today: `${row.daily_spend:.2f}`\n"
f"Daily Average (last {args.spend_anomaly_baseline_days} days): `${baseline_daily_avg:.2f}`\n"
f"Trigger: spend above `{args.spend_anomaly_multiplier}x` the daily average "
f"(minimum `${args.spend_anomaly_min_spend:.2f}`)"
),
cache_key=f"user_spend_alert_anomaly_{row.user_id}_{today_str}",
cache_ttl=DAY_SECONDS,
)
def evaluate_user_spend(
row: UserSpendRow,
args: SlackAlertingArgs,
today: datetime.date,
thresholds_enabled: bool,
anomalies_enabled: bool,
) -> tuple[UserSpendAlertEvent, ...]:
today_str: Final = today.strftime("%Y-%m-%d")
month_str: Final = today.strftime("%Y-%m")
threshold_events: Final = (
(
_daily_threshold_event(row=row, args=args, today_str=today_str),
_monthly_threshold_event(row=row, args=args, month_str=month_str),
)
if thresholds_enabled
else ()
)
anomaly_events: Final = (_anomaly_event(row=row, args=args, today_str=today_str),) if anomalies_enabled else ()
return tuple(event for event in (*threshold_events, *anomaly_events) if event is not None)

65
litellm/integrations/custom_guardrail.py
View file

@ -1,4 +1,5 @@
import contextvars
import copy
import hashlib
import os
import secrets
@ -39,6 +40,7 @@ except ImportError:
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
dc: Final = DualCache()
@ -852,6 +854,69 @@ class CustomGuardrail(CustomLogger):
return result
async def async_logging_hook(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
result: object,
call_type: str,
) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract
"""logging_only: run apply_guardrail on copies of the logged request/response and record the verdict."""
from litellm.llms import get_guardrail_translation_mapping
if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
return kwargs, result
try:
translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))()
except ValueError:
verbose_logger.debug(
"Guardrail %s: no guardrail translation for call_type=%s, skipping logging_only scan",
self.guardrail_name,
call_type,
)
return kwargs, result
litellm_params: Final = kwargs.get("litellm_params") or {}
scratch_metadata: Final = {
key: value
for key, value in (litellm_params.get("metadata") or {}).items()
if key != "standard_logging_guardrail_information"
}
try:
await self._scan_logged_call(kwargs, result, translation, scratch_metadata)
except Exception as e:
verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e)
recorded: Final = scratch_metadata.get("standard_logging_guardrail_information")
standard_logging_object: Final = kwargs.get("standard_logging_object")
if not recorded or not isinstance(standard_logging_object, dict):
return kwargs, result
entries: Final = recorded if isinstance(recorded, list) else [recorded]
existing: Final = standard_logging_object.get("guardrail_information") or []
return {
**kwargs,
"standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]},
}, result
async def _scan_logged_call(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
result: object,
translation: "BaseTranslation",
scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata
) -> None:
optional_params: Final = kwargs.get("optional_params") or {}
scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input"))
scratch_request: Final = {
"model": kwargs.get("model"),
"messages": scratch_input,
"input": scratch_input,
"tools": copy.deepcopy(optional_params.get("tools")),
"litellm_call_id": kwargs.get("litellm_call_id"),
"metadata": scratch_metadata,
}
await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
await translation.process_output_response(
response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request
)
def supports_scan_only_tool_results(self) -> bool:
"""Whether this guardrail can scan tool-result content.

393
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -11,6 +11,7 @@ import json
import os
from collections.abc import Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import Any, Final, Literal
import httpx
@ -30,12 +31,16 @@ from litellm.integrations.datadog.datadog_mock_client import (
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
handle_any_messages_to_chat_completion_str_messages_conversion,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import (
CallTypes,
@ -44,6 +49,189 @@ from litellm.types.utils import (
StandardLoggingPayloadErrorInformation,
)
_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({})
_EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""}
_MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024
def _mapping_field(source: Mapping[str, Any], key: str) -> Mapping[str, Any]:
"""The value at `key` when it is a mapping, else an empty one."""
value: Final = source.get(key)
return value if isinstance(value, dict) else _EMPTY_MAPPING
def _content_blocks(message: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
content: Final = message.get("content")
if not isinstance(content, list):
return ()
return tuple(block for block in content if isinstance(block, dict))
def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str:
"""
Arguments as the object LLM Obs types them as, or the raw string when they are not one.
Strings past the size bound ship unparsed: decoding multiplies memory on hostile compact
JSON, and the raw string is what the intake receives either way.
"""
if not isinstance(raw_arguments, str):
return raw_arguments if isinstance(raw_arguments, dict) else str(raw_arguments)
if len(raw_arguments) > _MAX_PARSED_TOOL_ARGUMENT_CHARS:
return raw_arguments
parsed: Final = safe_json_loads(raw_arguments)
return parsed if isinstance(parsed, dict) else raw_arguments
def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]:
"""
The tool calls a message carries, in LLM Obs' ToolCall schema, from either dialect.
OpenAI puts them in `tool_calls` with the callee nested under `function` and `arguments`
serialized; Anthropic puts them in `content` as `tool_use` blocks with `input` already an
object. LLM Obs reads `name` / `arguments` / `tool_id` either way.
"""
raw_tool_calls: Final = message.get("tool_calls")
openai_calls: Final = tuple(
ToolCall(
name=function.get("name", ""),
arguments=_to_dd_arguments(function.get("arguments", "")),
tool_id=tool_call.get("id", ""),
type=tool_call.get("type", "function"),
)
for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
if isinstance(tool_call, dict)
for function in [_mapping_field(tool_call, "function")]
)
anthropic_calls: Final = tuple(
ToolCall(
name=block.get("name", ""),
arguments=_to_dd_arguments(block.get("input") or {}),
tool_id=block.get("id", ""),
type="tool_use",
)
for block in _content_blocks(message)
if block.get("type") == "tool_use"
)
return openai_calls + anthropic_calls
def _to_dd_tool_results(message: Mapping[str, Any], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]:
"""
The tool results a message carries, linked back to the call each answers.
OpenAI models a result as a whole `role: "tool"` message keyed by `tool_call_id`;
Anthropic nests `tool_result` blocks inside a user message, keyed by `tool_use_id`.
"""
def to_result(tool_id: str, result: object) -> ToolResult:
return ToolResult(
name=tool_call_names.get(tool_id, ""),
result=result if isinstance(result, str) else safe_dumps(result),
tool_id=tool_id,
type="function",
)
if message.get("role") == "tool":
return (to_result(str(message.get("tool_call_id", "")), message.get("content") or ""),)
return tuple(
to_result(str(block.get("tool_use_id", "")), block.get("content") or "")
for block in _content_blocks(message)
if block.get("type") == "tool_result"
)
def _tool_call_names_by_id(messages: Sequence[object]) -> Mapping[str, str]:
"""Ids to tool names for result linking; reads names structurally and parses nothing."""
openai_pairs: Final = tuple(
(tool_call.get("id"), function.get("name", ""))
for message in messages
if isinstance(message, dict) and isinstance(message.get("tool_calls"), list)
for tool_call in message["tool_calls"]
if isinstance(tool_call, dict)
for function in [_mapping_field(tool_call, "function")]
)
anthropic_pairs: Final = tuple(
(block.get("id"), block.get("name", ""))
for message in messages
if isinstance(message, dict)
for block in _content_blocks(message)
if block.get("type") == "tool_use"
)
return MappingProxyType({str(tool_id): str(name) for tool_id, name in openai_pairs + anthropic_pairs if tool_id})
def _to_dd_message(message: object, tool_call_names: Mapping[str, str]) -> Message:
"""
Map one chat message onto LLM Obs' Message schema, adding fields and never destroying content.
Content collapses to its text only when it has text; a content list with none (tool blocks,
images) rides along unchanged so nothing the caller logged is lost. Tool calls and results
move into the fields the LLM Obs Tools panel reads, from both the OpenAI and Anthropic shapes.
"""
if not isinstance(message, dict):
converted: Final = handle_any_messages_to_chat_completion_str_messages_conversion(message)
return converted[0] if converted else _EMPTY_MESSAGE
text: Final = convert_content_list_to_str(message) # pyright: ignore[reportArgumentType] # caller-supplied dict
original_content: Final = message.get("content")
content: Final = (
text if text or not isinstance(original_content, list) or not original_content else original_content
)
reasoning: Final = message.get("reasoning_content")
tool_calls: Final = _to_dd_tool_calls(message)
tool_results: Final = _to_dd_tool_results(message, tool_call_names)
dd_message: Final[Message] = {
"role": message.get("role", ""),
"content": content,
**({"reasoning_content": reasoning} if reasoning is not None else {}),
**({"tool_calls": tool_calls} if tool_calls else {}),
**({"tool_results": tool_results} if tool_results else {}),
}
return dd_message
def _to_dd_messages(messages: object) -> tuple[Message, ...]:
"""Map a whole conversation, resolving each tool result against the calls that precede it."""
if messages is None:
return ()
if not isinstance(messages, list):
return tuple(handle_any_messages_to_chat_completion_str_messages_conversion(messages))
tool_call_names: Final = _tool_call_names_by_id(messages)
return tuple(_to_dd_message(message, tool_call_names) for message in messages)
def _to_dd_tool_definition(entry: Mapping[str, Any]) -> ToolDefinition | None:
function: Final = entry.get("function")
declared: Final[Mapping[str, Any]] = function if isinstance(function, dict) else entry
name: Final = declared.get("name")
if not name:
return None
schema: Final = declared.get("parameters") or declared.get("input_schema")
description: Final = declared.get("description", "")
if not isinstance(schema, dict):
return ToolDefinition(name=name, description=description)
return ToolDefinition(name=name, description=description, schema=schema)
def _to_dd_tool_definitions(model_parameters: object) -> tuple[ToolDefinition, ...]:
"""
Map the request's declared tools onto LLM Obs' ToolDefinition schema.
Handles the wrapped chat-completions shape and the bare shape the Anthropic and
Responses surfaces use, since both reach this logger through `model_parameters`.
"""
if not isinstance(model_parameters, dict):
return ()
raw_tools: Final = model_parameters.get("tools") or model_parameters.get("functions")
if not isinstance(raw_tools, list):
return ()
return tuple(
definition
for entry in raw_tools
if isinstance(entry, dict)
if (definition := _to_dd_tool_definition(entry)) is not None
)
class DataDogLLMObsLogger(CustomBatchLogger):
def __init__(self, **kwargs):
@ -222,12 +410,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise Exception("DataDogLLMObs: standard_logging_object is not set")
messages = standard_logging_payload["messages"]
messages = self._ensure_string_content(messages=messages)
metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {})
input_meta: Final = InputMeta(messages=handle_any_messages_to_chat_completion_str_messages_conversion(messages))
input_meta: Final = InputMeta(messages=_to_dd_messages(standard_logging_payload["messages"]))
output_meta: Final = OutputMeta(
messages=self._get_response_messages(
standard_logging_payload=standard_logging_payload,
@ -241,22 +426,20 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if isinstance(metadata, dict):
metadata_parent_id = metadata.get("parent_id")
meta: Final = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type"), metadata_parent_id),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
error=error_info,
)
tool_definitions: Final = _to_dd_tool_definitions(standard_logging_payload.get("model_parameters"))
span_kind: Final = self._get_datadog_span_kind(standard_logging_payload.get("call_type"), metadata_parent_id)
payload_metadata: Final = self._get_dd_llm_obs_payload_metadata(standard_logging_payload)
# Calculate metrics (you may need to adjust these based on available data)
metrics: Final = LLMMetrics(
input_tokens=float(standard_logging_payload.get("prompt_tokens", 0)),
output_tokens=float(standard_logging_payload.get("completion_tokens", 0)),
total_tokens=float(standard_logging_payload.get("total_tokens", 0)),
total_cost=float(standard_logging_payload.get("response_cost", 0)),
time_to_first_token=self._get_time_to_first_token_seconds(standard_logging_payload),
)
meta: Final[Meta] = {
"kind": span_kind,
"input": input_meta,
"output": output_meta,
"metadata": payload_metadata,
"error": error_info,
**({"tool_definitions": tool_definitions} if tool_definitions else {}),
}
metrics: Final = self._assemble_metrics(standard_logging_payload)
payload: Final[LLMObsPayload] = LLMObsPayload(
parent_id=metadata_parent_id if metadata_parent_id else "undefined",
@ -314,6 +497,45 @@ class DataDogLLMObsLogger(CustomBatchLogger):
)
return error_info
def _assemble_metrics(self, standard_logging_payload: StandardLoggingPayload) -> LLMMetrics:
"""
Build the span metrics, including the prompt-cache counts LLM Obs charts cache savings from.
Cache counts resolve through the same owners the savings dashboard uses, so every provider
spelling is covered, and `non_cached_input_tokens` subtracts BOTH cache categories because
litellm's normalized prompt count includes both (the invariant the cost calculator's custom
pricing helper documents). A zero residual on a fully cached request is real data and is
emitted; a zero read or write count is absence and is not.
"""
prompt_tokens: Final = float(standard_logging_payload.get("prompt_tokens", 0))
completion_tokens: Final = float(standard_logging_payload.get("completion_tokens", 0))
total_tokens: Final = float(standard_logging_payload.get("total_tokens", 0))
total_cost: Final = float(standard_logging_payload.get("response_cost", 0))
time_to_first_token: Final = self._get_time_to_first_token_seconds(standard_logging_payload)
raw_usage: Final = (standard_logging_payload.get("metadata") or {}).get("usage_object")
usage_object: Final = raw_usage if isinstance(raw_usage, dict) else None
cache_read: Final = float(extract_cache_read_tokens(usage_object))
cache_write: Final = float(extract_cache_creation_tokens(usage_object))
metrics: Final[LLMMetrics] = {
"input_tokens": prompt_tokens,
"output_tokens": completion_tokens,
"total_tokens": total_tokens,
"total_cost": total_cost,
"time_to_first_token": time_to_first_token,
**(
{
**({"cache_read_input_tokens": cache_read} if cache_read else {}),
**({"cache_write_input_tokens": cache_write} if cache_write else {}),
"non_cached_input_tokens": max(prompt_tokens - cache_read - cache_write, 0.0),
}
if cache_read or cache_write
else {}
),
}
return metrics
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
"""
Get the time to first token in seconds
@ -335,7 +557,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
def _get_response_messages(
self, standard_logging_payload: StandardLoggingPayload, call_type: str | None
) -> list[object]:
) -> tuple[Message, ...]:
"""
Get the messages from the response object
@ -344,7 +566,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
response_obj = standard_logging_payload.get("response")
if response_obj is None:
return []
return ()
# edge case: handle response_obj is a string representation of a dict
if isinstance(response_obj, str):
@ -357,7 +579,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
# fallback to json parsing
response_obj = json.loads(str(response_obj))
except json.JSONDecodeError:
return []
return ()
if call_type in [
CallTypes.completion.value,
@ -375,12 +597,12 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if isinstance(response_obj, dict) and "choices" in response_obj:
choices: Final = response_obj["choices"]
if choices and len(choices) > 0 and "message" in choices[0]:
return [choices[0]["message"]]
return []
return _to_dd_messages([choices[0]["message"]])
return ()
except (KeyError, IndexError, TypeError):
# In case of any error accessing the response structure, return empty list
return []
return []
return ()
return ()
def _get_datadog_span_kind(
self, call_type: str | None, parent_id: str | None = None
@ -485,17 +707,6 @@ class DataDogLLMObsLogger(CustomBatchLogger):
# Default fallback for unknown or passthrough operations
return "llm"
def _ensure_string_content(self, messages: str | Sequence[object] | Mapping[object, object] | None) -> list[object]:
if messages is None:
return []
if isinstance(messages, str):
return [messages]
elif isinstance(messages, list):
return [message for message in messages]
elif isinstance(messages, dict):
return [str(messages.get("content", ""))]
return []
def _get_dd_llm_obs_payload_metadata(self, standard_logging_payload: StandardLoggingPayload) -> dict[str, object]:
"""
Fields to track in DD LLM Observability metadata from litellm standard logging payload
@ -524,10 +735,6 @@ class DataDogLLMObsLogger(CustomBatchLogger):
spend_metrics: Final = self._get_spend_metrics(standard_logging_payload)
_metadata.update({"spend_metrics": dict(spend_metrics)})
## extract tool calls and add to metadata
tool_call_metadata: Final = self._extract_tool_call_metadata(standard_logging_payload)
_metadata.update(tool_call_metadata)
_standard_logging_metadata: Final[dict] = dict(standard_logging_payload.get("metadata", {})) or {}
_metadata.update(_standard_logging_metadata)
return _metadata
@ -647,107 +854,3 @@ class DataDogLLMObsLogger(CustomBatchLogger):
verbose_logger.debug("Original value: %s", user_api_key_budget_reset_at)
return spend_metrics
def _process_input_messages_preserving_tool_calls(self, messages: Sequence[object]) -> list[dict[str, object]]:
"""
Process input messages while preserving tool_calls and tool message types.
This bypasses the lossy string conversion when tool calls are present,
allowing complex nested tool_calls objects to be preserved for Datadog.
"""
processed: Final = []
for msg in messages:
if isinstance(msg, dict):
# Preserve messages with tool_calls or tool role as-is
if "tool_calls" in msg or msg.get("role") == "tool":
processed.append(msg)
else:
# For regular messages, still apply string conversion
converted = handle_any_messages_to_chat_completion_str_messages_conversion([msg])
processed.extend(converted)
else:
# For non-dict messages, apply string conversion
converted = handle_any_messages_to_chat_completion_str_messages_conversion([msg])
processed.extend(converted)
return processed
@staticmethod
def _tool_calls_kv_pair(tool_calls: list[dict[str, Any]]) -> dict[str, object]:
"""
Extract tool call information into key-value pairs for Datadog metadata.
Similar to OpenTelemetry's implementation but adapted for Datadog's format.
"""
kv_pairs: Final[dict[str, object]] = {}
for idx, tool_call in enumerate(tool_calls):
try:
# Extract tool call ID
tool_id = tool_call.get("id")
if tool_id:
kv_pairs[f"tool_calls.{idx}.id"] = tool_id
# Extract tool call type
tool_type = tool_call.get("type")
if tool_type:
kv_pairs[f"tool_calls.{idx}.type"] = tool_type
# Extract function information
function = tool_call.get("function")
if function:
function_name = function.get("name")
if function_name:
kv_pairs[f"tool_calls.{idx}.function.name"] = function_name
function_arguments = function.get("arguments")
if function_arguments:
# Store arguments as JSON string for Datadog
if isinstance(function_arguments, str):
kv_pairs[f"tool_calls.{idx}.function.arguments"] = function_arguments
else:
import json
kv_pairs[f"tool_calls.{idx}.function.arguments"] = json.dumps(function_arguments)
except (KeyError, TypeError, ValueError) as e:
verbose_logger.debug("DataDogLLMObs: Error processing tool call %s: %s", idx, e)
continue
return kv_pairs
def _extract_tool_call_metadata(self, standard_logging_payload: StandardLoggingPayload) -> dict[str, object]:
"""
Extract tool call information from both input messages and response for Datadog metadata.
"""
tool_call_metadata: Final[dict[str, object]] = {}
try:
# Extract tool calls from input messages
messages: Final = standard_logging_payload.get("messages", [])
if messages and isinstance(messages, list):
for message in messages:
if isinstance(message, dict) and "tool_calls" in message:
tool_calls = message.get("tool_calls")
if tool_calls:
input_tool_calls_kv = self._tool_calls_kv_pair(tool_calls)
# Prefix with "input_" to distinguish from response tool calls
for key, value in input_tool_calls_kv.items():
tool_call_metadata[f"input_{key}"] = value
# Extract tool calls from response
response_obj: Final = standard_logging_payload.get("response")
if response_obj and isinstance(response_obj, dict):
choices: Final = response_obj.get("choices", [])
for choice in choices:
if isinstance(choice, dict):
message = choice.get("message")
if message and isinstance(message, dict):
tool_calls = message.get("tool_calls")
if tool_calls:
response_tool_calls_kv = self._tool_calls_kv_pair(tool_calls)
# Prefix with "output_" to distinguish from input tool calls
for key, value in response_tool_calls_kv.items():
tool_call_metadata[f"output_{key}"] = value
except Exception as e:
verbose_logger.debug("DataDogLLMObs: Error extracting tool call metadata: %s", e)
return tool_call_metadata

43
litellm/integrations/otel/model/payloads.py
View file

@ -6,6 +6,7 @@ import json
from collections.abc import Mapping
from dataclasses import dataclass, field
from enum import Enum
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, cast
from urllib.parse import urlsplit
@ -62,6 +63,31 @@ if TYPE_CHECKING:
# --- typed sub-structures ---------------------------------------------------- #
def _cache_token_value(*values: object) -> int | None:
explicit_zero = False
invalid_before_zero = False
for raw_value in values:
if raw_value is None:
continue
if isinstance(raw_value, bool):
parsed = None
else:
try:
parsed = as_int(raw_value)
except (OverflowError, ValueError):
parsed = None
if parsed is None:
if not explicit_zero:
invalid_before_zero = True
elif parsed > 0:
return parsed
elif parsed == 0:
explicit_zero = True
elif not explicit_zero:
invalid_before_zero = True
return 0 if explicit_zero and not invalid_before_zero else None
@dataclass(frozen=True)
class LLMRequestParams:
temperature: float | None = None
@ -104,12 +130,25 @@ class LLMUsage:
metadata: Final[Mapping[str, object]] = payload.get("metadata") or {}
raw_usage: Final = metadata.get("usage_object")
usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {}
raw_details: Final = usage_object.get("prompt_tokens_details")
prompt_details: Final[Mapping[str, object]] = (
raw_details if isinstance(raw_details, Mapping) else MappingProxyType({})
)
return cls(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")),
cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")),
cache_creation_input_tokens=_cache_token_value(
usage_object.get("cache_creation_input_tokens"),
prompt_details.get("cache_write_tokens"),
prompt_details.get("cache_creation_tokens"),
prompt_details.get("cache_creation_input_tokens"),
),
cache_read_input_tokens=_cache_token_value(
usage_object.get("cache_read_input_tokens"),
prompt_details.get("cached_tokens"),
usage_object.get("prompt_cache_hit_tokens"),
),
)

172
litellm/integrations/prometheus.py
View file

@ -8,6 +8,7 @@ import math
import os
import sys
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast
@ -58,7 +59,10 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from prometheus_client import Gauge
from prometheus_client.metrics import MetricWrapperBase
from litellm.router import Router
else:
AsyncIOScheduler = Any
@ -67,6 +71,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel)
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0
UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other"
_NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset(
(
"guardrail_name",
@ -154,6 +160,44 @@ def _get_budget_metrics_per_request_timeout() -> float:
return parsed
def _get_proxy_llm_router() -> Router | None:
try:
from litellm.proxy.proxy_server import llm_router
except Exception:
return None
return llm_router
def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None:
"""
Bound ``requested_model`` label cardinality: names the router recognizes
(model names, deployment ids, aliases, routing groups, team public model
names) or matches via a global or team wildcard/pattern route keep their
own label value; any other client-supplied string collapses into the
single ``other`` bucket. With no proxy router to vouch for the string,
client-supplied values collapse to ``other`` while ``router_originated``
values (emitted by an SDK ``Router``'s own deployment failure and
fallback events, where the proxy router never exists) pass through.
"""
if not requested_model:
return requested_model
llm_router: Final = _get_proxy_llm_router()
if llm_router is None:
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
if llm_router.is_recognized_model(requested_model):
return requested_model
if requested_model in llm_router.team_public_model_names:
return requested_model
if llm_router.pattern_router.route(requested_model) is not None:
return requested_model
if any(
team_pattern_router.route(requested_model) is not None
for team_pattern_router in llm_router.team_pattern_routers.values()
):
return requested_model
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -434,6 +478,30 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_remaining_api_key_tokens_for_model"),
)
self.litellm_api_key_rate_limit_allowed_metric = self._gauge_factory(
"litellm_api_key_rate_limit_allowed_metric",
"Configured rate limit for the API Key in the current window (rpm_limit / tpm_limit), by rate_limit_type",
labelnames=self.get_labels_for_metric("litellm_api_key_rate_limit_allowed_metric"),
)
self.litellm_api_key_rate_limit_used_metric = self._gauge_factory(
"litellm_api_key_rate_limit_used_metric",
"Requests or tokens the API Key has consumed in the current rate limit window, by rate_limit_type",
labelnames=self.get_labels_for_metric("litellm_api_key_rate_limit_used_metric"),
)
self.litellm_team_rate_limit_allowed_metric = self._gauge_factory(
"litellm_team_rate_limit_allowed_metric",
"Configured rate limit for the Team in the current window (team rpm_limit / tpm_limit), by rate_limit_type",
labelnames=self.get_labels_for_metric("litellm_team_rate_limit_allowed_metric"),
)
self.litellm_team_rate_limit_used_metric = self._gauge_factory(
"litellm_team_rate_limit_used_metric",
"Requests or tokens the Team has consumed in the current rate limit window, by rate_limit_type",
labelnames=self.get_labels_for_metric("litellm_team_rate_limit_used_metric"),
)
########################################
# LLM API Deployment Metrics / analytics
########################################
@ -1433,6 +1501,11 @@ class PrometheusLogger(CustomLogger):
model_id=enum_values.model_id,
)
self._set_key_and_team_rate_limit_metrics(
standard_logging_payload=standard_logging_payload, # pyright: ignore[reportArgumentType] # isinstance(dict) above narrows the TypedDict to dict[Unknown, Unknown]
enum_values=enum_values,
)
# set latency metrics
self._set_latency_metrics(
kwargs=kwargs,
@ -1960,17 +2033,102 @@ class PrometheusLogger(CustomLogger):
"""
if standard_logging_payload is None:
return None
return PrometheusLogger._get_int_from_v3_rate_limit_headers(
standard_logging_payload=standard_logging_payload,
header_name=f"x-ratelimit-model_per_key-remaining-{rate_limit_type}",
)
@staticmethod
def _get_int_from_v3_rate_limit_headers(
standard_logging_payload: StandardLoggingPayload,
header_name: str,
) -> int | None:
hidden_params: Final = standard_logging_payload.get("hidden_params")
if hidden_params is None:
return None
additional_headers: Final = hidden_params.get("additional_headers")
additional_headers: Final[Mapping[str, object] | None] = hidden_params.get("additional_headers")
if additional_headers is None:
return None
value: Final = dict(additional_headers).get(f"x-ratelimit-model_per_key-remaining-{rate_limit_type}")
value: Final = additional_headers.get(header_name)
if isinstance(value, bool) or not isinstance(value, int):
return None
return value
def _set_key_and_team_rate_limit_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
) -> None:
"""
Export the key-level and team-level RPM / TPM limit and current window
usage from the ``x-ratelimit-{api_key,team}-{limit,remaining}-*``
headers the v3 rate limiter mirrors into the logging payload. The
limiter already read these counters (from Redis when configured) on
the request path, so no extra store lookup happens here. Descriptors
without a configured limit emit no header, so their series is removed
rather than left at the value from before the limit was dropped.
"""
descriptor_gauges: Final[
tuple[tuple[Literal["api_key", "team"], DEFINED_PROMETHEUS_METRICS, Gauge, Gauge], ...]
] = (
(
"api_key",
"litellm_api_key_rate_limit_allowed_metric",
self.litellm_api_key_rate_limit_allowed_metric,
self.litellm_api_key_rate_limit_used_metric,
),
(
"team",
"litellm_team_rate_limit_allowed_metric",
self.litellm_team_rate_limit_allowed_metric,
self.litellm_team_rate_limit_used_metric,
),
)
for descriptor_key, metric_name, allowed_gauge, used_gauge in descriptor_gauges:
for rate_limit_type in ("requests", "tokens"):
self._set_rate_limit_allowed_and_used_gauges(
standard_logging_payload=standard_logging_payload,
enum_values=enum_values,
descriptor_key=descriptor_key,
metric_name=metric_name,
allowed_gauge=allowed_gauge,
used_gauge=used_gauge,
rate_limit_type=rate_limit_type,
)
def _set_rate_limit_allowed_and_used_gauges(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
descriptor_key: Literal["api_key", "team"],
metric_name: DEFINED_PROMETHEUS_METRICS,
allowed_gauge: Gauge,
used_gauge: Gauge,
rate_limit_type: Literal["requests", "tokens"],
) -> None:
limit: Final = self._get_int_from_v3_rate_limit_headers(
standard_logging_payload=standard_logging_payload,
header_name=f"x-ratelimit-{descriptor_key}-limit-{rate_limit_type}",
)
remaining: Final = self._get_int_from_v3_rate_limit_headers(
standard_logging_payload=standard_logging_payload,
header_name=f"x-ratelimit-{descriptor_key}-remaining-{rate_limit_type}",
)
labelled_values: Final = replace(enum_values, rate_limit_type=rate_limit_type)
labelnames: Final = self.get_labels_for_metric(metric_name)
labels: Final = prometheus_label_factory(
supported_enum_labels=labelnames,
enum_values=labelled_values,
label_context=PrometheusLabelFactoryContext(labelled_values),
)
if limit is None or remaining is None:
label_values: Final = tuple(labels.get(label) for label in labelnames)
self._bounded_prometheus_series_tracker.remove_series(allowed_gauge, label_values)
self._bounded_prometheus_series_tracker.remove_series(used_gauge, label_values)
return
allowed_gauge.labels(**labels).set(limit)
used_gauge.labels(**labels).set(limit - remaining)
def _set_virtual_key_rate_limit_metrics(
self,
user_api_key: str | None,
@ -2407,7 +2565,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=request_data.get("model", ""),
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
@ -2627,7 +2785,9 @@ class PrometheusLogger(CustomLogger):
label_model_id = ""
label_api_base = ""
label_api_provider = ""
label_requested_model = litellm_model_name or model_group or ""
label_requested_model = (
_bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or ""
)
enum_values: Final = UserAPIKeyLabelValues(
litellm_model_name=label_litellm_model_name,
@ -3186,7 +3346,7 @@ class PrometheusLogger(CustomLogger):
_tags: Final = cast(list[str], kwargs.get("tags") or [])
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],
@ -3227,7 +3387,7 @@ class PrometheusLogger(CustomLogger):
)
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],

4
litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py
View file

@ -60,6 +60,10 @@ class BoundedPrometheusSeriesTracker:
break
del series[tracked_label_values]
def remove_series(self, metric: object, label_values: tuple[str | None, ...]) -> bool:
"""Drop one child series, True when it is gone (removed or never existed)."""
return self._remove_metric_child(metric, label_values)
def _should_run_ttl_cleanup(
self,
metric_name: str,

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -127,7 +127,7 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
@ -135,7 +135,7 @@ def declared_authenticating_provider(model: str, custom_llm_provider: str | None
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or model.split("/", 1)[0]
declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None

126
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -12,6 +12,7 @@ import asyncio
import json
import os
import random
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime, timezone
@ -154,18 +155,6 @@ class GetModelCostMap:
return True
@staticmethod
def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict:
"""
Fetch the model cost map from a remote URL.
Returns the parsed JSON dict. Raises on network/parse errors
(caller is expected to handle).
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
@ -212,6 +201,13 @@ class _AsyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ...
class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
def _default_reload_client() -> _AsyncGetClient:
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -219,13 +215,30 @@ def _default_reload_client() -> _AsyncGetClient:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
async def _attempt_fetch(
client: _AsyncGetClient, url: str, timeout: int
) -> ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable:
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
return ModelCostMapReloadUnavailable(reason=reason)
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
try:
response: Final = await client.get(url, timeout=timeout)
except httpx.HTTPError as e:
return _FetchAttemptRetryable(reason=f"{type(e).__name__} fetching {url}: {e}", retry_after_seconds=None)
except (httpx.HTTPError, httpx.InvalidURL) as e:
return _classify_fetch_error(e, url)
return _classify_fetch_response(response, url)
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
return _classify_fetch_error(e, url)
return _classify_fetch_response(response, url)
def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemptOutcome:
if response.status_code in RETRYABLE_FETCH_STATUS_CODES:
return _FetchAttemptRetryable(
reason=f"HTTP {response.status_code} from {url}",
@ -242,6 +255,22 @@ async def _attempt_fetch(
return ModelCostMapReloaded(model_cost_map=parsed)
def _next_retry_wait(
outcome: _FetchAttemptRetryable, attempt: int, max_attempts: int, rng: random.Random
) -> float | ModelCostMapReloadUnavailable:
if attempt == max_attempts:
return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)")
wait_seconds: Final = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng)
verbose_logger.warning(
"LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs",
attempt,
max_attempts,
outcome.reason,
wait_seconds,
)
return wait_seconds
async def _fetch_remote_model_cost_map_with_retry(
url: str,
timeout: int,
@ -254,20 +283,32 @@ async def _fetch_remote_model_cost_map_with_retry(
outcome = await _attempt_fetch(client=client, url=url, timeout=timeout)
if not isinstance(outcome, _FetchAttemptRetryable):
return outcome
if attempt == max_attempts:
return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)")
wait_seconds = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng)
verbose_logger.warning(
"LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs",
attempt,
max_attempts,
outcome.reason,
wait_seconds,
)
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
return wait_seconds
await sleep(wait_seconds)
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
def _fetch_remote_model_cost_map_with_retry_sync(
url: str,
timeout: int,
max_attempts: int,
sleep: Callable[[float], None],
rng: random.Random,
client: _SyncGetClient,
) -> ModelCostMapReloadResult:
for attempt in range(1, max_attempts + 1):
outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout)
if not isinstance(outcome, _FetchAttemptRetryable):
return outcome
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
return wait_seconds
sleep(wait_seconds)
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
async def refetch_model_cost_map(
url: str,
timeout: int = 5,
@ -423,13 +464,21 @@ def _finalize_model_cost_map(model_cost: dict) -> dict:
return _expand_model_aliases(model_cost)
def get_model_cost_map(url: str) -> dict:
def get_model_cost_map(
url: str,
timeout: int = 5,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
rng: random.Random | None = None,
client: "_SyncGetClient | None" = None,
) -> dict:
"""
Public entry point — returns the model cost map dict.
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only.
2. Otherwise fetches from ``url``, validates integrity, and falls back
to the local backup on any failure.
2. Otherwise fetches from ``url``, retrying transient HTTP errors
(429/5xx/transport) with Retry-After-aware backoff, validates
integrity, and falls back to the local backup on any failure.
Only the backup model count is cached (a single int) for validation.
The full backup dict is only parsed when it must be *returned* as a
@ -448,17 +497,24 @@ def get_model_cost_map(url: str) -> dict:
_cost_map_source_info.url = url
_cost_map_source_info.is_env_forced = False
try:
content: Final = GetModelCostMap.fetch_remote_model_cost_map(url)
except Exception as e:
result: Final = _fetch_remote_model_cost_map_with_retry_sync(
url=url,
timeout=timeout,
max_attempts=max_attempts,
sleep=sleep,
rng=rng if rng is not None else random.Random(),
client=client if client is not None else httpx,
)
if isinstance(result, ModelCostMapReloadUnavailable):
verbose_logger.warning(
"LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.",
url,
str(e),
result.reason,
)
_cost_map_source_info.source = "local"
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {e}"
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}"
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
content: Final = result.model_cost_map
# Validate using cached count (cheap int comparison, no file I/O)
if not GetModelCostMap.validate_model_cost_map(

71
litellm/litellm_core_utils/litellm_logging.py
View file

@ -2959,13 +2959,25 @@ class Logging(LiteLLMLoggingBaseClass):
"Model=%s not found in completion cost map. Setting 'response_cost' to None", self.model
)
self.model_call_details["response_cost"] = None
except Exception: # noqa: BLE001 # cost calculation must never block later callbacks (slot release)
verbose_logger.exception(
"Error calculating streaming response cost for model=%s. Setting 'response_cost' to None",
self.model,
)
self.model_call_details["response_cost"] = None
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
try:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release)
verbose_logger.exception(
"LiteLLM.LoggingError: [Non-Blocking] Exception building the standard logging payload "
"for a streaming response; callbacks still run without it"
)
# print standard logging payload
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
@ -3005,32 +3017,39 @@ class Logging(LiteLLMLoggingBaseClass):
## LOGGING HOOK ##
for callback in callbacks:
if isinstance(callback, CustomGuardrail):
from litellm.types.guardrails import GuardrailEventHooks
try:
if isinstance(callback, CustomGuardrail):
from litellm.types.guardrails import GuardrailEventHooks
if (
callback.should_run_guardrail(
data=self.model_call_details,
event_type=GuardrailEventHooks.logging_only,
if (
callback.should_run_guardrail(
data=self.model_call_details,
event_type=GuardrailEventHooks.logging_only,
)
is not True
):
continue
self.model_call_details, result = await callback.async_logging_hook(
kwargs=self.model_call_details,
result=result,
call_type=self.call_type,
)
is not True
):
continue
self.model_call_details, result = await callback.async_logging_hook(
kwargs=self.model_call_details,
result=result,
call_type=self.call_type,
)
elif isinstance(callback, CustomLogger):
result = redact_message_input_output_from_custom_logger(
result=result, litellm_logging_obj=self, custom_logger=callback
)
self.model_call_details, result = await callback.async_logging_hook(
kwargs=self.model_call_details,
result=result,
call_type=self.call_type,
elif isinstance(callback, CustomLogger):
result = redact_message_input_output_from_custom_logger(
result=result, litellm_logging_obj=self, custom_logger=callback
)
self.model_call_details, result = await callback.async_logging_hook(
kwargs=self.model_call_details,
result=result,
call_type=self.call_type,
)
except Exception: # noqa: BLE001 # one failing hook must not skip later callbacks (slot release)
verbose_logger.error(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred in async_logging_hook %s",
traceback.format_exc(),
)
self._handle_callback_failure(callback=callback)
self.has_run_logging(event_type="async_success")

13
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1281,6 +1281,19 @@ def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mappin
return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) # mutable-ok: fresh per-call $ref memo
def tool_with_flattened_parameters(tool: Mapping[str, object]) -> Mapping[str, object]:
function: Final = tool.get("function")
if not isinstance(function, dict):
return tool
parameters: Final = function.get("parameters")
if not isinstance(parameters, dict):
return tool
flattened: Final = flatten_top_level_schema_combinators(parameters)
if flattened is parameters:
return tool
return {**tool, "function": {**function, "parameters": flattened}} # mutable-ok: request tools are JSON dicts
def _get_image_mime_type_from_url(url: str) -> str | None:
"""
Get mime type for common image URLs

7
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4957,10 +4957,13 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockToolBlock | None:
from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock
from litellm.llms.bedrock.common_utils import (
bedrock_model_accepts_cache_points,
is_claude_4_5_on_bedrock,
)
cache_control: Final = tool.get("cache_control", None)
if cache_control is not None:
if cache_control is not None and bedrock_model_accepts_cache_points(model):
cache_point: Final = cache_control.get("type", "ephemeral")
if cache_point == "ephemeral":
cache_point_block: Final[CachePointBlock] = {"type": "default"}

8
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -36,6 +36,8 @@ from litellm.types.utils import (
from litellm.utils import print_verbose, token_counter
if TYPE_CHECKING:
from openai.types.completion_usage import CompletionUsage
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import (
UsagePerChunk,
@ -794,7 +796,7 @@ class ChunkProcessor:
@staticmethod
def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None:
usage_chunk: Usage | None = None
usage_chunk: Usage | CompletionUsage | None = None
if hasattr(chunk, "usage") and chunk.usage is not None:
usage_chunk = chunk.usage
elif "usage" in chunk:
@ -806,7 +808,9 @@ class ChunkProcessor:
if isinstance(usage_chunk, dict):
return Usage(**usage_chunk)
return usage_chunk
if usage_chunk is None or isinstance(usage_chunk, Usage):
return usage_chunk
return Usage(**usage_chunk.model_dump())
def _calculate_usage_per_chunk(
self,

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -1516,7 +1516,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled)
if _tool is None:
continue
if not is_thinking_enabled:
if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model):
_tool_choice = {
"name": RESPONSE_FORMAT_TOOL_NAME,
"type": "tool",

23
litellm/llms/anthropic/common_utils.py
View file

@ -325,13 +325,17 @@ class AnthropicModelInfo(BaseLLMModelInfo):
status_code=400,
)
@staticmethod
def forced_tool_use_unsupported(model: str) -> bool:
return AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is False
@staticmethod
def forced_tool_use_downgraded(model: str, drop_params: bool) -> bool:
"""True when the model map flags the model with
``supports_forced_tool_use: false`` (Fable 5.1 / Mythos 5.1 400 on
``any``/``tool``) and ``drop_params`` asks for the ``auto`` downgrade;
raises a clean client-side 400 for such models without ``drop_params``."""
if AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is not False:
if not AnthropicModelInfo.forced_tool_use_unsupported(model):
return False
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
@ -1374,31 +1378,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict:
return additional_headers
def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]:
def _anthropic_model_entry(
model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str]
) -> Mapping[str, object]:
return { # mutable-ok: JSON response body, serialized by the route and never mutated
"type": "model",
"id": model["id"],
"display_name": model["id"],
"display_name": display_names.get(model["id"], model["id"]),
"created_at": created_at,
"max_input_tokens": model.get("max_input_tokens"),
"max_tokens": model.get("max_output_tokens"),
}
def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]:
def create_anthropic_model_list_response(
models: Sequence[ModelInfoResponse],
display_names: Mapping[str, str] = MappingProxyType({}),
) -> Mapping[str, object]:
"""Build the Anthropic-native /v1/models envelope.
Clients that send an anthropic-version header parse the Anthropic Models API
shape (type/display_name/created_at plus has_more/first_id/last_id) and filter
the list themselves, so every model is returned here. The token limits carry
over from the OpenAI-shaped listing, named as the Messages API names them, and
are always present because the vendor shape declares them nullable, not optional
are always present because the vendor shape declares them nullable, not optional.
display_names maps a listed model id to a configured human-readable name; ids
without an entry fall back to the id itself, matching the vendor behavior
"""
created_at: Final = (
datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z")
)
data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated
_anthropic_model_entry(model, created_at) for model in models
_anthropic_model_entry(model, created_at, display_names) for model in models
]
return { # mutable-ok: JSON response body, serialized by the route and never mutated
"data": data,

25
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -86,22 +86,41 @@ def _decoded_sse_data_line(line: bytes) -> object | None:
return None
def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
def _anthropic_event_payload(chunk: object, event_type: str) -> Mapping[str, object] | None:
if isinstance(chunk, dict):
return chunk if chunk.get("type") == "error" else None
return chunk if chunk.get("type") == event_type else None
if isinstance(chunk, (bytes, bytearray)):
decoded_lines: Final = (_decoded_sse_data_line(line) for line in chunk.splitlines())
return next(
(
candidate
for candidate in decoded_lines
if isinstance(candidate, dict) and candidate.get("type") == "error"
if isinstance(candidate, dict) and candidate.get("type") == event_type
),
None,
)
return None
def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
return _anthropic_event_payload(chunk, "error")
def parse_anthropic_refusal_stop_details(chunk: object) -> Mapping[str, object] | None:
"""
Return the ``stop_details`` object of an Anthropic SSE ``message_delta``
chunk whose delta carries ``stop_reason: "refusal"`` (a safeguard refusal:
https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback),
or None for any other chunk, a plain refusal without ``stop_details`` included.
"""
payload: Final = _anthropic_event_payload(chunk, "message_delta")
delta: Final = payload.get("delta") if payload is not None else None
if not isinstance(delta, dict) or delta.get("stop_reason") != "refusal":
return None
stop_details: Final = delta.get("stop_details")
return stop_details if isinstance(stop_details, dict) else None
def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
"""Return the ``error`` object of an Anthropic SSE ``event: error`` chunk, or None."""
payload: Final = _anthropic_error_event_payload(chunk)

49
litellm/llms/anthropic/experimental_pass_through/messages/utils.py
View file

@ -1,11 +1,40 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import Any, Final, cast, get_type_hints
from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints
from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
if TYPE_CHECKING:
from litellm.exceptions import ContentPolicyViolationError
def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | None:
"""
Return the ``stop_details`` of an Anthropic Messages response refused by a
safeguard (``stop_reason: "refusal"`` carrying ``stop_details``:
https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback),
or None for any other response, a plain refusal without ``stop_details`` included.
"""
if not isinstance(response, dict) or response.get("stop_reason") != "refusal":
return None
stop_details: Final = response.get("stop_details")
return stop_details if isinstance(stop_details, dict) else None
def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError":
"""The exception a safeguard-refused Anthropic response converts into so the
content-policy fallback chain can re-dispatch it."""
from litellm.exceptions import ContentPolicyViolationError
return ContentPolicyViolationError(
message=f"Anthropic safeguard refusal (category: {stop_details.get('category')}).",
model=model,
llm_provider="anthropic",
)
@lru_cache(maxsize=1)
def _anthropic_messages_optional_param_keys() -> frozenset[str]:
@ -100,14 +129,12 @@ def mock_response(
model=model,
)
return AnthropicMessagesResponse(
**{
"content": [{"text": mock_response, "type": "text"}],
"id": "msg_013Zva2CMHLNnXjNJJKqJ2EF",
"model": "claude-sonnet-4-20250514",
"role": "assistant",
"stop_reason": "end_turn",
"stop_sequence": None,
"type": "message",
"usage": {"input_tokens": 2095, "output_tokens": 503},
}
content=[{"text": mock_response, "type": "text"}],
id="msg_013Zva2CMHLNnXjNJJKqJ2EF",
model="claude-sonnet-4-20250514",
role="assistant",
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage={"input_tokens": 2095, "output_tokens": 503},
)

17
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -1,3 +1,5 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
from httpx._models import Headers, Response
@ -6,6 +8,7 @@ import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
hoist_images_from_tool_messages,
tool_with_flattened_parameters,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_azure_openai_messages,
@ -32,6 +35,19 @@ else:
LoggingClass = Any
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
def flattened_tools_update(optional_params: Mapping[str, object]) -> Mapping[str, object]:
tools: Final = optional_params.get("tools")
if not isinstance(tools, list):
return _NO_TOOLS_UPDATE
flattened: Final = [ # mutable-ok: request tools are a JSON list
tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools
]
return MappingProxyType({"tools": flattened})
class AzureOpenAIConfig(BaseConfig):
"""
Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions
@ -261,6 +277,7 @@ class AzureOpenAIConfig(BaseConfig):
"model": model,
"messages": azure_messages,
**optional_params,
**flattened_tools_update(optional_params),
}
def transform_response(

7
litellm/llms/azure/chat/o_series_transformation.py
View file

@ -20,6 +20,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.utils import get_model_info, supports_reasoning
from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig
from .gpt_transformation import flattened_tools_update
class AzureOpenAIO1Config(OpenAIOSeriesConfig):
@ -108,4 +109,8 @@ class AzureOpenAIO1Config(OpenAIOSeriesConfig):
headers: dict,
) -> dict:
model = model.replace("o_series/", "") # handle o_series/my-random-deployment-name
return super().transform_request(model, messages, optional_params, litellm_params, headers)
flattened_params: Final = { # mutable-ok: transform_request's contract takes a plain JSON params dict
**optional_params,
**flattened_tools_update(optional_params),
}
return super().transform_request(model, messages, flattened_params, litellm_params, headers)

2
litellm/llms/azure_ai/vector_stores/transformation.py
View file

@ -19,6 +19,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -115,6 +116,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict[str, Any]]:
"""
Transform search request for Azure AI Search API

4
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -155,8 +155,8 @@ class BaseTranslation(ABC):
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: list[Any] | None = None,
) -> list[bytes] | None:
responses_so_far: Sequence[Any] | None = None,
) -> Sequence[bytes] | None:
"""
Build the streaming chunks that deliver a guardrail block message and
cleanly terminate the stream in this provider's wire format.

55
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -124,6 +124,61 @@ def blocked_responses_api_usage(original_response: object) -> ResponseAPIUsage:
)
def stream_item_field(item: object, field: str) -> object | None:
if isinstance(item, dict):
return item.get(field)
return getattr(item, field, None)
def blocked_chat_stream_usage(original_response: object) -> tuple[int, int]:
"""
``(prompt_tokens, completion_tokens)`` for a synthetic guardrail-blocked
chat completions stream.
A mid-stream block carries the chunks received so far as a list; real usage
rides on the final chunk when the upstream sent one
(``stream_options.include_usage``). Non-list originals defer to
``blocked_response_usage``.
"""
if not isinstance(original_response, list):
usage: Final = blocked_response_usage(original_response)
return usage.get("input_tokens", 0), usage.get("output_tokens", 0)
usage_obj: Final = next(
(
chunk_usage
for item in reversed(original_response)
if (chunk_usage := stream_item_field(item, "usage")) is not None
),
None,
)
return (
_usage_tokens(usage_obj, "prompt_tokens", "input_tokens"),
_usage_tokens(usage_obj, "completion_tokens", "output_tokens"),
)
def blocked_responses_stream_usage(original_response: object) -> ResponseAPIUsage:
"""
``ResponseAPIUsage`` for a synthetic guardrail-blocked /v1/responses stream.
A mid-stream block carries the events received so far as a list; real usage
rides on the ``response.completed`` event's response when the upstream sent
one. Non-list originals defer to ``blocked_responses_api_usage``.
"""
if not isinstance(original_response, list):
return blocked_responses_api_usage(original_response)
completed: Final = next(
(
response
for item in reversed(original_response)
if stream_item_field(item, "type") == "response.completed"
and (response := stream_item_field(item, "response")) is not None
),
None,
)
return blocked_responses_api_usage(completed)
def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per: Final = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None)
if per is not None:

5
litellm/llms/base_llm/vector_store/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
from ..chat.transformation import BaseLLMException as _BaseLLMException
@ -57,6 +58,7 @@ class BaseVectorStoreConfig:
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
pass
@ -69,6 +71,7 @@ class BaseVectorStoreConfig:
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
"""
Optional async version of transform_search_vector_store_request.
@ -84,6 +87,7 @@ class BaseVectorStoreConfig:
litellm_logging_obj=litellm_logging_obj,
litellm_params=litellm_params,
extra_body=extra_body,
router=router,
)
@abstractmethod
@ -197,6 +201,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
extra_body: Mapping[str, object] | None = None,
router: "Router | None" = None,
) -> NoReturn:
raise NotImplementedError("Direct vector store providers execute the search themselves; no HTTP request shape")

6
litellm/llms/bedrock/base_aws_llm.py
View file

@ -1442,7 +1442,7 @@ class BaseAWSLLM:
@tracer.wrap()
def get_request_headers(
self,
credentials: Credentials,
credentials: Credentials | None,
aws_region_name: str,
extra_headers: dict | None,
endpoint_url: str,
@ -1469,9 +1469,13 @@ class BaseAWSLLM:
try:
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.exceptions import NoCredentialsError
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
if credentials is None:
raise NoCredentialsError()
# Filter headers for AWS signature calculation
# AWS SigV4 only includes specific headers in signature calculation
aws_signature_headers: Final = self._filter_headers_for_aws_signature(headers)

40
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -1,4 +1,6 @@
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -24,6 +26,22 @@ from ..common_utils import BedrockError, _get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
def _sigv4_principal(credentials: Credentials | None) -> Mapping[str, str]:
if credentials is None:
return MappingProxyType({})
return MappingProxyType(
{
key: value
for key, value in (
("aws_access_key_id", credentials.access_key),
("aws_secret_access_key", credentials.secret_key),
("aws_session_token", credentials.token),
)
if value is not None
}
)
def make_sync_call(
client: HTTPHandler | None,
api_base: str,
@ -95,7 +113,7 @@ class BedrockConverseLLM(BaseAWSLLM):
stream,
optional_params: dict,
litellm_params: dict,
credentials: Credentials,
credentials: Credentials | None,
logger_fn=None,
headers={},
client: AsyncHTTPHandler | None = None,
@ -167,7 +185,7 @@ class BedrockConverseLLM(BaseAWSLLM):
stream,
optional_params: dict,
litellm_params: dict,
credentials: Credentials,
credentials: Credentials | None,
logger_fn=None,
headers: dict = {},
client: AsyncHTTPHandler | None = None,
@ -331,7 +349,7 @@ class BedrockConverseLLM(BaseAWSLLM):
litellm_params["aws_region_name"] = aws_region_name # [DO NOT DELETE] important for async calls
credentials: Final[Credentials] = self.get_credentials(
credentials: Final[Credentials | None] = self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
@ -368,19 +386,13 @@ class BedrockConverseLLM(BaseAWSLLM):
# The Rust core owns the whole call for the subset it accepts. Ask
# before transforming so whichever path runs emits pre_call once, and
# hand down the credentials, region and endpoint this handler already
# resolved so both paths sign as the same principal.
# resolved so both paths sign as the same principal. Bearer-token auth
# resolves no SigV4 principal at all, and each path reads that token
# itself.
rust_optional_params: Final = { # mutable-ok: json.dumps in the bridge rejects a mappingproxy
**optional_params,
**{ # mutable-ok: merged into its mutable parent above
key: value
for key, value in (
("aws_access_key_id", credentials.access_key),
("aws_secret_access_key", credentials.secret_key),
("aws_session_token", credentials.token),
("aws_region_name", aws_region_name),
)
if value is not None
},
**_sigv4_principal(credentials),
"aws_region_name": aws_region_name,
}
serves_via_rust: Final = rust_chat_completions_accepts(
model=model,

6
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -87,6 +87,7 @@ from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_bedrock_application_inference_profile_arn,
@ -1073,6 +1074,7 @@ class AmazonConverseConfig(BaseConfig):
if (
litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
and not is_thinking_enabled
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
):
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
@ -1148,7 +1150,7 @@ class AmazonConverseConfig(BaseConfig):
model: str | None = None,
) -> SystemContentBlock | ContentBlock | None:
cache_control: Final = message_block.get("cache_control", None)
if cache_control is None:
if cache_control is None or not bedrock_model_accepts_cache_points(model):
return None
cache_point: Final = self._build_cache_point_block(cache_control, model)
@ -1612,7 +1614,7 @@ class AmazonConverseConfig(BaseConfig):
# Append cachePoint to tools if cache_control_injection_points has tool_config
cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None)
if cache_injection_points and len(bedrock_tools) > 0:
if cache_injection_points and len(bedrock_tools) > 0 and bedrock_model_accepts_cache_points(model):
for point in cache_injection_points:
if point.get("location") == "tool_config":
cache_point = self._build_cache_point_block(point.get("control"), model)

22
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
@ -11,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
@ -74,10 +76,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
drop_params: bool,
) -> dict:
# Force tool-based structured outputs for Bedrock Invoke
# (similar to VertexAI fix in #19201)
# Bedrock Invoke doesn't support output_format parameter
# (similar to VertexAI fix in #19201) unless the model map advertises
# native structured output
from litellm.utils import supports_native_structured_output
original_model: Final = model
if "response_format" in non_default_params:
if "response_format" in non_default_params and not supports_native_structured_output(
model=model, custom_llm_provider="bedrock"
):
# Use a model name that forces tool-based approach
model = "claude-3-sonnet-20240229"
@ -101,6 +107,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
# Restore original model name
model = original_model
# The stub model hides the original model from the parent's forced-tool-use backstop
response_format_tool_choice: Final = optional_params.get("tool_choice")
if (
"response_format" in non_default_params
and isinstance(response_format_tool_choice, dict)
and response_format_tool_choice.get("name") == RESPONSE_FORMAT_TOOL_NAME
and AnthropicModelInfo.forced_tool_use_unsupported(original_model)
):
optional_params.pop("tool_choice")
return optional_params
@staticmethod

24
litellm/llms/bedrock/common_utils.py
View file

@ -816,6 +816,30 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
)
def bedrock_model_accepts_cache_points(model: str | None) -> bool:
"""
Whether Converse ``cachePoint`` blocks may be sent to this model.
Bedrock rejects requests carrying cachePoint blocks for models without prompt
caching support ("You invoked an unsupported model or your request did not allow
prompt caching"), so a model whose cost-map entry does not declare
``supports_prompt_caching`` must not receive them. A model absent from the map
(an application inference profile ARN, a model newer than the map) keeps emitting
so existing caching setups never silently degrade. ``litellm.utils.supports_prompt_caching``
is not reusable here: it returns False for unmapped models, the opposite polarity.
"""
if model is None:
return True
entries: Final = tuple(
entry
for candidate in (model, get_bedrock_base_model(model))
if (entry := litellm.model_cost.get(candidate)) is not None
)
if not entries:
return True
return any(entry.get("supports_prompt_caching") is True for entry in entries)
def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model supports Bedrock prompt caching with an extended '1h' TTL

2
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -27,6 +27,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
@ -196,6 +197,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
if isinstance(query, list):
query = " ".join(query)

13
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -3,6 +3,7 @@ import concurrent.futures
import contextlib
import os
import ssl
import sys
import typing
import urllib.request
from collections.abc import Callable, Generator
@ -75,10 +76,22 @@ except ImportError:
pass
def _current_task_is_cancelling() -> bool:
task: Final = asyncio.current_task()
if task is None or sys.version_info < (3, 11):
return True
return task.cancelling() > 0
@contextlib.contextmanager
def map_aiohttp_exceptions() -> Generator[None, None, None]:
try:
yield
except asyncio.CancelledError as exc:
# a closing connector cancels its shielded DNS task; that surfaces here without the request task being cancelled
if _current_task_is_cancelling():
raise
raise httpx.ConnectError("aiohttp transport cancelled the request internally") from exc
except Exception as exc:
mapped_exc: type[Exception] | None = None

11
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -178,6 +178,7 @@ if TYPE_CHECKING:
AnthropicMessagesStreamingResponse,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.router import Router
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
CancelRunResponse,
@ -2923,7 +2924,7 @@ class BaseLLMHTTPHandler:
final_response: Final = await self._call_agentic_completion_hooks(
response=initial_response,
model=model,
messages=(input if isinstance(input, list) else [{"role": "user", "content": input}]),
messages=(input if isinstance(input, list) else [{"role": "user", "content": input}]), # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; the hook accepts response input items at runtime
anthropic_messages_provider_config=responses_api_provider_config,
anthropic_messages_optional_request_params=response_api_optional_request_params,
logging_obj=logging_obj,
@ -5415,7 +5416,7 @@ class BaseLLMHTTPHandler:
try:
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses(
model=patch.model or model,
input=patch.messages,
input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime
**optional_params,
**kwargs_for_followup,
)
@ -9688,6 +9689,7 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
_is_async: bool = False,
router: "Router | None" = None,
) -> VectorStoreSearchResponse:
if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
self._pre_call_direct_vector_store_search(
@ -9738,6 +9740,7 @@ class BaseLLMHTTPHandler:
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
extra_body=extra_body,
router=router,
)
else:
(
@ -9751,6 +9754,7 @@ class BaseLLMHTTPHandler:
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
extra_body=extra_body,
router=router,
)
all_optional_params: Final[dict[str, object]] = dict(litellm_params)
all_optional_params.update(vector_store_search_optional_params or {})
@ -9802,6 +9806,7 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
_is_async: bool = False,
router: "Router | None" = None,
) -> VectorStoreSearchResponse | Coroutine[object, object, VectorStoreSearchResponse]:
if _is_async:
return self.async_vector_store_search_handler(
@ -9816,6 +9821,7 @@ class BaseLLMHTTPHandler:
extra_body=extra_body,
timeout=timeout,
client=client,
router=router,
)
if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
@ -9862,6 +9868,7 @@ class BaseLLMHTTPHandler:
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
extra_body=extra_body,
router=router,
)
all_optional_params: Final[dict[str, object]] = dict(litellm_params)

2
litellm/llms/gemini/vector_stores/transformation.py
View file

@ -33,6 +33,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
@ -168,6 +169,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: Mapping[str, object] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
"""
Transform search request to Gemini's generateContent format.

27
litellm/llms/langflow/a2a.py
View file

@ -1,28 +1,9 @@
import hashlib
from typing import Any, Final
def get_session_id_from_a2a_params(params: dict[str, Any]) -> str | None:
message: Final = params.get("message", {})
if isinstance(message, dict):
return message.get("contextId")
return getattr(message, "contextId", None)
def scope_session_to_principal(session_id: str, principal: str | None) -> str:
"""
Bind a client-supplied A2A contextId to the authenticated principal.
Without this, two distinct keys authorized for the same LangFlow agent could
set the same contextId and read/append to each other's LangFlow memory. The
principal is hashed (it is already a hashed token) so the raw value is never
sent to the LangFlow backend, while the original contextId is kept as a
suffix for operator-side correlation.
"""
if not principal:
return session_id
principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16]
return f"{principal_prefix}-{session_id}"
from litellm.a2a_protocol.utils import (
get_session_id_from_a2a_params,
scope_session_to_principal,
)
def merge_a2a_session_into_litellm_params(

2
litellm/llms/milvus/vector_stores/transformation.py
View file

@ -19,6 +19,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -123,6 +124,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict[str, Any]]:
"""
Transform search request for Azure AI Search API

17
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -20,9 +20,9 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
flatten_top_level_schema_combinators,
get_tool_call_names,
hoist_images_from_tool_messages,
tool_with_flattened_parameters,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
@ -70,19 +70,6 @@ else:
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
def _tool_with_flattened_parameters(tool: Mapping[str, object]) -> Mapping[str, object]:
function: Final = tool.get("function")
if not isinstance(function, dict):
return tool
parameters: Final = function.get("parameters")
if not isinstance(parameters, dict):
return tool
flattened: Final = flatten_top_level_schema_combinators(parameters)
if flattened is parameters:
return tool
return {**tool, "function": {**function, "parameters": flattened}} # mutable-ok: request tools are JSON dicts
class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
"""
Reference: https://platform.openai.com/docs/api-reference/chat/create
@ -466,7 +453,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
):
return _NO_TOOLS_UPDATE
flattened: Final = [ # mutable-ok: request tools are a JSON list
_tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools
tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools
]
return MappingProxyType({"tools": flattened})

138
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -14,9 +14,14 @@ Pattern Overview:
This pattern can be replicated for other message formats (e.g., Anthropic).
"""
import json
import time
import uuid
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Union, cast
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import (
@ -24,6 +29,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_chat_stream_usage,
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
@ -32,6 +38,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
openai_tool_name,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
stream_item_field,
)
from litellm.main import stream_chunk_builder
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@ -49,7 +56,10 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
@ -1005,3 +1015,129 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
else:
# Subsequent chunks - clear the text
content_item["text"] = ""
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
"""
True once any relayed chunk carries a non-null ``finish_reason``.
The unified guardrail's ``end_of_stream_only`` streaming path probes
this via ``hasattr`` to withhold the terminal chunks until
end-of-stream moderation runs, so a block can replace the finish
instead of trailing after a ``finish_reason`` the client already saw.
"""
return any(
stream_item_field(choice, "finish_reason") is not None
for item in responses_so_far
for choice in _stream_chunk_choices(item)
)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
"""
Build OpenAI chat-completions SSE chunks that deliver the guardrail
block message and terminate the stream cleanly, mirroring the
non-streaming block response: ``finish_reason`` ``content_filter`` plus
the real usage the upstream call consumed.
- ``stream_started`` False (buffered / pre-stream): nothing has been
sent, so open a standalone completion with a ``role`` delta.
- ``stream_started`` True (sampling / mid-stream): chunks already
reached the client, so continue the in-progress completion (reuse its
id/created/model, content-only delta).
The proxy's data generator appends ``data: [DONE]`` itself.
"""
chunk_id, created, model = _blocked_stream_identity(exc, responses_so_far or ())
prompt_tokens, completion_tokens = blocked_chat_stream_usage(exc.original_response)
continuation_delta: Final[_BlockedChunkDelta] = {"content": exc.message}
standalone_delta: Final[_BlockedChunkDelta] = {"role": "assistant", "content": exc.message}
message_chunk: Final[_BlockedChunk] = {
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": (
{
"index": 0,
"delta": continuation_delta if stream_started else standalone_delta,
"finish_reason": None,
},
),
}
final_chunk: Final[_BlockedChunk] = {
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": ({"index": 0, "delta": {}, "finish_reason": "content_filter"},),
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
}
return _chat_sse_chunk(message_chunk), _chat_sse_chunk(final_chunk)
class _BlockedChunkDelta(TypedDict, total=False):
role: ReadOnly[str]
content: ReadOnly[str]
class _BlockedChunkChoice(TypedDict):
index: ReadOnly[int]
delta: ReadOnly[_BlockedChunkDelta]
finish_reason: ReadOnly[str | None]
class _BlockedChunkUsage(TypedDict):
prompt_tokens: ReadOnly[int]
completion_tokens: ReadOnly[int]
total_tokens: ReadOnly[int]
class _BlockedChunk(TypedDict):
id: ReadOnly[str]
object: ReadOnly[str]
created: ReadOnly[int]
model: ReadOnly[str]
choices: ReadOnly[tuple[_BlockedChunkChoice, ...]]
usage: NotRequired[ReadOnly[_BlockedChunkUsage]]
def _chat_sse_chunk(payload: _BlockedChunk) -> bytes:
return f"data: {json.dumps(payload)}\n\n".encode()
def _stream_chunk_choices(item: object) -> Sequence[object]:
choices: Final = stream_item_field(item, "choices")
if isinstance(choices, Sequence) and not isinstance(choices, (str, bytes)):
return choices
return ()
def _blocked_stream_identity(
exc: "ModifyResponseException", responses_so_far: Sequence[object]
) -> tuple[str, int, str]:
identified: Final = next(
(
(chunk_id, item)
for item in responses_so_far
if isinstance(chunk_id := stream_item_field(item, "id"), str) and chunk_id
),
None,
)
if identified is None:
return f"chatcmpl-{uuid.uuid4()}", int(time.time()), exc.model
chunk_id, source = identified
created: Final = stream_item_field(source, "created")
model: Final = stream_item_field(source, "model")
return (
chunk_id,
created if isinstance(created, int) else int(time.time()),
model if isinstance(model, str) and model else exc.model,
)

388
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -28,12 +28,16 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
- text: str
"""
from collections.abc import Sequence
import time
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.tool_param import FunctionToolParam
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -41,17 +45,33 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_responses_stream_usage,
stream_item_field,
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.openai import (
AllMessageValues,
BaseLiteLLMOpenAIResponseObject,
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
ContentPartAddedEvent,
ContentPartDoneEvent,
ContentPartDonePartOutputText,
ErrorEvent,
ErrorEventError,
OpenAIMcpServerTool,
OutputItemAddedEvent,
OutputItemDoneEvent,
OutputTextDeltaEvent,
OutputTextDoneEvent,
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
@ -63,11 +83,13 @@ from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import ResponseInputParam
from litellm.types.utils import ResponsesAPIResponse
class ResponseOutputEnvelope(TypedDict, total=False):
@ -82,6 +104,10 @@ class ResponsesStreamChunk(TypedDict, total=False):
type: ReadOnly[str]
text: ReadOnly[str]
delta: ReadOnly[str]
item_id: ReadOnly[str]
output_index: ReadOnly[int]
content_index: ReadOnly[int]
def _next_stream_sequence_number(responses_so_far: Sequence[Any] | None) -> int:
@ -658,8 +684,32 @@ class OpenAIResponsesHandler(BaseTranslation):
def get_streaming_string_so_far(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> str:
"""
Get the string so far from the responses so far.
``response.output_text.done`` events carry the whole part in ``text``, while
``response.output_text.delta`` events carry fragments in ``delta``. A stream
that dies before its done event (``response.failed`` / ``response.incomplete``)
has text only in deltas, so per content part the done text wins when present
and the joined deltas fill in otherwise, never both.
"""
return "".join([response.get("text", "") for response in responses_so_far])
keyed_events: Final = tuple(
(
(event.get("item_id"), event.get("output_index"), event.get("content_index")),
event.get("text"),
event.get("delta"),
)
for event in responses_so_far
if isinstance(event.get("text"), str) or isinstance(event.get("delta"), str)
)
def part_text(part_key: tuple[object, object, object]) -> str:
done_texts: Final = tuple(
text for key, text, _ in keyed_events if key == part_key and isinstance(text, str)
)
if done_texts:
return done_texts[-1]
return "".join(delta for key, _, delta in keyed_events if key == part_key and isinstance(delta, str))
return "".join(part_text(key) for key in dict.fromkeys(key for key, _, _ in keyed_events))
def _has_text_content(self, response: "ResponsesAPIResponse") -> bool:
"""
@ -837,3 +887,331 @@ class OpenAIResponsesHandler(BaseTranslation):
content[content_idx]["text"] = guardrail_response
elif hasattr(content[content_idx], "text"):
content[content_idx].text = guardrail_response
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
"""
Build Responses API SSE events that deliver the guardrail block message
and terminate the stream cleanly, mirroring the non-streaming block
response: a completed response whose only output is the violation text,
with the real usage the upstream call consumed.
- ``stream_started`` False (buffered / pre-stream): nothing has been
sent, so emit the full synthetic sequence (``response.created``
through ``response.completed``).
- ``stream_started`` True (sampling / mid-stream): events already
reached the client, so continue the in-progress response: close the
output item still open on the wire, deliver the block message as a
new output item under the same response id, and close with a
``response.completed`` carrying only the replacement item.
The proxy's data generator appends ``data: [DONE]`` itself.
"""
events: Final = (
self._block_continuation_events(exc, responses_so_far or ())
if stream_started
else self._standalone_block_events(exc)
)
return tuple(
f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode()
for event in events
)
@staticmethod
def _standalone_block_events(exc: "ModifyResponseException") -> Sequence[ResponsesAPIStreamingResponse]:
from litellm.responses.streaming_iterator import build_synthetic_response_events
return build_synthetic_response_events(
transformed=_blocked_response(exc, response_id=f"resp_{uuid.uuid4()}", model=exc.model),
logging_obj=None,
chunk_size=max(len(exc.message), 1),
)
@staticmethod
def _block_continuation_events(
exc: "ModifyResponseException", responses_so_far: Sequence[object]
) -> Sequence[ResponsesAPIStreamingResponse]:
response_id, model, output_index = _continuation_identity(exc, responses_so_far)
item: Final = _blocked_output_item(exc)
item_id: Final = item.id
part: Final[_BlockedContentPart] = {"type": "output_text", "text": exc.message, "annotations": ()}
done_part: Final[_BlockedDoneContentPart] = {
"type": "output_text",
"text": exc.message,
"annotations": (),
"logprobs": None,
}
return (
*_open_item_closing_events(responses_so_far),
OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
output_index=output_index,
item=item,
),
ContentPartAddedEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
item_id=item_id,
output_index=output_index,
content_index=0,
part=BaseLiteLLMOpenAIResponseObject.model_validate(part),
),
OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
item_id=item_id,
output_index=output_index,
content_index=0,
delta=exc.message,
),
OutputTextDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
item_id=item_id,
output_index=output_index,
content_index=0,
text=exc.message,
),
ContentPartDoneEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
item_id=item_id,
output_index=output_index,
content_index=0,
part=ContentPartDonePartOutputText.model_validate(done_part),
),
OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=output_index,
item=item,
),
ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=_blocked_response(exc, response_id=response_id, model=model, output_item=item),
),
)
class _BlockedContentPart(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
annotations: ReadOnly[tuple[object, ...]]
class _BlockedDoneContentPart(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
annotations: ReadOnly[tuple[object, ...]]
logprobs: ReadOnly[None]
class _BlockedItemPayload(TypedDict):
type: ReadOnly[str]
id: ReadOnly[str]
status: ReadOnly[str]
role: ReadOnly[str]
content: ReadOnly[tuple[_BlockedContentPart, ...]]
class _BlockedResponsePayload(TypedDict):
id: ReadOnly[str]
object: ReadOnly[str]
created_at: ReadOnly[int]
model: ReadOnly[str]
output: ReadOnly[tuple[GenericResponseOutputItem, ...]]
status: ReadOnly[str]
usage: ReadOnly[ResponseAPIUsage]
def _blocked_output_item(exc: "ModifyResponseException") -> GenericResponseOutputItem:
payload: Final[_BlockedItemPayload] = {
"type": "message",
"id": f"msg_{uuid.uuid4()}",
"status": "completed",
"role": "assistant",
"content": ({"type": "output_text", "text": exc.message, "annotations": ()},),
}
return GenericResponseOutputItem.model_validate(payload)
def _blocked_response(
exc: "ModifyResponseException",
response_id: str,
model: str,
output_item: GenericResponseOutputItem | None = None,
) -> ResponsesAPIResponse:
payload: Final[_BlockedResponsePayload] = {
"id": response_id,
"object": "response",
"created_at": int(time.time()),
"model": model,
"output": (output_item if output_item is not None else _blocked_output_item(exc),),
"status": "completed",
"usage": blocked_responses_stream_usage(exc.original_response),
}
return ResponsesAPIResponse.model_validate(payload)
def _continuation_identity(exc: "ModifyResponseException", responses_so_far: Sequence[object]) -> tuple[str, str, int]:
responses: Final = tuple(
response for item in responses_so_far if (response := stream_item_field(item, "response")) is not None
)
response_id: Final = next(
(rid for response in responses if isinstance(rid := stream_item_field(response, "id"), str) and rid),
f"resp_{uuid.uuid4()}",
)
model: Final = next(
(m for response in responses if isinstance(m := stream_item_field(response, "model"), str) and m),
exc.model,
)
indices: Final = tuple(
index for item in responses_so_far if isinstance(index := stream_item_field(item, "output_index"), int)
)
return response_id, model, max(indices) + 1 if indices else 0
@dataclass(frozen=True, slots=True)
class _OpenItemState:
item_id: str
item_type: str
role: str
output_index: int
content_index: int
text: str
part_open: bool
payload: object
def _open_item_state(responses_so_far: Sequence[object]) -> _OpenItemState | None:
typed: Final = tuple((stream_item_field(event, "type"), event) for event in responses_so_far)
added: Final = tuple(
(added_index, stream_item_field(event, "item"))
for event_type, event in typed
if event_type == "response.output_item.added"
and isinstance(added_index := stream_item_field(event, "output_index"), int)
)
done_indices: Final = frozenset(
done_index
for event_type, event in typed
if event_type == "response.output_item.done"
and isinstance(done_index := stream_item_field(event, "output_index"), int)
)
open_added: Final = tuple((index, payload) for index, payload in added if index not in done_indices)
if not open_added:
return None
output_index, item_payload = open_added[-1]
if item_payload is None:
return None
item_id: Final = stream_item_field(item_payload, "id")
if not isinstance(item_id, str) or not item_id:
return None
raw_type: Final = stream_item_field(item_payload, "type")
raw_role: Final = stream_item_field(item_payload, "role")
part_added: Final = tuple(
part_index
for event_type, event in typed
if event_type == "response.content_part.added"
and stream_item_field(event, "item_id") == item_id
and isinstance(part_index := stream_item_field(event, "content_index"), int)
)
part_done: Final = frozenset(
part_done_index
for event_type, event in typed
if event_type == "response.content_part.done"
and stream_item_field(event, "item_id") == item_id
and isinstance(part_done_index := stream_item_field(event, "content_index"), int)
)
open_parts: Final = tuple(index for index in part_added if index not in part_done)
text: Final = "".join(
delta
for event_type, event in typed
if event_type == "response.output_text.delta"
and stream_item_field(event, "item_id") == item_id
and isinstance(delta := stream_item_field(event, "delta"), str)
)
return _OpenItemState(
item_id=item_id,
item_type=raw_type if isinstance(raw_type, str) and raw_type else "message",
role=raw_role if isinstance(raw_role, str) and raw_role else "assistant",
output_index=output_index,
content_index=open_parts[-1] if open_parts else 0,
text=text,
part_open=bool(open_parts),
payload=item_payload,
)
_item_fields_adapter: Final = TypeAdapter(Mapping[str, object])
_no_item_fields: Final[Mapping[str, object]] = MappingProxyType({})
def _incomplete_item_fields(payload: object) -> Mapping[str, object]:
raw: Final = payload.model_dump() if isinstance(payload, BaseModel) else payload
if not isinstance(raw, dict):
return _no_item_fields
return _item_fields_adapter.validate_python(raw)
def _open_item_closing_events(responses_so_far: Sequence[object]) -> Sequence[ResponsesAPIStreamingResponse]:
"""Close the output item still in progress on the relayed stream before the
block item is appended: strict Responses clients reject a
``response.completed`` that arrives while an earlier ``output_item.added``
was never closed. A message item closes ``completed`` with exactly the text
the client has received so far; any other item type (a function call the
guardrail rejected, for instance) closes ``incomplete`` so the synthetic
done event can never authorize acting on it."""
open_item: Final = _open_item_state(responses_so_far)
if open_item is None:
return ()
if open_item.item_type != "message":
return (
OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=open_item.output_index,
item=BaseLiteLLMOpenAIResponseObject.model_validate(
MappingProxyType({**_incomplete_item_fields(open_item.payload), "status": "incomplete"})
),
),
)
partial_part: Final[_BlockedContentPart] = {
"type": "output_text",
"text": open_item.text,
"annotations": (),
}
closed_payload: Final[_BlockedItemPayload] = {
"type": open_item.item_type,
"id": open_item.item_id,
"status": "completed",
"role": open_item.role,
"content": (partial_part,),
}
item_done: Final = OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=open_item.output_index,
item=GenericResponseOutputItem.model_validate(closed_payload),
)
if not open_item.part_open:
return (item_done,)
partial_done_part: Final[_BlockedDoneContentPart] = {
"type": "output_text",
"text": open_item.text,
"annotations": (),
"logprobs": None,
}
return (
OutputTextDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
item_id=open_item.item_id,
output_index=open_item.output_index,
content_index=open_item.content_index,
text=open_item.text,
),
ContentPartDoneEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
item_id=open_item.item_id,
output_index=open_item.output_index,
content_index=open_item.content_index,
part=ContentPartDonePartOutputText.model_validate(partial_done_part),
),
item_done,
)

2
litellm/llms/openai/vector_stores/transformation.py
View file

@ -21,6 +21,7 @@ from litellm.utils import add_openai_metadata
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -99,6 +100,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id")
url: Final = f"{api_base}/{encoded_vector_store_id}/search"

90
litellm/llms/parallel_ai/search/cost_calculator.py Normal file
View file

@ -0,0 +1,90 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.utils import get_model_info
PARALLEL_AI_DEFAULT_RESULTS: Final = 10
PARALLEL_AI_ADDITIONAL_RESULT_COST: Final = 0.001
PARALLEL_AI_USAGE_PARAM: Final = "_parallel_ai_usage"
PARALLEL_AI_STANDARD_SEARCH_MODEL: Final = "parallel_ai/search"
PARALLEL_AI_FAST_SEARCH_MODEL: Final = "parallel_ai/search-fast"
PARALLEL_AI_TURBO_SEARCH_MODEL: Final = "parallel_ai/search-turbo"
PARALLEL_AI_PRICING_MODEL_BY_MODE: Final[Mapping[str, str]] = MappingProxyType(
{
"fast": PARALLEL_AI_FAST_SEARCH_MODEL,
"turbo": PARALLEL_AI_TURBO_SEARCH_MODEL,
}
)
ADVANCED_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
def _non_negative_int(value: object) -> int | None:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def _usage_count(usage: Sequence[Mapping[str, object]], sku: str) -> int | None:
counts: Final = tuple(
count
for item in usage
if item.get("name") == sku
if (count := _non_negative_int(item.get("count"))) is not None
)
return sum(counts) if counts else None
def _effective_mode(optional_params: Mapping[str, object]) -> str:
mode: Final = optional_params.get("mode")
if isinstance(mode, str):
return mode
processor: Final = optional_params.get("processor")
if processor == "pro":
return "advanced"
return "basic"
def _effective_max_results(optional_params: Mapping[str, object]) -> int:
try:
advanced_settings: Final = ADVANCED_SETTINGS_ADAPTER.validate_python(optional_params.get("advanced_settings"))
advanced_max_results: Final = _non_negative_int(advanced_settings.get("max_results"))
if advanced_max_results is not None:
return advanced_max_results
except ValidationError:
pass
max_results: Final = _non_negative_int(optional_params.get("max_results"))
return max_results if max_results is not None else PARALLEL_AI_DEFAULT_RESULTS
def _request_cost(mode: str) -> float:
pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL)
model_info: Final = get_model_info(model=pricing_model, custom_llm_provider="parallel_ai")
return float(model_info.get("input_cost_per_query") or 0.0)
def _additional_results(
optional_params: Mapping[str, object],
usage: Sequence[Mapping[str, object]] | None,
) -> int:
usage_count: Final = _usage_count(usage, "sku_search_additional_results") if usage is not None else None
if usage_count is not None:
return usage_count
if usage is not None:
return 0
return max(_effective_max_results(optional_params) - PARALLEL_AI_DEFAULT_RESULTS, 0)
def parallel_ai_search_cost(
optional_params: Mapping[str, object],
usage: Sequence[Mapping[str, object]] | None,
) -> float:
request_cost: Final = _request_cost(_effective_mode(optional_params))
request_count_from_usage: Final = _usage_count(usage, "sku_search") if usage is not None else None
request_count: Final = request_count_from_usage if request_count_from_usage is not None else 1
additional_results: Final = _additional_results(optional_params, usage)
return request_count * request_cost + additional_results * PARALLEL_AI_ADDITIONAL_RESULT_COST

143
litellm/llms/parallel_ai/search/transformation.py
View file

@ -4,9 +4,13 @@ Calls Parallel AI's /v1/search endpoint to search the web.
Parallel AI API Reference: https://docs.parallel.ai/api-reference/search/search
"""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypedDict
import httpx
from pydantic import BaseModel, ConfigDict
from typing_extensions import ReadOnly
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
@ -14,9 +18,29 @@ from litellm.llms.base_llm.search.transformation import (
SearchResponse,
SearchResult,
)
from litellm.llms.parallel_ai.search.cost_calculator import PARALLEL_AI_USAGE_PARAM
from litellm.secret_managers.main import get_secret_str
class _ParallelAIV1SearchResult(BaseModel):
model_config = ConfigDict(extra="ignore")
url: str | None = None
title: str | None = None
publish_date: str | None = None
excerpts: Sequence[str] | None = None
class _ParallelAIV1SearchResponse(BaseModel):
model_config = ConfigDict(extra="ignore")
search_id: str | None = None
session_id: str | None = None
results: Sequence[_ParallelAIV1SearchResult] = ()
usage: Sequence[Mapping[str, object]] | None = None
warnings: Sequence[Mapping[str, object]] | None = None
class _ParallelAISourcePolicy(TypedDict, total=False):
include_domains: list[str]
exclude_domains: list[str]
@ -27,10 +51,16 @@ class _ParallelAIExcerptSettings(TypedDict, total=False):
max_chars_per_result: int
class _ParallelAIFetchPolicy(TypedDict, total=False):
max_age_seconds: ReadOnly[int]
timeout_seconds: ReadOnly[float]
disable_cache_fallback: ReadOnly[bool]
class _ParallelAIAdvancedSettings(TypedDict, total=False):
source_policy: _ParallelAISourcePolicy
excerpt_settings: _ParallelAIExcerptSettings
fetch_policy: dict
fetch_policy: _ParallelAIFetchPolicy
location: str
max_results: int
@ -43,14 +73,14 @@ class ParallelAISearchRequest(TypedDict, total=False):
search_queries: list[str] # Required - at least one keyword search query
objective: str # Optional - natural-language description of search goal
mode: str # Optional - 'turbo', 'basic', or 'advanced' (default 'advanced')
mode: str # Optional - 'turbo', 'fast', 'basic', or 'advanced' (default 'advanced')
max_chars_total: int # Optional - upper bound on total excerpt characters
session_id: str # Optional - tracks calls across search/extract requests
client_model: str # Optional - model consuming the results
advanced_settings: _ParallelAIAdvancedSettings
LEGACY_PROCESSOR_TO_MODE: Final = {"base": "basic", "pro": "advanced"}
LEGACY_PROCESSOR_TO_MODE: Final = MappingProxyType({"base": "basic", "pro": "advanced"})
class ParallelAISearchConfig(BaseSearchConfig):
@ -67,16 +97,16 @@ class ParallelAISearchConfig(BaseSearchConfig):
api_base: str | None = None,
**kwargs,
) -> dict:
api_key = self.resolve_server_api_key(
resolved_api_key: Final = self.resolve_server_api_key(
caller_api_key=api_key,
caller_api_base=api_base,
key_env_vars=("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"),
base_env_var="PARALLEL_AI_API_BASE",
default_api_base=self.PARALLEL_AI_API_BASE,
)
if not api_key:
if not resolved_api_key:
raise ValueError("PARALLEL_API_KEY is not set. Set `PARALLEL_API_KEY` environment variable.")
headers["x-api-key"] = api_key
headers["x-api-key"] = resolved_api_key
headers["Content-Type"] = "application/json"
return headers
@ -87,13 +117,12 @@ class ParallelAISearchConfig(BaseSearchConfig):
data: dict | list[dict] | None = None,
**kwargs,
) -> str:
api_base = api_base or get_secret_str("PARALLEL_AI_API_BASE") or self.PARALLEL_AI_API_BASE
resolved_api_base: Final = api_base or get_secret_str("PARALLEL_AI_API_BASE") or self.PARALLEL_AI_API_BASE
api_base = api_base.rstrip("/")
if not api_base.endswith("/v1/search"):
api_base = f"{api_base.removesuffix('/v1')}/v1/search"
return api_base
trimmed: Final = resolved_api_base.rstrip("/")
if trimmed.endswith("/v1/search"):
return trimmed
return f"{trimmed.removesuffix('/v1')}/v1/search"
def transform_search_request(
self,
@ -109,14 +138,17 @@ class ParallelAISearchConfig(BaseSearchConfig):
- If string: maps to `search_queries` (single item) and `objective`
- If list: maps to `search_queries` (keyword queries)
optional_params: Optional parameters for the request
- mode: Search mode ('turbo', 'basic', 'advanced'); defaults to 'basic'
- mode: Search mode ('turbo', 'fast', 'basic', 'advanced'); defaults to 'basic'
- processor: Legacy v1beta param; 'base' maps to mode 'basic', 'pro' to 'advanced'
- max_results: Maximum number of search results -> `advanced_settings.max_results`
- search_domain_filter: Domains to include -> `advanced_settings.source_policy.include_domains`
- search_domain_filter / include_domains: Domains to include -> `advanced_settings.source_policy.include_domains`
- exclude_domains: Domains to exclude -> `advanced_settings.source_policy.exclude_domains`
- country: ISO 3166-1 alpha-2 code -> `advanced_settings.location`
- after_date: RFC 3339 date (YYYY-MM-DD) -> `advanced_settings.source_policy.after_date`
- country / location: ISO 3166-1 alpha-2 code -> `advanced_settings.location`
- max_chars_per_result: -> `advanced_settings.excerpt_settings.max_chars_per_result`
- Any other params are passed through to the request body as-is
- fetch_policy: Cache vs live-fetch policy -> `advanced_settings.fetch_policy`
- Any other params (objective, max_chars_total, session_id, client_model, ...)
are passed through to the request body as-is
Returns:
Dict with request data following the v1 search request spec
@ -137,7 +169,7 @@ class ParallelAISearchConfig(BaseSearchConfig):
mode = LEGACY_PROCESSOR_TO_MODE.get(processor, processor)
# the v1 API defaults to 'advanced' when mode is omitted; default to 'basic'
# instead to keep v1beta's default tier (processor 'base') and litellm's
# $0.004/query cost map entry for `parallel_ai/search` accurate
# cost map entry for `parallel_ai/search` accurate
request_data["mode"] = mode or "basic"
advanced_settings: Final[_ParallelAIAdvancedSettings] = {}
@ -148,17 +180,29 @@ class ParallelAISearchConfig(BaseSearchConfig):
if "country" in params:
advanced_settings["location"] = params.pop("country")
if "location" in params:
advanced_settings["location"] = params.pop("location")
if "max_chars_per_result" in params:
advanced_settings["excerpt_settings"] = {"max_chars_per_result": params.pop("max_chars_per_result")}
if "fetch_policy" in params:
advanced_settings["fetch_policy"] = params.pop("fetch_policy")
source_policy: Final[_ParallelAISourcePolicy] = {}
if "search_domain_filter" in params:
source_policy["include_domains"] = params.pop("search_domain_filter")
if "include_domains" in params:
source_policy["include_domains"] = params.pop("include_domains")
if "exclude_domains" in params:
source_policy["exclude_domains"] = params.pop("exclude_domains")
if "after_date" in params:
source_policy["after_date"] = params.pop("after_date")
if source_policy:
advanced_settings["source_policy"] = source_policy
@ -170,9 +214,11 @@ class ParallelAISearchConfig(BaseSearchConfig):
# unified-spec param with no v1 equivalent
params.pop("max_tokens_per_page", None)
result_data: Final[dict] = dict(request_data)
result_data.update(params)
return result_data
# reserved for the provider's own reported usage, which prices the request;
# a caller-supplied value would otherwise set its own cost
params.pop(PARALLEL_AI_USAGE_PARAM, None)
return {**request_data, **params}
def transform_search_response(
self,
@ -186,26 +232,49 @@ class ParallelAISearchConfig(BaseSearchConfig):
Parallel AI -> LiteLLM mappings:
- results[].title -> SearchResult.title
- results[].url -> SearchResult.url
- results[].excerpts (array) -> SearchResult.snippet (joined string)
- results[].excerpts (array) -> SearchResult.snippet (joined string); the raw
array is preserved as an extra `excerpts` field on each result
- results[].publish_date -> SearchResult.date
- search_id / session_id / warnings are preserved as extra fields on the
response; usage is preserved as `parallel_usage` (the `usage` name is
reserved for LiteLLM's token-usage object)
"""
response_json: Final = raw_response.json()
parsed: Final = _ParallelAIV1SearchResponse.model_validate(raw_response.json())
results: Final = []
for result in response_json.get("results", []):
excerpts = result.get("excerpts") or []
snippet = " ... ".join(excerpts) if excerpts else ""
# written unconditionally: leaving a caller-supplied value in place when the
# provider reports no usage would let the caller price its own request
logging_obj.optional_params = {
**logging_obj.optional_params,
PARALLEL_AI_USAGE_PARAM: parsed.usage,
}
search_result = SearchResult(
title=result.get("title") or "",
url=result.get("url") or "",
snippet=snippet,
date=result.get("publish_date"),
last_updated=None,
results: Final = tuple(
SearchResult.model_validate(
MappingProxyType(
{
"title": result.title or "",
"url": result.url or "",
"snippet": " ... ".join(result.excerpts or ()),
"date": result.publish_date,
"last_updated": None,
"excerpts": result.excerpts or (),
}
)
)
results.append(search_result)
return SearchResponse(
results=results,
object="search",
for result in parsed.results
)
extra_fields: Final = MappingProxyType(
{
key: value
for key, value in (
("search_id", parsed.search_id),
("session_id", parsed.session_id),
("parallel_usage", parsed.usage),
("warnings", parsed.warnings),
)
if value is not None
}
)
return SearchResponse.model_validate(MappingProxyType({"results": results, "object": "search", **extra_fields}))

2
litellm/llms/pg_vector/vector_stores/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
@ -80,6 +81,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id")
url: Final = f"{api_base}/{encoded_vector_store_id}/search"

2
litellm/llms/ragflow/vector_stores/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
@ -92,6 +93,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
"""RAGFlow vector stores are management-only, search is not supported."""
raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval")

38
litellm/llms/s3_vectors/vector_stores/transformation.py
View file

@ -1,8 +1,8 @@
import re
from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.caching._embedding_router import resolve_embedding_router
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.types.router import GenericLiteLLMParams
@ -18,6 +18,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.router import Router
else:
LiteLLMLoggingObj = Any
@ -58,13 +59,20 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
return headers
def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str:
aws_region_name: Final = litellm_params.get("aws_region_name")
if not aws_region_name:
raise ValueError("aws_region_name is required for S3 Vectors")
if not re.match(r"^[a-z][a-z0-9-]*$", aws_region_name):
raise ValueError("Invalid aws_region_name format")
# Resolve region the same way the ingestion path does:
# dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name"))
return f"https://s3vectors.{aws_region_name}.api.aws"
def _resolve_query_embedding_router(self, embedding_model: str, router: "Router | None") -> "Router | None":
"""Return the router iff it serves ``embedding_model`` as a deployment."""
if router is None:
return None
model_list: Final = [
dict(m) for m in (router.get_model_list() or ())
] # mutable-ok: resolve_embedding_router requires list[dict]
return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list)
def transform_search_vector_store_request(
self,
vector_store_id: str,
@ -74,6 +82,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
"""Sync version - generates embedding synchronously."""
# For S3 Vectors, vector_store_id should be in format: bucket_name:index_name
@ -99,10 +108,16 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
# Generate embedding for the query
embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small")
embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router)
import litellm as litellm_module
embedding_response: Final = litellm_module.embedding(model=embedding_model, input=[query])
embedding_input: Final = [query] # mutable-ok: the embedding API takes list input
embedding_response: Final = (
embedding_router.embedding(model=embedding_model, input=embedding_input)
if embedding_router is not None
else litellm_module.embedding(model=embedding_model, input=embedding_input)
)
query_embedding: Final = embedding_response.data[0]["embedding"]
url: Final = f"{api_base}/QueryVectors"
@ -128,6 +143,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict]:
"""Async version - generates embedding asynchronously."""
# For S3 Vectors, vector_store_id should be in format: bucket_name:index_name
@ -153,10 +169,16 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
# Generate embedding for the query asynchronously
embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small")
embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router)
import litellm as litellm_module
embedding_response: Final = await litellm_module.aembedding(model=embedding_model, input=[query])
embedding_input: Final = [query] # mutable-ok: the embedding API takes list input
embedding_response: Final = (
await embedding_router.aembedding(model=embedding_model, input=embedding_input)
if embedding_router is not None
else await litellm_module.aembedding(model=embedding_model, input=embedding_input)
)
query_embedding: Final = embedding_response.data[0]["embedding"]
url: Final = f"{api_base}/QueryVectors"

2
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
if thinking_enabled:
if thinking_budget is None or thinking_budget == 0:
if thinking_budget == 0:
params["includeThoughts"] = False
else:
params["includeThoughts"] = True

3
litellm/llms/vertex_ai/ocr/deepseek_transformation.py
View file

@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
content_item = {"type": "image_url", "image_url": document_url}
# Build DeepSeek OCR request
provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}"
data: Final = {
"model": "deepseek-ai/" + model,
"model": provider_model,
"messages": [{"role": "user", "content": [content_item]}],
}

2
litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
View file

@ -21,6 +21,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -161,6 +162,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: Mapping[str, object] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict[str, object]]:
"""
Transform search request for Vertex AI RAG API

2
litellm/llms/vertex_ai/vector_stores/search_api/transformation.py
View file

@ -25,6 +25,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -245,6 +246,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: Mapping[str, object] | None = None,
router: "Router | None" = None,
) -> tuple[str, dict[str, object]]:
"""
Transform a search request for the Vertex AI Search (Discovery Engine) API.

15
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -153,14 +153,17 @@ class VertexAIAnthropicConfig(AnthropicConfig):
drop_params: bool,
) -> dict:
"""
Override parent method to ensure VertexAI always uses tool-based structured outputs.
VertexAI doesn't support the output_format parameter, so we force all models
to use the tool-based approach for structured outputs.
Override parent method so VertexAI uses tool-based structured outputs
unless the vertex map entry advertises native structured output
(``output_format``, which Vertex AI Claude forwards for those models).
"""
# Temporarily override model name to force tool-based approach
# This ensures Claude Sonnet 4.5 uses tools instead of output_format
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
original_model: Final = model
if "response_format" in non_default_params:
native_structured_output: Final = AnthropicModelInfo._get_provider_resolved_capability(
model, "supports_native_structured_output", "vertex_ai"
)
if "response_format" in non_default_params and native_structured_output is not True:
model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach
# Call parent method with potentially modified model name

22
litellm/main.py
View file

@ -8637,6 +8637,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti
hidden_params["response_cost"] = response_cost
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
return
computed_cost: Final = logging_obj._response_cost_calculator(result=response)
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
setattr(usage, "cost", computed_cost)
def stream_chunk_builder(
chunks: list,
messages: list | None = None,
@ -8731,12 +8741,7 @@ def stream_chunk_builder(
)
break
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(
usage,
"cost",
logging_obj._response_cost_calculator(result=response),
)
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
@ -8915,10 +8920,7 @@ def stream_chunk_builder(
)
break
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)

1018
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

10
litellm/models/user.py
View file

@ -7,7 +7,7 @@ Canonical definition for ``litellm_usertable``. Re-exported from
from datetime import datetime
from pydantic import ConfigDict, Field, model_validator
from pydantic import BaseModel, ConfigDict, Field, model_validator
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.models.organization_membership import (
@ -67,3 +67,11 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase):
if not self.models:
return True
return model_name in self.models
class SCIMPlaceholder(BaseModel):
"""A user row keyed by a value that names another account by SSO identity or email."""
placeholder_user_id: str
resolved_user_ids: tuple[str, ...]
team_ids: tuple[str, ...]

103
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,13 +1,16 @@
import asyncio
import importlib
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
import anyio
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from litellm._logging import verbose_logger
from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT
from litellm.exceptions import (
BlockedPiiEntityError,
GuardrailRaisedException,
@ -18,8 +21,11 @@ from litellm.proxy._experimental.mcp_server.exceptions import (
MCPUpstreamAuthError,
)
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
ServerListOk,
ServerOutcome,
classify_list_exception,
list_fault_http_status,
outcome_wire_value,
)
from litellm.proxy._experimental.mcp_server.ui_session_utils import (
acting_user_auth,
@ -86,8 +92,6 @@ def _connection_error_message(exc: BaseException) -> str:
if MCP_AVAILABLE:
from mcp.types import Tool as MCPTool
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
global_mcp_server_manager,
@ -99,6 +103,7 @@ if MCP_AVAILABLE:
ListMCPToolsRestAPIResponseObject,
MCPInfo,
MCPServer,
_aggregate_server_key, # pyright: ignore[reportPrivateUsage] # same per-server key as the tools/list _meta outcomes
_apply_toolset_scope,
_fire_mcp_tool_call_logging,
execute_mcp_tool,
@ -803,9 +808,6 @@ if MCP_AVAILABLE:
list(allowed_server_ids_set), _rest_client_ip
)
list_tools_result: Final = []
error_message = None
# If server_id is specified, only query that specific server
if server_id:
return await _list_tools_for_single_server(
@ -849,22 +851,19 @@ if MCP_AVAILABLE:
else {}
)
# Query all servers the user has access to
errors: Final = []
for allowed_server_id in allowed_server_ids:
server = global_mcp_server_manager.get_mcp_server_by_id(allowed_server_id)
if server is None:
continue
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
user_oauth_extra_headers = await _get_user_oauth_extra_headers(
async def list_server(
server: MCPServer,
) -> tuple[Sequence[ListMCPToolsRestAPIResponseObject], ServerOutcome]:
server_auth_header: Final = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
user_oauth_extra_headers: Final = await _get_user_oauth_extra_headers(
server,
user_api_key_dict,
prefetched_creds=prefetched_oauth_creds,
)
try:
tools_result = await _get_tools_for_single_server(
tools_result: Final = await _get_tools_for_single_server(
server,
server_auth_header,
raw_headers_from_request,
@ -872,24 +871,36 @@ if MCP_AVAILABLE:
extra_headers=user_oauth_extra_headers,
apply_tool_filters=apply_tool_filters,
)
list_tools_result.extend(tools_result)
except Exception as e:
verbose_logger.exception("Error getting tools from %s: %s", server.name, e)
errors.append(
f"{get_server_prefix(server)}: {classify_list_exception(e).tag}"
if isinstance(e, (MCPServerListError, MCPUpstreamAuthError))
else f"{get_server_prefix(server)}: {e}"
)
continue
return (), classify_list_exception(e)
return tools_result, ServerListOk(tool_count=len(tools_result))
if errors and not list_tools_result:
error_message = "Failed to get tools from servers: " + "; ".join(errors)
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": (error_message if error_message else "Successfully retrieved tools"),
}
# Query all servers the user has access to
queried_servers: Final = tuple(
server
for server in map(global_mcp_server_manager.get_mcp_server_by_id, allowed_server_ids)
if server is not None
)
listings: Final = tuple([await list_server(server) for server in queried_servers])
list_tools_result: Final = [tool for tools, _ in listings for tool in tools]
server_outcomes: Final = MappingProxyType(
{_aggregate_server_key(server): outcome for server, (_, outcome) in zip(queried_servers, listings)}
)
errors: Final = tuple(
f"{key}: {outcome.tag}" for key, outcome in server_outcomes.items() if outcome.tag != "ok"
)
error_message: Final = (
"Failed to get tools from servers: " + "; ".join(errors)
if errors and not list_tools_result
else None
)
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": (error_message if error_message else "Successfully retrieved tools"),
"server_outcomes": {key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items()},
}
except MCPUpstreamAuthError as e:
# Surface upstream pass-through 401/403 challenges to the client so
@ -1173,6 +1184,7 @@ if MCP_AVAILABLE:
transport=request.transport,
auth_type=request.auth_type,
mcp_info=request.mcp_info,
timeout=request.timeout,
command=request.command,
args=request.args,
env=request.env,
@ -1402,11 +1414,28 @@ if MCP_AVAILABLE:
oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers)
async def _list_tools_operation(client):
async def _list_tools_session_operation(session):
return await session.list_tools()
list_tools_response: Final = await client.run_with_session(_list_tools_session_operation)
list_tools_result: Final[list[MCPTool]] = list_tools_response.tools
# Bound the whole pagination walk: without this the preview is limited only by the
# per-request timeout times the page cap. max() keeps the pre-pagination guarantee
# that a single slow page within the client timeout still succeeds, and a
# per-server timeout above the global default extends the deadline with it.
listing_deadline: Final = max(
getattr(client, "timeout", MCP_CLIENT_TIMEOUT) or MCP_CLIENT_TIMEOUT,
MCP_TOOL_LISTING_TIMEOUT,
)
list_tools_result = None # rebind-ok: set inside the timeout scope below
with anyio.move_on_after(listing_deadline):
list_tools_result = await client.list_tools(raise_on_error=True) # rebind-ok: fills the init above
if list_tools_result is None:
verbose_logger.warning(
"MCP tools/list preview timed out after %s seconds while paginating upstream tools",
listing_deadline,
)
return { # mutable-ok: error response payload
"status": "error",
"error": True,
"message": f"Timed out listing tools after {listing_deadline} seconds. "
"The MCP server may be responding slowly or paginating excessively.",
}
model_dumped_tools: Final[list[dict]] = [tool.model_dump() for tool in list_tools_result]
return {
"tools": model_dumped_tools,

179
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -32002,6 +32002,62 @@
"title": "SCIMPatchOperation",
"type": "object"
},
"SCIMPlaceholder": {
"description": "A user row keyed by a value that names another account by SSO identity or email.",
"properties": {
"placeholder_user_id": {
"title": "Placeholder User Id",
"type": "string"
},
"resolved_user_ids": {
"items": {
"type": "string"
},
"title": "Resolved User Ids",
"type": "array"
},
"team_ids": {
"items": {
"type": "string"
},
"title": "Team Ids",
"type": "array"
}
},
"required": [
"placeholder_user_id",
"resolved_user_ids",
"team_ids"
],
"title": "SCIMPlaceholder",
"type": "object"
},
"SCIMPlaceholderMergeResult": {
"properties": {
"merged_into_user_id": {
"title": "Merged Into User Id",
"type": "string"
},
"placeholder_user_id": {
"title": "Placeholder User Id",
"type": "string"
},
"team_ids": {
"items": {
"type": "string"
},
"title": "Team Ids",
"type": "array"
}
},
"required": [
"placeholder_user_id",
"merged_into_user_id",
"team_ids"
],
"title": "SCIMPlaceholderMergeResult",
"type": "object"
},
"SCIMServiceProviderConfig": {
"properties": {
"authenticationSchemes": {
@ -33641,6 +33697,129 @@
"scim"
]
}
},
"/scim/v2/placeholders": {
"get": {
"description": "List user rows whose id is another account's SSO identity or email.\n\nAn earlier release provisioned a group member it could not match as a user keyed\nby the raw member value, and that row now shadows the account the value really\nnames, so every push of that member is refused. This lists those rows so an\noperator can fold each one into the account it shadows with\n``POST /scim/v2/placeholders/{user_id}/merge``. A row that has an SSO identity of\nits own or owns virtual keys is left out: someone uses that account.",
"operationId": "list_placeholders_scim_v2_placeholders_get",
"parameters": [
{
"in": "query",
"name": "feature",
"required": false,
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Feature"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/SCIMPlaceholder"
},
"title": "Response List Placeholders Scim V2 Placeholders Get",
"type": "array"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "List Placeholders",
"tags": [
"scim"
]
}
},
"/scim/v2/placeholders/{user_id}/merge": {
"post": {
"description": "Fold a placeholder user into the one account its id names by SSO identity or email.\n\nThe account is added to every team the placeholder is on, then the placeholder is\ndeleted the way ``DELETE /scim/v2/Users/{id}`` deletes a user, so the next group\npush resolves the member value to the real account. Refused with 409 when the row\nhas an SSO identity of its own, owns virtual keys, or names no account or several.",
"operationId": "merge_placeholder_scim_v2_placeholders__user_id__merge_post",
"parameters": [
{
"in": "path",
"name": "user_id",
"required": true,
"schema": {
"title": "User ID",
"type": "string"
}
},
{
"in": "query",
"name": "feature",
"required": false,
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Feature"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SCIMPlaceholderMergeResult"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Merge Placeholder",
"tags": [
"scim"
]
}
}
}
},

26
litellm/proxy/_types.py
View file

@ -1216,6 +1216,13 @@ class GenerateKeyRequest(KeyRequestBase):
organization_id: str | None = None
project_id: str | None = None
@field_validator("team_id", mode="before")
@classmethod
def treat_cleared_team_id_as_unset(cls, v: object) -> object:
if v == "":
return None
return v
class GenerateKeyResponse(KeyRequestBase):
key: str
@ -2436,9 +2443,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
database_socket_timeout: float | None = Field(
None,
description=(
"Prisma `socket_timeout` URL param (seconds). When set, an idle/slow "
"connection that has not produced data within this window is closed. "
"This is the main knob for capping idle DB connections from LiteLLM."
"Prisma `socket_timeout` URL param (seconds). When set, an in-flight "
"operation that has not produced data within this window is aborted. "
"For capping how long idle pooled connections are kept, see "
"`database_max_idle_connection_lifetime`."
),
)
database_max_idle_connection_lifetime: float | None = Field(
60,
description=(
"Prisma `max_idle_connection_lifetime` URL param (seconds). A pooled "
"connection idle longer than this is closed and replaced instead of "
"being handed to the next request. Defaults to 60 so connections are "
"recycled before common infra idle timeouts (AWS NLB / RDS Proxy "
"~350s, many LBs 60-350s) silently drop them and requests fail with "
"`Error { kind: Closed }`. A value pinned on the DATABASE_URL or set "
"via `database_extra_connection_params` takes precedence."
),
)
database_extra_connection_params: dict[str, Any] | None = Field(

2
litellm/proxy/agent_endpoints/a2a_endpoints.py
View file

@ -1019,4 +1019,6 @@ async def invoke_agent_a2a(
)
except Exception:
pass
if isinstance(e, litellm.BadRequestError):
return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)

174
litellm/proxy/anthropic_endpoints/streaming_model_restamp.py Normal file
View file

@ -0,0 +1,174 @@
"""
Restamp the public ``model`` on the Anthropic Messages ``message_start`` event, the only
stream event carrying a model, so streamed responses report the requested model like
non-streaming ones do.
Chunks reach the serializer either as already-encoded SSE frames (``bytes``/``str``, the
provider passthrough path) or as event dicts (fake-stream and agentic paths).
"""
import json
import re
from collections.abc import Mapping
from typing import Final
from pydantic import TypeAdapter, ValidationError
_MESSAGE_START_EVENT: Final = "message_start"
_MESSAGE_START_MARKER: Final = b"message_start"
_SSE_DATA_FIELD: Final = "data:"
_SSE_FRAME_END_PATTERN: Final = re.compile(rb"\r\n\r\n|\r\r|\n\n")
_MAX_HELD_BYTES: Final = 65536
_PING_MARKERS: Final = (b"event: ping", b'"type": "ping"', b'"type":"ping"')
_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _restamped_event(event: Mapping[str, object], requested_model: str) -> Mapping[str, object] | None:
message: Final = event.get("message")
if event.get("type") != _MESSAGE_START_EVENT or not isinstance(message, dict):
return None
if message.get("model") == requested_model:
return None
return {**event, "message": {**message, "model": requested_model}} # mutable-ok: SSE payload, re-serialized as is
def _restamped_data_line(line: str, requested_model: str) -> str | None:
stripped: Final = line.strip()
if not stripped.startswith(_SSE_DATA_FIELD):
return None
payload: Final = stripped[len(_SSE_DATA_FIELD) :].strip()
if not payload or payload == "[DONE]":
return None
try:
event: Final = _EVENT_ADAPTER.validate_json(payload)
except ValidationError:
return None
restamped: Final = _restamped_event(event, requested_model)
if restamped is None:
return None
terminator: Final = line[len(line.rstrip("\r\n")) :]
return f"data: {json.dumps(restamped, separators=(',', ':'))}{terminator}"
def _restamped_frame(frame: str, requested_model: str) -> str | None:
lines: Final = frame.splitlines(keepends=True)
restamped: Final = tuple(_restamped_data_line(line, requested_model) for line in lines)
if all(line is None for line in restamped):
return None
return "".join(new if new is not None else old for new, old in zip(restamped, lines))
def restamp_anthropic_stream_chunk_model(chunk: object, requested_model: str) -> object:
"""
Return ``chunk`` with the ``message_start`` model replaced by ``requested_model``.
Chunks that carry no model are returned unchanged.
"""
if isinstance(chunk, dict):
try:
event: Final = _EVENT_ADAPTER.validate_python(chunk)
except ValidationError:
return chunk
return _restamped_event(event, requested_model) or chunk
if isinstance(chunk, (bytes, bytearray)):
if _MESSAGE_START_EVENT.encode() not in chunk:
return chunk
restamped_bytes: Final = _restamped_frame(chunk.decode("utf-8", errors="ignore"), requested_model)
return chunk if restamped_bytes is None else restamped_bytes.encode("utf-8")
if isinstance(chunk, str):
if _MESSAGE_START_EVENT not in chunk:
return chunk
restamped_text: Final = _restamped_frame(chunk, requested_model)
return chunk if restamped_text is None else restamped_text
return chunk
def _is_ping_frame(frame: bytes) -> bool:
return any(marker in frame for marker in _PING_MARKERS)
class AnthropicStreamModelRestamper:
"""
Per-stream restamper for the encoded passthrough path, where chunks are raw
transport reads: the ``message_start`` SSE frame can arrive split across
chunks or coalesced with later frames. Complete frames (``\\n\\n``,
``\\r\\n\\r\\n``, or ``\\r\\r`` terminated) are emitted as their terminator
closes them and an incomplete tail is held until it completes, so the
restamp never misses a torn frame; ``flush`` returns whatever is still held
when the stream ends so no bytes are swallowed. Once ``message_start`` has
been handled, or the first real event proves the stream carries none, every
later chunk passes through untouched.
"""
def __init__(self, requested_model: str) -> None:
self._requested_model: Final = requested_model
self._held = b""
self._armed = True
def process(self, chunk: object) -> object:
if not self._armed:
return chunk
if isinstance(chunk, (bytes, bytearray)):
return self._process_encoded(bytes(chunk))
if isinstance(chunk, str):
return self._process_encoded(chunk.encode("utf-8"))
restamped: Final = restamp_anthropic_stream_chunk_model(chunk, self._requested_model)
if isinstance(chunk, dict) and chunk.get("type") not in (None, "ping"):
self._armed = False
return restamped
def flush(self) -> bytes:
held: Final = self._held
self._held = b""
self._armed = False
if not held:
return b""
restamped: Final = restamp_anthropic_stream_chunk_model(held, self._requested_model)
return restamped if isinstance(restamped, bytes) else held
def _process_encoded(self, data: bytes) -> bytes:
combined: Final = self._held + data
boundaries: Final = tuple(match.end() for match in _SSE_FRAME_END_PATTERN.finditer(combined))
if not boundaries:
if len(combined) > _MAX_HELD_BYTES:
self._held = b""
self._armed = False
return combined
self._held = combined
return b""
emitted: Final = self._restamped_closed_block(combined[: boundaries[-1]])
tail: Final = combined[boundaries[-1] :]
if not self._armed:
self._held = b""
return emitted + tail
self._held = tail
return emitted
def _restamped_closed_block(self, closed: bytes) -> bytes:
boundaries: Final = tuple(match.end() for match in _SSE_FRAME_END_PATTERN.finditer(closed))
frames: Final = tuple(closed[start:end] for start, end in zip((0, *boundaries[:-1]), boundaries))
decider: Final = next(
(
index
for index, frame in enumerate(frames)
if _MESSAGE_START_MARKER in frame or (b"data:" in frame and not _is_ping_frame(frame))
),
None,
)
if decider is None:
return closed
self._armed = False
if _MESSAGE_START_MARKER not in frames[decider]:
return closed
restamped_text: Final = _restamped_frame(
frames[decider].decode("utf-8", errors="ignore"), self._requested_model
)
if restamped_text is None:
return closed
return b"".join(
restamped_text.encode("utf-8") if index == decider else frame for index, frame in enumerate(frames)
)

25
litellm/proxy/client/cli/commands/agents.py
View file

@ -9,6 +9,7 @@ import click
import requests
from .auth import context_secret_vault, get_stored_api_key, login
from .cmd_quoting import quote_for_cmd
ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL"
ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN"
@ -151,31 +152,9 @@ def verify_proxy_key(
_WINDOWS_SHIM_SUFFIXES: Final[frozenset[str]] = frozenset({".cmd", ".bat"})
_CMD_PERCENT_GUARD: Final = "%%cd:~,%"
_CMD_LINE_BREAKS: Final = ("\r", "\n")
def _double_trailing_backslashes(segment: str) -> str:
bare: Final = segment.rstrip("\\")
return bare + "\\" * 2 * (len(segment) - len(bare))
def _quote_for_cmd(token: str) -> str:
"""Quote one token so both parsers that read it see the original text.
Follows the algorithm the Rust standard library settled on for batch files
after CVE-2024-24576. Two parsers see this token: cmd.exe, which ends a
quoted string on a lone `"` and so wants an embedded one doubled, and the
shim's own interpreter, which re-splits `%*` under C runtime rules where a
backslash escapes the quote that follows it, so every backslash run standing
before a quote is doubled. Quoting cannot stop cmd expanding `%VAR%`, so each
`%` is prefixed with `%%cd:~,`: the zero-length substring of the always
defined `cd` expands to nothing and leaves no `%` pair for cmd to match.
"""
escaped: Final = '""'.join(_double_trailing_backslashes(part) for part in token.split('"'))
return '"' + escaped.replace("%", _CMD_PERCENT_GUARD) + '"'
def _windows_command(path: str, args: Sequence[str]) -> str | tuple[str, ...]:
"""Build what CreateProcess runs, routing batch shims through cmd.exe.
@ -202,7 +181,7 @@ def _windows_command(path: str, args: Sequence[str]) -> str | tuple[str, ...]:
f"Cannot pass an argument containing a line break to `{os.path.basename(path)}` on "
"Windows: cmd.exe ends the command line there, so the agent would silently lose it."
)
inner: Final = " ".join(_quote_for_cmd(token) for token in (path, *rest))
inner: Final = " ".join(quote_for_cmd(token) for token in (path, *rest))
return f'cmd.exe /d /e:on /v:off /s /c "{inner}"'

11
litellm/proxy/client/cli/commands/claude_settings.py
View file

@ -8,6 +8,7 @@ live here rather than in either command module.
import shlex
import shutil
import sys
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
@ -17,6 +18,8 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.litellm_core_utils.private_json import write_private_json
from .cmd_quoting import quote_for_cmd
ENV_KEY: Final = "env"
API_KEY_HELPER_KEY: Final = "apiKeyHelper"
ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL"
@ -87,9 +90,12 @@ def merge_claude_settings(
return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper}
def resolve_api_key_helper(base_url: str) -> str:
def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str:
"""Build the shell command Claude Code should run for its apiKeyHelper.
Claude Code hands the string to the system shell, `sh` on POSIX and cmd.exe
on Windows, so every token is quoted for the shell that will read it.
Resolves `lite` to an absolute path so the helper works regardless of the
PATH visible to whatever subprocess Claude Code spawns it from. Passing
--base-url explicitly (rather than relying on the bare invocation Claude
@ -106,7 +112,8 @@ def resolve_api_key_helper(base_url: str) -> str:
raise ClaudeSettingsError(
"Could not find `lite` on your PATH. Claude Code's apiKeyHelper needs an absolute path to it."
)
return f"{shlex.quote(lite_path)} --base-url {shlex.quote(base_url)} auth print-token"
quote: Final = quote_for_cmd if platform.startswith("win") else shlex.quote
return " ".join(quote(token) for token in (lite_path, "--base-url", base_url, "auth", "print-token"))
def write_claude_settings(base_url: str, settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None:

26
litellm/proxy/client/cli/commands/cmd_quoting.py Normal file
View file

@ -0,0 +1,26 @@
"""Quoting for command lines that cmd.exe reads before handing them to a program."""
from typing import Final
_CMD_PERCENT_GUARD: Final = "%%cd:~,%"
def _double_trailing_backslashes(segment: str) -> str:
bare: Final = segment.rstrip("\\")
return bare + "\\" * 2 * (len(segment) - len(bare))
def quote_for_cmd(token: str) -> str:
"""Quote one token so both parsers that read it see the original text.
Follows the algorithm the Rust standard library settled on for batch files
after CVE-2024-24576. Two parsers see this token: cmd.exe, which ends a
quoted string on a lone `"` and so wants an embedded one doubled, and the
program's own C runtime argv split, where a backslash escapes the quote that
follows it, so every backslash run standing before a quote is doubled.
Quoting cannot stop cmd expanding `%VAR%`, so each `%` is prefixed with
`%%cd:~,`: the zero-length substring of the always defined `cd` expands to
nothing and leaves no `%` pair for cmd to match.
"""
escaped: Final = '""'.join(_double_trailing_backslashes(part) for part in token.split('"'))
return '"' + escaped.replace("%", _CMD_PERCENT_GUARD) + '"'

36
litellm/proxy/common_request_processing.py
View file

@ -176,6 +176,9 @@ if TYPE_CHECKING:
ProxyConfig = _ProxyConfig
else:
ProxyConfig = Any
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
AnthropicStreamModelRestamper,
)
from litellm.proxy.litellm_pre_call_utils import (
add_litellm_data_to_request,
refresh_proxy_server_request_body_snapshot,
@ -2490,6 +2493,9 @@ class ProxyBaseLLMRequestProcessing:
request_data=self.data,
proxy_logging_obj=proxy_logging_obj,
request=request,
restamp_model=(
None if _should_return_raw_model_name(self.data) else requested_model_from_client
),
)
return await create_response(
generator=wrap_sse_stream_with_keepalive_pings(
@ -3442,6 +3448,16 @@ class ProxyBaseLLMRequestProcessing:
else:
return chunk
@staticmethod
def _sse_chunk_serializer(restamper: AnthropicStreamModelRestamper | None) -> StreamChunkSerializer:
if restamper is None:
return ProxyBaseLLMRequestProcessing.return_sse_chunk
def serialize(chunk: object) -> str:
return ProxyBaseLLMRequestProcessing.return_sse_chunk(restamper.process(chunk))
return serialize
@staticmethod
async def _finalize_streaming_generator_cleanup(
request: Request | None,
@ -3502,11 +3518,16 @@ class ProxyBaseLLMRequestProcessing:
serialize_chunk: StreamChunkSerializer,
serialize_error: StreamErrorSerializer,
request: Request | None = None,
flush_tail: Callable[[], bytes] | None = None,
) -> AsyncGenerator[str, None]:
"""
Shared streaming data generator: runs proxy iterator hook, per-chunk hook,
cost injection, then yields chunks via serialize_chunk; on exception runs
failure hook and yields via serialize_error. Use for SSE or NDJSON.
``flush_tail`` runs once after the upstream iterator completes cleanly and
its non-empty result is yielded, so a serializer that buffers bytes across
chunks can emit anything still held at end of stream.
"""
verbose_proxy_logger.debug("inside generator")
# Resolve per-stream (not per-chunk) whether the heavy per-chunk path
@ -3569,6 +3590,9 @@ class ProxyBaseLLMRequestProcessing:
# so it must not suppress that refund.
delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
yield serialize_chunk(chunk)
held_tail: Final = flush_tail() if flush_tail is not None else b""
if held_tail:
yield serialize_chunk(held_tail)
stream_completed = True
except (asyncio.CancelledError, GeneratorExit):
# Client disconnected mid-stream. CancelledError / GeneratorExit
@ -3579,8 +3603,7 @@ class ProxyBaseLLMRequestProcessing:
# billing and release exactly once. This is the outermost generator
# Starlette closes on disconnect, so the nested iterator hook (which
# only sees GeneratorExit on GC) cannot own the refund.
if not stream_completed:
client_disconnected = True
client_disconnected = not stream_completed
if not delivered_chunk and not _withheld_provider_output(response):
from litellm.proxy.spend_tracking.budget_reservation import (
release_budget_reservation_on_cancel,
@ -3634,6 +3657,7 @@ class ProxyBaseLLMRequestProcessing:
request_data: dict,
proxy_logging_obj: ProxyLogging,
request: Request | None = None,
restamp_model: str | None = None,
) -> AsyncGenerator[str, None]:
"""
Anthropic /messages and Google /generateContent streaming data generator require SSE events.
@ -3642,17 +3666,23 @@ class ProxyBaseLLMRequestProcessing:
SSE serializers directly (rather than re-wrapping it in another
``async for: yield`` trampoline), so a streamed chunk traverses one
fewer async-generator layer / coroutine resume on the hot path.
``restamp_model`` publishes that name on the Anthropic ``message_start``
event in place of the provider's model, matching what the non-streaming
response reports.
"""
restamper: Final = AnthropicStreamModelRestamper(restamp_model) if restamp_model else None
return ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
response=response,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
proxy_logging_obj=proxy_logging_obj,
serialize_chunk=ProxyBaseLLMRequestProcessing.return_sse_chunk,
serialize_chunk=ProxyBaseLLMRequestProcessing._sse_chunk_serializer(restamper),
serialize_error=lambda proxy_exc: (
f"{STREAM_SSE_DATA_PREFIX}{json.dumps({'error': proxy_exc.to_dict()})}\n\n"
),
request=request,
flush_tail=None if restamper is None else restamper.flush,
)
@overload

25
litellm/proxy/common_utils/model_listing_utils.py
View file

@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`.
from __future__ import annotations
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
if TYPE_CHECKING:
from litellm.router import Router
def configured_display_names(
entries: Sequence[tuple[str, str]],
llm_router: Router | None,
) -> Mapping[str, str]:
"""response_id -> configured `model_info.display_name` for the listing entries
that have one.
Metadata is looked up by each entry's internal lookup id (so team-scoped rows
resolve), while the returned map is keyed by the public response id the
Anthropic-shaped listing is built from. Entries without a configured name are
omitted so the listing falls back to the id itself.
"""
if llm_router is None:
return MappingProxyType({})
resolved: Final = (
(response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries
)
return MappingProxyType(
{response_id: display_name for response_id, display_name in resolved if display_name is not None}
)
class TeamModelNameTranslator:
"""Translates internal team routing keys to their public names for the model
listing/retrieve responses. Stateless; the live router and general_settings

20
litellm/proxy/db/db_url_settings.py
View file

@ -82,10 +82,30 @@ CONNECTION_PARAM_KEYS: Final[frozenset[str]] = frozenset(
"pool_timeout",
"connect_timeout",
"socket_timeout",
"max_idle_connection_lifetime",
"pgbouncer",
}
)
# Quaint never tests pooled connections on checkout and keeps them idle for
# 300s by default, past many infra idle timeouts, so dead sockets surface as
# `Error { kind: Closed }`. 60s recycles them first; explicit values win.
DEFAULT_MAX_IDLE_CONNECTION_LIFETIME: Final = 60
IDLE_LIFETIME_DEFAULT_PARAMS: Final[Mapping[str, int]] = MappingProxyType(
{"max_idle_connection_lifetime": DEFAULT_MAX_IDLE_CONNECTION_LIFETIME}
)
def idle_lifetime_params(configured: float | None) -> Mapping[str, str | int | float]:
"""The `max_idle_connection_lifetime` to add to URLs that do not pin one.
Applied via ``add_missing_query_params`` so a URL-pinned value always wins,
whether the operator configured `database_max_idle_connection_lifetime` or not.
"""
if configured is None:
return IDLE_LIFETIME_DEFAULT_PARAMS
return MappingProxyType({"max_idle_connection_lifetime": configured})
def add_missing_query_params(url: str, params: Mapping[str, str | int | float]) -> str:
"""Return ``url`` with the ``params`` it does not already carry appended.

29
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -31,6 +31,7 @@ from litellm.caching import DualCache
from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS
from litellm.exceptions import ModifyResponseException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys
from litellm.litellm_core_utils.litellm_logging import (
_get_masked_values, # pyright: ignore[reportPrivateUsage] # the shared header-masking helper has no public name
@ -215,6 +216,16 @@ def _redact_assessment_match_fields(assessments: list[dict]) -> list[dict]:
return redacted if isinstance(redacted, list) else assessments
_RESPONSES_API_CALL_TYPES: Final = frozenset({CallTypes.responses, CallTypes.aresponses})
def _is_responses_api_route(request_route: str | None) -> bool:
if request_route is None:
return False
call_types: Final = get_call_types_for_route(request_route)
return call_types is not None and any(call_type in _RESPONSES_API_CALL_TYPES for call_type in call_types)
class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# During-call must use async_moderation_hook (not unified apply_guardrail), otherwise
# OpenAI translation always passes input_type="request" and spend/UI show PRE-CALL.
@ -2709,6 +2720,24 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
yield streamed_chunk
return
# Responses-API events are neither chat-completions chunks nor raw
# Anthropic SSE, so the assembly below cannot scan them; the unified
# guardrail's translation layer can, with buffering semantics kept.
if _is_responses_api_route(user_api_key_dict.request_route):
from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
UnifiedLLMGuardrails,
)
async for translated_chunk in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=response,
request_data=request_data,
guardrail_to_apply=self,
buffer_until_moderated_default=True,
):
yield translated_chunk
return
# Import here to avoid circular imports
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.main import stream_chunk_builder

120
litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
View file

@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard
import httpx
from fastapi import HTTPException
from httpx import Response as HttpxResponse
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -52,6 +53,10 @@ BYPASS_HEADER: Final = "x-headroom-bypass"
HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve"
_HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})")
_HASH_CACHE_TTL_SECONDS: Final = 15 * 60
# Narrows the base class's bare-dict ``request_data`` at the boundary so its
# untranslated messages can be read with concrete types (values pass through by
# reference, so this is a shallow top-level reconstruction).
_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object])
def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip
@ -116,16 +121,119 @@ def _restore_content_shapes(
return restored
def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]:
def _tool_call_name(tool_call: Mapping[str, object]) -> str | None:
function: Final = tool_call.get("function")
if not _is_str_object_dict(function):
return None
name: Final = function.get("name")
return name if isinstance(name, str) else None
def _is_retrieve_tool_name(name: str | None) -> bool:
"""Match the retrieve tool whether called directly or via the MCP gateway.
Server-side the tool is ``headroom_retrieve``; exposed through LiteLLM's MCP
gateway a client calls it as ``mcp__<server>__headroom_retrieve``.
"""
return name is not None and (
name == HEADROOM_RETRIEVE_TOOL_NAME or name.endswith(f"__{HEADROOM_RETRIEVE_TOOL_NAME}")
)
def _retrieve_call_ids_in_message(message: Mapping[str, object]) -> frozenset[str]:
if message.get("role") != "assistant":
return frozenset()
tool_calls: Final = message.get("tool_calls")
if not _is_object_list(tool_calls):
return frozenset()
return frozenset(
str(tool_call["id"])
for tool_call in tool_calls
if _is_str_object_dict(tool_call) and tool_call.get("id") and _is_retrieve_tool_name(_tool_call_name(tool_call))
)
def _anthropic_tool_use_retrieve_id(block: object) -> str | None:
if not _is_str_object_dict(block) or block.get("type") != "tool_use":
return None
name: Final = block.get("name")
call_id: Final = block.get("id")
if isinstance(name, str) and call_id is not None and _is_retrieve_tool_name(name):
return str(call_id)
return None
def _anthropic_retrieve_ids_in_message(message: Mapping[str, object]) -> frozenset[str]:
content: Final = message.get("content")
if not _is_object_list(content):
return frozenset()
return frozenset(call_id for block in content if (call_id := _anthropic_tool_use_retrieve_id(block)) is not None)
def _raw_retrieve_call_ids(messages: object) -> frozenset[str]:
"""Retrieve-tool call ids read from the request's own, untranslated messages.
The guardrail otherwise scans an OpenAI-translated view where a tool name
over 64 chars is truncated to ``{prefix}_{hash}``, which drops the
``__headroom_retrieve`` suffix a long ``mcp__<server>__`` prefix pushes past
the limit. Tool-call ids are never truncated, so pairing the tool result to
an id read from the original request keeps the match intact. Both wire
shapes are handled: OpenAI ``tool_calls`` and Anthropic ``tool_use`` blocks.
"""
if not _is_object_list(messages):
return frozenset()
return frozenset(
call_id
for message in messages
if _is_str_object_dict(message)
for call_id in _retrieve_call_ids_in_message(message) | _anthropic_retrieve_ids_in_message(message)
)
def _retrieval_result_indices(
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
) -> frozenset[int]:
"""Indices of tool-result rows that carry ``headroom_retrieve`` output.
When the retrieve tool is exposed to a client that runs its own tool loop
(the LiteLLM MCP gateway path), the client executes the call and sends the
recovered original content back as a tool result on the next turn. That
content is exactly what a prior compression stubbed, so compressing it again
re-derives the identical content hash: a no-op that strands the model on the
marker and loops the agent. Hold those rows back so the expansion survives.
``extra_retrieve_call_ids`` carries ids recovered from the untruncated
request so the pairing survives tool-name truncation (see
``_raw_retrieve_call_ids``).
"""
retrieve_call_ids: Final = extra_retrieve_call_ids | frozenset(
call_id for message in messages for call_id in _retrieve_call_ids_in_message(message)
)
if not retrieve_call_ids:
return frozenset()
return frozenset(
index
for index, message in enumerate(messages)
if message.get("role") in ("tool", "function") and str(message.get("tool_call_id")) in retrieve_call_ids
)
def _protected_indices(
messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset()
) -> frozenset[int]:
"""Indices headroom must not send to the compression service.
``get_protected_indices`` is litellm's own compression policy: the system
rows, the last user row, the last assistant row. It is expanded over whole
rows, the last user row, the last assistant row. Rows carrying just-retrieved
``headroom_retrieve`` output are added so re-compression can't collapse them
back to the marker they were expanded from. The union is expanded over whole
tool exchanges the way ``compress()`` expands it, so a protected assistant
tool call cannot end up answered by a marker standing in for the result the
model just asked for.
"""
protected: Final = frozenset(get_protected_indices(messages))
protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices(
messages, extra_retrieve_call_ids
)
return protected | frozenset(
index
for group in group_tool_exchanges(messages)
@ -634,7 +742,11 @@ class HeadroomGuardrail(CustomGuardrail):
# /v1/compress grows a field for sending the live turn as the retrieval
# query without compressing it: query-aware compression reads the newest
# user message, so it is withheld here at some cost to history ranking.
protected_indices: Final = _protected_indices(messages)
# request_data is a bare dict on the base signature; narrow it before
# reading the untranslated messages so long tool names can be recovered.
raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages")
raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages)
protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids)
compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices]
if not compressible:
return inputs

35
litellm/proxy/hooks/model_max_budget_limiter.py
View file

@ -1,6 +1,6 @@
import json
import time
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
@ -199,18 +199,10 @@ async def build_model_max_budget_usage(
)
for budget_model, budget_config in budgets
)
batched: Final = await cache.async_batch_get_cache(
keys=list(spend_keys) # mutable-ok: async_batch_get_cache annotates keys as list, so one must exist here
)
# async_batch_get_cache returns None if it fails internally, and its result is
# index-aligned with `keys` otherwise. An unusable result reads as a miss,
# which is what a never-written counter already reads as.
current_spends: Final = (
tuple(batched) if isinstance(batched, list) and len(batched) == len(budgets) else (None,) * len(budgets)
)
current_spends: Final = await _current_window_spends(cache=cache, spend_keys=spend_keys)
return {
budget_model: {
"current_spend": round(_as_spend(current_spend), 4),
"current_spend": round(current_spend, 4),
"budget_limit": budget_config.max_budget,
"time_period": budget_config.budget_duration,
}
@ -218,6 +210,22 @@ async def build_model_max_budget_usage(
}
async def _current_window_spends(cache: DualCache, spend_keys: Sequence[str]) -> tuple[float, ...]:
"""Redis holds the window total across replicas; the in-memory copy is one replica's share."""
keys: Final = list(spend_keys) # mutable-ok: both batch readers annotate their key argument as list
redis_cache: Final = cache.redis_cache
if redis_cache is not None:
shared: Final = await redis_cache.async_batch_get_cache(key_list=keys)
return tuple(_as_spend(shared.get(key)) for key in keys)
# async_batch_get_cache returns None if it fails internally, and its result is
# index-aligned with `keys` otherwise. An unusable result reads as a miss,
# which is what a never-written counter already reads as.
batched: Final = await cache.async_batch_get_cache(keys=keys)
if not isinstance(batched, list) or len(batched) != len(keys):
return (0.0,) * len(keys)
return tuple(_as_spend(current_spend) for current_spend in batched)
def _usable_budget_config(raw_budget_config: object) -> BudgetConfig | None:
try:
budget_config: Final = BudgetConfig.model_validate(raw_budget_config)
@ -404,7 +412,10 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
async def _cached_spend(self, spend_key: str) -> float | None:
return await self.dual_cache.async_get_cache(key=spend_key)
redis_cache: Final = self.dual_cache.redis_cache
if redis_cache is None:
return await self.dual_cache.async_get_cache(key=spend_key)
return await redis_cache.async_get_cache(key=spend_key)
async def async_filter_deployments(
self,

48
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -54,7 +54,10 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
publish_config_change,
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
from litellm.proxy.management_endpoints.team_endpoints import (
@ -701,6 +704,12 @@ async def patch_model(
param="blocked",
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=patch_data.litellm_params,
user_api_key_dict=user_api_key_dict,
existing_litellm_params=db_model.litellm_params,
)
_raise_on_strategy_router_write_violation(
incoming_params=patch_data.litellm_params,
existing_params=db_model.litellm_params,
@ -1464,6 +1473,32 @@ class ModelManagementAuthChecks:
)
return True
@staticmethod
def can_user_attach_credential(
litellm_params: GenericLiteLLMParams | None,
user_api_key_dict: UserAPIKeyAuth,
existing_litellm_params: GenericLiteLLMParams | None = None,
) -> Literal[True]:
if litellm_params is None or litellm_params.litellm_credential_name is None:
return True
if existing_litellm_params is not None and existing_litellm_params.litellm_credential_name is not None:
existing_credential_name: Final = decrypt_value_helper(
value=existing_litellm_params.litellm_credential_name,
key="litellm_credential_name",
exception_type="debug",
return_original_value=True,
)
if litellm_params.litellm_credential_name == existing_credential_name:
return True
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
return True
raise ProxyException(
message=f"Only a proxy admin can attach a stored credential (litellm_credential_name) to a model. Your role={user_api_key_dict.user_role}.",
type=ProxyErrorTypes.auth_error.value,
code=status.HTTP_403_FORBIDDEN,
param="litellm_credential_name",
)
@staticmethod
async def allow_team_model_action(
model_params: Deployment | updateDeployment,
@ -1786,6 +1821,11 @@ async def add_new_model(
premium_user=premium_user,
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
user_api_key_dict=user_api_key_dict,
)
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=None,
@ -1958,6 +1998,12 @@ async def update_model(
premium_user=premium_user,
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
user_api_key_dict=user_api_key_dict,
existing_litellm_params=deployment.litellm_params,
)
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=deployment.litellm_params,

124
litellm/proxy/management_endpoints/scim/scim_v2.py
View file

@ -29,6 +29,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.models.user import SCIMPlaceholder
from litellm.proxy._types import (
LiteLLM_TeamTable,
LiteLLM_UserTable,
@ -585,6 +586,37 @@ async def _users_named_by_member_value(
return tuple(dict.fromkeys(row.user_id for row in rows))
async def _accounts_named_by_member_value(value: str, prisma_client: PrismaClient) -> tuple[str, ...]:
"""Every user id this member value names, by user id, SSO identity or email.
Classification needs to know whether the value is one account's ``user_id`` and
whether it names any other account, so all three fields are read in one pass. The
id is compared exactly and unstripped, as a primary key lookup would; the
identities compare as ``_users_named_by_member_value`` describes. Two rows are
enough to tell one account from several, so the read stops there. Only a full
read that lacks the row keyed by the value leaves that row's existence open, and
only then is the id read on its own.
"""
subject: Final = value.strip()
email: Final[_CaseInsensitiveMatch] = {"equals": subject, "mode": "insensitive"}
users: Final = _table(UserRepository(prisma_client))
rows: Final = await users.find_many(
where={ # mutable-ok: Prisma filter
"OR": [ # mutable-ok: Prisma filter
{"user_id": value}, # mutable-ok: Prisma filter
{"sso_user_id": subject}, # mutable-ok: Prisma filter
{"user_email": email}, # mutable-ok: Prisma filter
],
},
take=2,
)
named: Final = tuple(dict.fromkeys(row.user_id for row in rows))
if len(named) < 2 or value in named:
return named
keyed: Final = await users.find_unique(where={"user_id": value})
return named if keyed is None else (value, *named)
async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient) -> _ClassifiedGroupMember:
"""
Decide what a single SCIM group member refers to.
@ -627,11 +659,9 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient
if member_type == "group":
return _SkippedGroupMember(value=value, reason="nested_group")
user: Final = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": value})
if user is not None:
shared_with: Final = tuple(
other for other in await _users_named_by_member_value(value, prisma_client) if other != value
)
named: Final = await _accounts_named_by_member_value(value, prisma_client)
if value in named:
shared_with: Final = tuple(other for other in named if other != value)
if shared_with:
verbose_proxy_logger.warning(
"SCIM: group member '%s' is one account's user id and is also account '%s' by SSO identity or email, "
@ -651,7 +681,6 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient
if team is not None and _team_metadata_has_scim_provenance(team.metadata):
return _SkippedGroupMember(value=value, reason="existing_team")
named: Final = await _users_named_by_member_value(value, prisma_client)
if len(named) == 1:
verbose_proxy_logger.info(
"SCIM: group member '%s' matched user_id '%s' by SSO identity or email",
@ -1834,6 +1863,89 @@ async def delete_user(
raise handle_exception_on_proxy(e)
@scim_router.get(
"/placeholders",
response_model=tuple[SCIMPlaceholder, ...],
dependencies=(Depends(user_api_key_auth),),
)
async def list_placeholders() -> tuple[SCIMPlaceholder, ...]:
"""
List user rows whose id is another account's SSO identity or email.
An earlier release provisioned a group member it could not match as a user keyed
by the raw member value, and that row now shadows the account the value really
names, so every push of that member is refused. This lists those rows so an
operator can fold each one into the account it shadows with
``POST /scim/v2/placeholders/{user_id}/merge``. A row that has an SSO identity of
its own or owns virtual keys is left out: someone uses that account.
"""
try:
prisma_client: Final = await _get_prisma_client_or_raise_exception()
async with prisma_client.tx() as tx:
return await UserRepository(prisma_client).find_shadowing_placeholders(tx)
except Exception as e:
raise handle_exception_on_proxy(e)
def _placeholder_rejection(placeholder: LiteLLM_UserTable, resolved: tuple[str, ...], key_count: int) -> str | None:
if placeholder.sso_user_id is not None:
return f"User '{placeholder.user_id}' has an SSO identity of its own, so it is an account someone signs in to"
if key_count:
return f"User '{placeholder.user_id}' owns {key_count} virtual keys. Move or delete them before merging it"
if not resolved:
return f"User '{placeholder.user_id}' shadows no account: no other user has that id as SSO identity or email"
if len(resolved) > 1:
return (
f"User '{placeholder.user_id}' names {len(resolved)} accounts ({', '.join(resolved)}). Resolve that first"
)
return None
@scim_router.post(
"/placeholders/{user_id}/merge",
response_model=SCIMPlaceholderMergeResult,
dependencies=(Depends(user_api_key_auth),),
)
async def merge_placeholder(
user_id: str = Path(..., title="User ID"),
) -> SCIMPlaceholderMergeResult:
"""
Fold a placeholder user into the one account its id names by SSO identity or email.
The account is added to every team the placeholder is on, then the placeholder is
deleted the way ``DELETE /scim/v2/Users/{id}`` deletes a user, so the next group
push resolves the member value to the real account. Refused with 409 when the row
has an SSO identity of its own, owns virtual keys, or names no account or several.
"""
try:
prisma_client: Final = await _get_prisma_client_or_raise_exception()
placeholder: Final = await _check_user_exists(user_id)
resolved: Final = tuple(
other for other in await _users_named_by_member_value(user_id, prisma_client, take=None) if other != user_id
)
owned_keys: Final[_UserIdWhere] = {"user_id": user_id}
keys: Final = await _table(VerificationTokenRepository(prisma_client)).find_many(where=owned_keys)
rejection: Final = _placeholder_rejection(placeholder, resolved, len(keys))
if rejection is not None:
detail: Final[_ScimErrorDetail] = {"error": rejection}
raise HTTPException(status_code=409, detail=detail)
target_user_id: Final = resolved[0]
team_ids: Final = tuple(placeholder.teams)
for team_id in team_ids:
await _add_user_to_team(user_id=target_user_id, team_id=team_id)
await delete_user(user_id=user_id)
await _recompute_scim_member_roles(prisma_client, (target_user_id,))
verbose_proxy_logger.info(
"SCIM: merged placeholder user '%s' into '%s', moving teams %s", user_id, target_user_id, team_ids
)
return SCIMPlaceholderMergeResult(
placeholder_user_id=user_id, merged_into_user_id=target_user_id, team_ids=team_ids
)
except Exception as e:
raise handle_exception_on_proxy(e)
def _parse_member_entry(entry: object) -> SCIMMember | None:
"""Parse one entry of a SCIM patch value, or None when it carries no id."""
if isinstance(entry, str):

6
litellm/proxy/management_helpers/access_group_key_sync.py
View file

@ -38,6 +38,7 @@ from litellm.proxy._types import (
from litellm.proxy.auth.auth_checks import (
_delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive
)
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.repositories.table_repositories import AccessGroupRepository
@ -72,8 +73,9 @@ _REPOINT_KEY_SQL: Final = (
def _raw_executor(prisma_client: object) -> _RawExecutor:
"""Narrow the untyped Prisma client down to the raw-query call this module makes."""
return AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
"""Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer."""
db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
async def _invalidate_access_group_cache(access_group_id: str) -> None:

2
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -812,6 +812,8 @@ def _resolve_team_callback_wiring(
else { # mutable-ok: Logging arg
**callback_vars,
TRUSTED_CALLBACK_VARS_FIELD: callback_vars,
"metadata": {}, # mutable-ok: Logging arg
"model_info": {}, # mutable-ok: Logging arg
}
)
return _TeamCallbackWiring(

19
litellm/proxy/proxy_cli.py
View file

@ -1225,6 +1225,7 @@ def run_server(
if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None:
from litellm.proxy.db.db_url_settings import (
add_missing_query_params,
idle_lifetime_params,
reader_shareable_params,
unsupported_db_scheme,
unsupported_db_scheme_message,
@ -1253,6 +1254,9 @@ def run_server(
disable_prepared_statements=db_disable_prepared_statements,
extra_params=db_extra_connection_params,
)
lifetime_params: Final = idle_lifetime_params(
general_settings.get("database_max_idle_connection_lifetime")
)
if os.getenv("DATABASE_URL", None) is not None:
database_url = get_secret("DATABASE_URL", default_value=None)
resolved_url: Final[str | None] = str(database_url) if database_url else None
@ -1270,11 +1274,11 @@ def run_server(
writer_url,
connection_url_params,
)
os.environ["DATABASE_URL"] = modified_url
os.environ["DATABASE_URL"] = add_missing_query_params(modified_url, lifetime_params)
if os.getenv("DIRECT_URL", None) is not None:
database_url = os.getenv("DIRECT_URL")
modified_url = append_query_params(database_url, connection_url_params)
os.environ["DIRECT_URL"] = modified_url
os.environ["DIRECT_URL"] = add_missing_query_params(modified_url, lifetime_params)
# The reader pool is a real pool against the same configured cap, so it
# gets the allowlisted pool params. Schema-affecting ones, including any
# the operator smuggled in through database_extra_connection_params, stay
@ -1288,10 +1292,13 @@ def run_server(
db_lock_timeout,
)
os.environ["DATABASE_URL_READ_REPLICA"] = add_missing_query_params(
_with_query_value(read_replica_url, "options", reader_options)
if reader_options
else read_replica_url,
reader_shareable_params(connection_url_params),
add_missing_query_params(
_with_query_value(read_replica_url, "options", reader_options)
if reader_options
else read_replica_url,
reader_shareable_params(connection_url_params),
),
lifetime_params,
)
subprocess.run(["prisma"], capture_output=True)
is_prisma_runnable = True

160
litellm/proxy/proxy_server.py
View file

@ -39,7 +39,7 @@ from typing import (
import anyio
import websockets
import websockets.exceptions
from pydantic import BaseModel, Json, JsonValue
from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, assert_never
from litellm._uuid import uuid
@ -60,6 +60,7 @@ from litellm.constants import (
LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
LITELLM_UI_ALLOW_HEADERS,
LITELLM_UI_SESSION_DURATION,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
)
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
@ -253,6 +254,8 @@ from litellm.constants import (
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG,
USER_SPEND_ALERTS_JOB_ID,
WEEKLY_SPEND_REPORT_JOB_ID,
)
from litellm.exceptions import RejectedRequestError
@ -351,7 +354,10 @@ from litellm.proxy.common_utils.load_config_utils import (
get_file_contents_from_s3,
)
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
from litellm.proxy.common_utils.model_listing_utils import (
TeamModelNameTranslator,
configured_display_names,
)
from litellm.proxy.common_utils.openai_endpoint_utils import (
remove_sensitive_info_from_deployment,
)
@ -2274,7 +2280,7 @@ user_api_key_cache: UserApiKeyCache = UserApiKeyCache(
)
spend_counter_cache: Final = DualCache(default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value)
cli_sso_session_cache: Final = DualCache(default_in_memory_ttl=CLI_SSO_SESSION_TTL_SECONDS)
model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=user_api_key_cache)
model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=spend_counter_cache)
litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter)
redis_usage_cache: RedisCache | None = None # redis cache used for tracking spend, tpm/rpm limits
polling_via_cache_enabled: Literal["all"] | list[str] | bool = False
@ -5709,13 +5715,9 @@ class ProxyConfig:
router_settings: Final = config.get("router_settings", None)
if router_settings and isinstance(router_settings, dict):
# model list and search_tools already set
exclude_args: Final = {
"model_list",
"search_tools",
}
available_args: Final = [x for x in litellm.Router.get_valid_args() if x not in exclude_args]
available_args: Final = [
x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
]
for k, v in router_settings.items():
if k in available_args:
@ -9866,6 +9868,35 @@ class ProxyStartupEvent:
replace_existing=True,
)
slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args
user_spend_check_interval: Final = (
slack_alerting_args.user_spend_check_interval
if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance
else SlackAlertingArgs().user_spend_check_interval
)
async def _scheduled_user_spend_alerts() -> None:
if (
await pod_lock_manager.acquire_lock(
cronjob_id=USER_SPEND_ALERTS_JOB_ID,
ttl=max(user_spend_check_interval - 60, 60),
allow_reentrant=False,
)
is False
):
return
await proxy_logging_obj.slack_alerting_instance.send_user_spend_alerts()
scheduler.add_job(
_scheduled_user_spend_alerts,
"interval",
seconds=user_spend_check_interval,
next_run_time=datetime.now(timezone.utc) + timedelta(seconds=10 + random.randint(0, 60)),
id=USER_SPEND_ALERTS_JOB_ID,
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
if os.getenv("PROMETHEUS_URL"):
from zoneinfo import ZoneInfo
@ -10193,7 +10224,8 @@ async def model_list(
# The internal routing key drives the metadata/fallback lookup, while the
# public name is what the client sees as the model id.
model_data = []
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
for response_id, lookup_id in admin_entries:
model_info = create_model_info_response(
model_id=lookup_id,
provider="openai",
@ -10206,7 +10238,10 @@ async def model_list(
if wants_anthropic_format:
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(admin_listing)
return create_anthropic_model_list_response(
admin_listing,
display_names=configured_display_names(admin_entries, llm_router),
)
return dict(
data=model_data,
@ -10237,7 +10272,8 @@ async def model_list(
# The internal routing key drives the metadata/fallback lookup, while the
# public name is what the client sees as the model id.
model_data = []
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
for response_id, lookup_id in entries:
model_info = create_model_info_response(
model_id=lookup_id,
provider="openai",
@ -10250,7 +10286,10 @@ async def model_list(
if wants_anthropic_format:
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(listing)
return create_anthropic_model_list_response(
listing,
display_names=configured_display_names(entries, llm_router),
)
return dict(
data=model_data,
@ -12481,11 +12520,21 @@ async def supported_openai_params(model: str):
--header 'Authorization: Bearer sk-1234'
```
"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
global llm_router
try:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
resolved_models: Final = llm_router.resolved_litellm_models(model) if llm_router is not None else ()
target_model: Final = resolved_models[0] if resolved_models else model
declared_provider: Final = declared_authenticating_provider(target_model)
litellm_model, custom_llm_provider = (
(target_model.removeprefix(f"{declared_provider}/"), declared_provider)
if declared_provider is not None
else litellm.get_llm_provider(model=target_model)[:2]
)
return {
"supported_openai_params": litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
model=litellm_model, custom_llm_provider=custom_llm_provider
)
}
except Exception:
@ -14962,17 +15011,25 @@ async def alerting_settings(
alerting_args_dict = {}
alerting_values = None
allowed_args: Final = {
"slack_alerting": {"type": "Boolean"},
"daily_report_frequency": {"type": "Integer"},
"report_check_interval": {"type": "Integer"},
"budget_alert_ttl": {"type": "Integer"},
"outage_alert_ttl": {"type": "Integer"},
"region_outage_alert_ttl": {"type": "Integer"},
"minor_outage_alert_threshold": {"type": "Integer"},
"major_outage_alert_threshold": {"type": "Integer"},
"max_outage_alert_list_size": {"type": "Integer"},
}
allowed_args: Final = MappingProxyType(
{
"slack_alerting": "Boolean",
"daily_report_frequency": "Integer",
"report_check_interval": "Integer",
"budget_alert_ttl": "Integer",
"outage_alert_ttl": "Integer",
"region_outage_alert_ttl": "Integer",
"minor_outage_alert_threshold": "Integer",
"major_outage_alert_threshold": "Integer",
"max_outage_alert_list_size": "Integer",
"daily_spend_per_user_threshold": "Float",
"monthly_spend_per_user_threshold": "Float",
"spend_anomaly_multiplier": "Float",
"spend_anomaly_baseline_days": "Integer",
"spend_anomaly_min_spend": "Float",
"user_spend_check_interval": "Integer",
}
)
_slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance
_slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump()
@ -14987,7 +15044,7 @@ async def alerting_settings(
_response_obj = ConfigList(
field_name="slack_alerting",
field_type=allowed_args["slack_alerting"]["type"],
field_type=allowed_args["slack_alerting"],
field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
field_value=is_slack_enabled,
stored_in_db=True if alerting_values is not None else False,
@ -15006,7 +15063,7 @@ async def alerting_settings(
_response_obj = ConfigList(
field_name=field_name,
field_type=allowed_args[field_name]["type"],
field_type=allowed_args[field_name],
field_description=field_info.description or "",
field_value=_slack_alerting_args_dict.get(field_name, None),
stored_in_db=_stored_in_db,
@ -16159,6 +16216,7 @@ async def invitation_delete(
)
async def update_config(
config_info: ConfigYAML,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -16174,6 +16232,26 @@ async def update_config(
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can update config")
request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python(
await request.json()
)
raw_router_settings: Final = request_body.get("router_settings")
if isinstance(raw_router_settings, dict):
supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | (
frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
)
unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings)
if unsupported_router_settings:
raise HTTPException(
status_code=400,
detail={
"error": (
f"Unsupported router settings: {', '.join(unsupported_router_settings)} "
"are not valid router settings"
)
},
)
if prisma_client is None:
raise Exception("No DB Connected")
@ -16275,11 +16353,19 @@ async def update_config(
)
# router_settings: merge existing + request, request wins.
if config_info.router_settings is not None:
if isinstance(raw_router_settings, dict):
existing = await _read_section("router_settings")
before_router_settings: Final = copy.deepcopy(existing)
updates = config_info.router_settings.dict(exclude_none=True)
new_router_settings: Final = {**existing, **updates}
typed_router_settings: Final = (
config_info.router_settings.dict(exclude_none=True) if config_info.router_settings is not None else {}
)
raw_router_settings_without_none: Final = {
key: value
for key, value in raw_router_settings.items()
if key not in typed_router_settings and value is not None
}
router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none}
new_router_settings: Final = {**existing, **router_settings_updates}
await _upsert_section("router_settings", new_router_settings)
asyncio.create_task(
create_config_audit_log(
@ -16434,6 +16520,16 @@ async def update_config_general_settings(
detail={"error": f"Invalid type of field value={type(data.field_value)} passed in."},
)
if data.field_name == "alerting_args":
try:
SlackAlertingArgs.model_validate(data.field_value)
except ValidationError as e:
errors: Final = "; ".join(f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors())
raise HTTPException(
status_code=400,
detail={"error": f"Invalid alerting_args: {errors}"},
)
## get general settings from db
db_general_settings: Final = await _config_param_table(prisma_client).find_first(
where={"param_name": "general_settings"}

56
litellm/proxy/rag_endpoints/endpoints.py
View file

@ -9,6 +9,7 @@ Provides:
import base64
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import orjson
@ -19,6 +20,9 @@ from starlette.datastructures import UploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
LiteLLM_ManagedVectorStore,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.proxy._types import *
from litellm.proxy.auth.auth_utils import is_request_body_safe
@ -36,6 +40,10 @@ from litellm.proxy.rag_endpoints.upload_security import (
RejectedUpload,
validate_upload,
)
from litellm.proxy.vector_store_endpoints.endpoints import (
build_request_data_from_managed_vector_store,
reject_caller_embedding_selection_params,
)
from litellm.proxy.vector_store_endpoints.utils import (
assert_user_can_access_vector_store_id,
)
@ -120,12 +128,21 @@ def _collect_vector_store_ids_from_payload(payload: object) -> set[str]:
async def _authorize_nested_vector_store_ids(
payload: object,
user_api_key_dict: UserAPIKeyAuth,
) -> None:
for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)):
await assert_user_can_access_vector_store_id(
vector_store_id=vector_store_id,
user_api_key_dict=user_api_key_dict,
)
) -> Mapping[str, LiteLLM_ManagedVectorStore]:
"""Authorize every nested vector store id and return the managed stores it resolved."""
return MappingProxyType(
{
vector_store_id: store
for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload))
if (
store := await assert_user_can_access_vector_store_id(
vector_store_id=vector_store_id,
user_api_key_dict=user_api_key_dict,
)
)
is not None
}
)
def _build_file_metadata_entry(
@ -700,11 +717,27 @@ async def rag_query(
status_code=400,
detail={"error": "retrieval_config must contain 'vector_store_id'"},
)
await _authorize_nested_vector_store_ids(
reject_caller_embedding_selection_params(payload=retrieval_config, source="retrieval_config")
resolved_stores: Final = await _authorize_nested_vector_store_ids(
payload=retrieval_config,
user_api_key_dict=user_api_key_dict,
)
# Merge litellm-managed vector store params (provider, region, embedding
# model, credentials, ...) from the registry: the same source the direct
# /vector_stores/{id}/search endpoint uses. Store-managed keys win on
# conflict so callers cannot override the store's provider or credentials.
managed_store: Final = resolved_stores.get(retrieval_config["vector_store_id"])
store_data: Final = (
await build_request_data_from_managed_vector_store(managed_store)
if managed_store is not None
else MappingProxyType({})
)
merged_retrieval_config: Final = {
**retrieval_config,
**store_data,
} # mutable-ok: litellm.aquery requires a plain dict payload
# Add litellm data
request_data: dict[str, object] = {}
request_data = await add_litellm_data_to_request(
@ -716,13 +749,18 @@ async def rag_query(
proxy_config=proxy_config,
)
verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, retrieval_config)
verbose_proxy_logger.debug(
"RAG Query - model: %s, vector_store_id: %s, custom_llm_provider: %s",
model,
retrieval_config["vector_store_id"],
merged_retrieval_config.get("custom_llm_provider"),
)
# Call query
response: Final = await litellm.aquery(
model=model,
messages=messages,
retrieval_config=retrieval_config,
retrieval_config=merged_retrieval_config,
rerank=rerank,
stream=stream,
router=llm_router,

36
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -120,13 +120,15 @@ async def _apply_over_budget_reservation_policy(
applied_entries: list[dict[str, float | str]],
reservation_cost: float,
current_spend: float,
fail_closed_budget_enforcement: bool = False,
) -> float:
"""
Decide what to do when a counter is over budget, and return the reservation
cost to carry into the next counter. Three outcomes: an over-budget key that
opted into throttling releases its own reservation (the rate limiter slows
it) and keeps the cost; a partially-remaining budget resizes the reservation
down to what is left; anything else hard-blocks by raising.
down to what is left, unless strict enforcement is on, because the known
estimate already does not fit; anything else hard-blocks by raising.
"""
if _key_reservation_should_release_for_throttle(counter.counter_key, valid_token):
await _release_applied_entries_best_effort(entries=[entry], default_reserved_cost=reservation_cost)
@ -134,21 +136,36 @@ async def _apply_over_budget_reservation_policy(
return reservation_cost
remaining_before_reservation: Final = counter.max_budget - (current_spend - reservation_cost)
if remaining_before_reservation > 1e-12:
await _resize_applied_reservation(
entries=applied_entries,
current_reserved_cost=reservation_cost,
new_reserved_cost=remaining_before_reservation,
if remaining_before_reservation <= 1e-12:
_raise_counter_budget_exceeded(counter=counter, current_cost=current_spend)
if fail_closed_budget_enforcement and current_spend - counter.max_budget > 1e-12:
_raise_counter_budget_exceeded(
counter=counter,
current_cost=current_spend - reservation_cost,
estimated_cost=reservation_cost,
)
return remaining_before_reservation
await _resize_applied_reservation(
entries=applied_entries,
current_reserved_cost=reservation_cost,
new_reserved_cost=remaining_before_reservation,
)
return remaining_before_reservation
def _raise_counter_budget_exceeded(
counter: _BudgetCounter,
current_cost: float,
estimated_cost: float | None = None,
) -> NoReturn:
estimate_detail: Final = "" if estimated_cost is None else f"Estimated request cost: {estimated_cost}, "
raise litellm.BudgetExceededError(
current_cost=current_spend,
current_cost=current_cost,
max_budget=counter.max_budget,
message=(
"Budget has been exceeded! "
f"{counter.entity_type}={counter.entity_id} "
f"Current cost: {current_spend}, "
f"Current cost: {current_cost}, "
f"{estimate_detail}"
f"Max budget: {counter.max_budget}"
),
entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type),
@ -258,6 +275,7 @@ async def reserve_budget_for_request(
applied_entries=applied_entries,
reservation_cost=reservation_cost,
current_spend=current_spend,
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
continue
except Exception:

110
litellm/proxy/vector_store_endpoints/endpoints.py
View file

@ -1,3 +1,5 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import (
Annotated,
Any, # noqa: TID251 # jsonify_object in proxy/utils.py is annotated with a bare dict
@ -27,11 +29,69 @@ from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse
from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry
router: Final = APIRouter()
BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS: Final = frozenset(
{
"embedding_model",
"litellm_embedding_model",
"litellm_embedding_config",
"litellm_credential_name",
}
)
def reject_caller_embedding_selection_params(payload: Mapping[str, object], source: str) -> None:
blocked: Final = sorted(BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS & payload.keys())
if blocked:
raise HTTPException(
status_code=400,
detail={
"error": f"'{blocked[0]}' cannot be set in {source}. "
"Embedding configuration comes from the vector store's server-side registration."
},
)
########################################################
# OpenAI Compatible Endpoints
########################################################
async def build_request_data_from_managed_vector_store(
vector_store: LiteLLM_ManagedVectorStore,
) -> Mapping[str, object]:
"""
Build request params (provider, credential ref, litellm_params) from an
already-resolved managed vector store.
``litellm_embedding_config`` is resolved here, at request-handling time,
instead of at row-creation time: the resolved api_key/api_base/api_version
lives only in the returned per-request mapping and is never persisted back
to the registry cache. Legacy rows that already carry a resolved
(cleartext) config skip the lookup and pass through unchanged.
"""
top_level: Final = MappingProxyType(
{
key: vector_store.get(key)
for key in ("custom_llm_provider", "litellm_credential_name")
if key in vector_store
}
)
litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({})
embedding_model: Final = litellm_params.get("litellm_embedding_model")
if not embedding_model or litellm_params.get("litellm_embedding_config"):
return MappingProxyType({**top_level, **litellm_params})
from litellm.proxy.proxy_server import prisma_client
resolved_config: Final = await _resolve_embedding_config(
embedding_model=embedding_model, prisma_client=prisma_client
)
if not resolved_config:
return MappingProxyType({**top_level, **litellm_params})
return MappingProxyType({**top_level, **litellm_params, "litellm_embedding_config": resolved_config})
async def _update_request_data_with_litellm_managed_vector_store_registry(
data: dict,
vector_store_id: str,
@ -51,47 +111,14 @@ async def _update_request_data_with_litellm_managed_vector_store_registry(
vector_store_to_run: Final[LiteLLM_ManagedVectorStore | None] = await get_litellm_managed_vector_store(
vector_store_id=vector_store_id
)
if vector_store_to_run is not None:
if user_api_key_dict is not None:
await assert_user_can_access_vector_store(
vector_store=vector_store_to_run,
user_api_key_dict=user_api_key_dict,
)
if "custom_llm_provider" in vector_store_to_run:
data["custom_llm_provider"] = vector_store_to_run.get("custom_llm_provider")
if "litellm_credential_name" in vector_store_to_run:
data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name")
if "litellm_params" in vector_store_to_run:
litellm_params = vector_store_to_run.get("litellm_params", {}) or {}
# Resolve ``litellm_embedding_config`` here, at request-handling
# time, instead of at row-creation time. The resolved
# ``api_key`` / ``api_base`` / ``api_version`` lives only in
# this per-request ``data`` dict and is never persisted.
# Legacy rows that already carry a resolved (cleartext)
# ``litellm_embedding_config`` skip the lookup and pass through
# unchanged so the embed call keeps working.
embedding_model: Final = litellm_params.get("litellm_embedding_model")
if embedding_model and not litellm_params.get("litellm_embedding_config"):
from litellm.proxy.proxy_server import prisma_client
resolved_config: Final = await _resolve_embedding_config(
embedding_model=embedding_model, prisma_client=prisma_client
)
if resolved_config:
# Build a fresh dict via spread instead of mutating
# ``litellm_params`` in place — the registry hands back
# a reference to its cached object, so an in-place
# update would persist the resolved cleartext into the
# in-memory cache for the lifetime of the process.
litellm_params = {
**litellm_params,
"litellm_embedding_config": resolved_config,
}
data.update(litellm_params)
return data
if vector_store_to_run is None:
return data
if user_api_key_dict is not None:
await assert_user_can_access_vector_store(
vector_store=vector_store_to_run,
user_api_key_dict=user_api_key_dict,
)
return {**data, **(await build_request_data_from_managed_vector_store(vector_store_to_run))}
@router.post(
@ -130,6 +157,7 @@ async def vector_store_search(
)
data = await _read_request_body(request=request)
reject_caller_embedding_selection_params(payload=data, source="the search request body")
data["vector_store_id"] = vector_store_id
# Check for legacy vector store registry (non-managed vector stores)

4
litellm/proxy/vector_store_endpoints/management_endpoints.py
View file

@ -470,7 +470,7 @@ async def create_vector_store_in_db(
# exposed every env-stored embedding-model credential on the
# ``/vector_store/{new,info,update,list}`` responses. Keep the user's
# raw ``litellm_embedding_model`` reference; resolution now happens in
# ``_update_request_data_with_litellm_managed_vector_store_registry``
# ``build_request_data_from_managed_vector_store``
# at request-handling time so the cleartext config exists only in
# per-request memory and never reaches the database.
if litellm_params:
@ -864,7 +864,7 @@ async def update_vector_store(
# embedding-config auto-resolve previously persisted cleartext
# credentials into the row; resolution now happens at request-
# handling time in
# ``_update_request_data_with_litellm_managed_vector_store_registry``
# ``build_request_data_from_managed_vector_store``
# so this row only ever stores the user-supplied
# ``litellm_embedding_model`` reference.
if "litellm_params" in update_data:

25
litellm/rag/main.py
View file

@ -14,6 +14,7 @@ import contextvars
from collections.abc import Coroutine, Iterator
from contextlib import contextmanager
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -50,6 +51,21 @@ INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = {
"vertex_ai": VertexAIRAGIngestion,
}
# Only these retrieval_config keys are forwarded to vector_stores.asearch as
# provider-specific params. The explicit allowlist keeps caller-controlled
# connection overrides (api_base, api_key, ...) away from the search call,
# where they could redirect store credentials to an attacker-chosen host.
_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset(
{
"aws_region_name",
"vector_bucket_name",
"embedding_model",
"litellm_embedding_model",
"litellm_embedding_config",
"litellm_credential_name",
}
)
def get_ingestion_class(provider: str) -> type[BaseRAGIngestion]:
"""
@ -224,13 +240,20 @@ async def _execute_query_pipeline(
raise ValueError("No query found in messages for RAG query")
# 2. Search vector store
# Forward allowlisted provider retrieval_config extras (region, embedding
# model, bucket, credential refs) to the search call; kwargs win on conflict.
provider_search_params: Final = MappingProxyType(
{k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS}
)
forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs})
with _suppressed_sub_call_billing():
search_response: Final = await litellm.vector_stores.asearch(
vector_store_id=retrieval_config["vector_store_id"],
query=query_text,
max_num_results=retrieval_config.get("top_k", 10),
custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"),
**kwargs,
router=router,
**forwarded_search_params,
)
search_provider: Final = retrieval_config.get("custom_llm_provider", "openai")

26
litellm/repositories/user_repository.py
View file

@ -6,15 +6,34 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from litellm.models.user import LiteLLM_UserTable
from pydantic import TypeAdapter
from litellm.models.user import LiteLLM_UserTable, SCIMPlaceholder
from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict
from litellm.repositories.prisma_protocols import TableActions
if TYPE_CHECKING:
from prisma import Prisma
from prisma import models as prisma_models
_JSON_ENCODED_COLUMNS: Final = frozenset({"metadata", "model_spend", "model_max_budget"})
_SHADOWING_PLACEHOLDERS_SQL: Final = """
SELECT p.user_id AS placeholder_user_id,
array_agg(r.user_id ORDER BY r.user_id) AS resolved_user_ids,
p.teams AS team_ids
FROM "LiteLLM_UserTable" p
JOIN "LiteLLM_UserTable" r
ON r.user_id <> p.user_id
AND (r.sso_user_id = p.user_id OR LOWER(r.user_email) = LOWER(p.user_id))
WHERE p.sso_user_id IS NULL
AND NOT EXISTS (SELECT 1 FROM "LiteLLM_VerificationToken" k WHERE k.user_id = p.user_id)
GROUP BY p.user_id, p.teams
ORDER BY p.user_id
"""
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@ -59,6 +78,11 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Find all users in a team."""
return await self.find_many(where={"teams": {"has": team_id}})
async def find_shadowing_placeholders(self, tx: "Prisma") -> tuple[SCIMPlaceholder, ...]:
"""Users with no SSO id and no virtual keys whose id is another user's SSO id or email."""
rows: Final = await tx.query_raw(_SHADOWING_PLACEHOLDERS_SQL)
return _PLACEHOLDER_ROWS_ADAPTER.validate_python(rows)
async def count_billable_users(self) -> int:
"""Number of users that count toward the license seat limit.

29
litellm/rerank_api/main.py
View file

@ -6,6 +6,7 @@ from typing import Any, Final, Literal
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
@ -43,10 +44,23 @@ async def arerank(
"""
Async: Reranks a list of documents based on their relevance to the query
"""
_custom_llm_provider: str | None = (
None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except
)
try:
loop: Final = asyncio.get_event_loop()
kwargs["arerank"] = True
declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared_provider is not None:
_custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above
else:
_, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above
model=model,
custom_llm_provider=custom_llm_provider,
api_base=kwargs.get("api_base", None),
)
func: Final = partial(
rerank,
model,
@ -70,7 +84,11 @@ async def arerank(
response = init_response
return response
except Exception as e:
raise e
raise exception_type(
model=model,
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
original_exception=e,
)
@client
@ -115,6 +133,7 @@ def rerank(
model_info: Final = kwargs.get("model_info", None)
user: Final = kwargs.get("user", None)
client: Final = kwargs.get("client", None)
_custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except
try:
_is_async: Final = kwargs.pop("arerank", False) is True
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -127,7 +146,7 @@ def rerank(
(
model,
_custom_llm_provider,
_custom_llm_provider, # rebind-ok: see pre-declaration above
dynamic_api_key,
dynamic_api_base,
) = litellm.get_llm_provider(
@ -538,4 +557,8 @@ def rerank(
return response
except Exception as e:
verbose_logger.error("Error in rerank: %s", e)
raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e)
raise exception_type(
model=model,
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
original_exception=e,
)

10
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -1169,16 +1169,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None:
if litellm_model_response:
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None:
usage: Final[object] = getattr(litellm_model_response, "usage", None)
if usage is not None:
setattr(
usage,
"cost",
self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response),
)
# Transform the response
responses_api_response: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(

54
litellm/responses/streaming_iterator.py
View file

@ -407,23 +407,7 @@ class BaseResponsesAPIStreamingIterator:
openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED,
):
self.completed_response = openai_responses_api_chunk
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and self.logging_obj is not None:
response_obj: Final[ResponsesAPIResponse | None] = getattr(
openai_responses_api_chunk, "response", None
)
if response_obj:
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
if usage_obj is not None:
try:
cost: Final[float | None] = self.logging_obj._response_cost_calculator(
result=response_obj
)
if cost is not None:
setattr(usage_obj, "cost", cost)
except Exception:
# Best-effort usage cost annotation should not break stream replay.
pass
_stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()
@ -1023,7 +1007,7 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
transformed: ResponsesAPIResponse,
logging_obj: LiteLLMLoggingObj,
) -> None:
self._events: list[ResponsesAPIStreamingResponse] = _build_synthetic_response_events(
self._events: Sequence[ResponsesAPIStreamingResponse] = build_synthetic_response_events(
transformed=transformed,
logging_obj=logging_obj,
chunk_size=self.CHUNK_SIZE,
@ -1090,7 +1074,7 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
transformed: ResponsesAPIResponse,
logging_obj: LiteLLMLoggingObj,
) -> None:
self._events = _build_synthetic_response_events(
self._events = build_synthetic_response_events(
transformed=transformed,
logging_obj=logging_obj,
chunk_size=MockResponsesAPIStreamingIterator.CHUNK_SIZE,
@ -1274,22 +1258,32 @@ def _add_text_like_part_events(
)
def _build_synthetic_response_events(
def _stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
return
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
if usage_obj is None:
return
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
return
try:
cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj)
except Exception:
return
if isinstance(cost, (int, float)) and cost > 0:
setattr(usage_obj, "cost", cost)
def build_synthetic_response_events(
*,
transformed: ResponsesAPIResponse,
logging_obj: LiteLLMLoggingObj,
logging_obj: LiteLLMLoggingObj | None,
chunk_size: int,
) -> list[ResponsesAPIStreamingResponse]:
openai_types: Final = _get_openai_response_types()
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None
if usage_obj is not None:
try:
cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed)
if cost is not None:
setattr(usage_obj, "cost", cost)
except Exception:
pass
_stamp_responses_usage_cost(transformed, logging_obj)
events: Final[list[ResponsesAPIStreamingResponse]] = [
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),

326
litellm/router.py
View file

@ -22,7 +22,7 @@ import traceback
import weakref
from collections import defaultdict
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping, Sequence
from functools import lru_cache
from functools import lru_cache, partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
@ -50,6 +50,7 @@ from litellm.constants import (
DEFAULT_HEALTH_CHECK_INTERVAL,
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
DEFAULT_MAX_LRU_CACHE_SIZE,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
@ -143,7 +144,11 @@ from litellm.router_utils.cooldown_handlers import (
from litellm.router_utils.fallback_event_handlers import (
AttemptedFallbackTargets,
_check_non_standard_fallback_format,
get_fallback_model_group,
clear_pre_routing_selection,
fallback_lookup_groups,
get_fallback_model_group_for_lookup_groups,
get_pre_routing_selection,
record_pre_routing_selection,
run_async_fallback,
)
from litellm.router_utils.get_retry_from_policy import (
@ -350,6 +355,13 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType(
{
"prompt_caching": PromptCachingDeploymentCheck,
"enforce_model_rate_limits": ModelRateLimitingCheck,
}
)
def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool:
for chunk in chunks:
@ -821,6 +833,7 @@ class Router:
self._zero_cost_cache: dict[str, bool] = {}
self._routing_group_rows: tuple[DeploymentTypedDict, ...] | None = None
self._init_routing_groups(None)
self._provider_unresolved_deployments: tuple[Callable[[], Deployment | None], ...] = ()
self.deployment_affinity_ttl_seconds = deployment_affinity_ttl_seconds
self.model_group_affinity_config = model_group_affinity_config
@ -2067,11 +2080,39 @@ class Router:
if _callback is None:
continue
if self.optional_callbacks is not None and any(
isinstance(callback, type(_callback)) for callback in self.optional_callbacks
):
continue
if self.optional_callbacks is None:
self.optional_callbacks = []
self.optional_callbacks.append(_callback)
litellm.logging_callback_manager.add_litellm_callback(_callback)
def set_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None) -> None:
if optional_pre_call_checks is None:
return
requested: Final = frozenset(optional_pre_call_checks)
for name, callback_cls in _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS.items():
if name not in requested:
self._remove_optional_callbacks_of_type(callback_cls)
self.add_optional_pre_call_checks(optional_pre_call_checks)
def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None:
if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks):
return
self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
if any(
router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or []))
for router in tuple(_live_routers)
):
return
for cb in tuple(litellm.callbacks):
if type(cb) is callback_cls:
litellm.logging_callback_manager.remove_callback_from_list_by_object(
litellm.callbacks, cb, require_self=False
)
def print_deployment(self, deployment: dict):
"""
returns a copy of the deployment with the api key masked
@ -2319,7 +2360,7 @@ class Router:
@overload
async def acompletion(
self, model: str, messages: list[AllMessageValues], stream: Literal[True, False] = False, **kwargs
) -> CustomStreamWrapper | ModelResponse:
) -> CustomStreamWrapper | ModelResponse:
...
# fmt: on
@ -4918,6 +4959,19 @@ class Router:
)
response = await response
if self._should_raise_anthropic_refusal_error(
model=model,
original_generic_function=original_generic_function,
response=response,
kwargs=kwargs,
):
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
safeguard_refusal_error,
)
refusal_details: Final = cast(dict, response["stop_details"]) # cast-ok: gate verified the shape
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
self.success_calls[model_name] += 1
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
@ -4964,6 +5018,11 @@ class Router:
# fallback to the original reference for any non-picklable value.
# The original_generic_function is preserved so the per-attempt
# helper knows which underlying API to call on fallback.
# The pre-routing hook stamps its tier selection into this bucket during the primary
# attempt; seeding it before the snapshot gives both the live kwargs and the copy a
# bucket, so the post-call carry-over below always has somewhere to read and write.
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
fallback_kwargs: Final[dict[str, object]] = kwargs.copy()
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
@ -4973,6 +5032,14 @@ class Router:
response: Final = await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs)
# The snapshot predates the pre-routing hook, so the tier it stamped into the live kwargs
# is carried over write-or-clear: a stale or caller-supplied selection left in the copy
# would key the mid-stream fallback lookup off a tier this attempt never routed to.
clear_pre_routing_selection(fallback_kwargs)
live_pre_routing_selection: Final = get_pre_routing_selection(kwargs)
if live_pre_routing_selection is not None:
record_pre_routing_selection(fallback_kwargs, live_pre_routing_selection)
if kwargs.get("stream") and isinstance(response, BaseResponsesAPIStreamingIterator):
return await self._aresponses_streaming_iterator(
response=response,
@ -5030,6 +5097,10 @@ class Router:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
aclose_if_supported,
parse_anthropic_error_event,
parse_anthropic_refusal_stop_details,
)
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
safeguard_refusal_error,
)
source_iterator: Final = response
@ -5068,13 +5139,35 @@ class Router:
continue
if _anthropic_stream_commits_now(chunk, has_generated_content, len(buffered_lifecycle_chunks)):
has_generated_content = True # rebind-ok: real content seen, or the buffer cap was hit
error_event = parse_anthropic_error_event(chunk)
# A transport can split one SSE data line across byte chunks, so pre-content
# detection parses the accumulated buffer plus the current chunk, never the
# chunk alone; the buffer is already capped, which bounds this window too.
parse_window = ( # rebind-ok: freshly computed each iteration, never carried over
b"".join(c for c in (*buffered_lifecycle_chunks, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime
if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime
else chunk
)
error_event = parse_anthropic_error_event(parse_window)
retriable_pending_error = ( # rebind-ok: freshly computed each iteration, never carried over
not has_generated_content
and error_event is not None
and _is_retriable_anthropic_status(error_event[2])
and not _anthropic_stream_error_is_gateway_verdict(chunk)
)
refusal_stop_details = ( # rebind-ok: freshly computed each iteration, never carried over
parse_anthropic_refusal_stop_details(parse_window)
if not has_generated_content and error_event is None
else None
)
if refusal_stop_details is not None and self._has_content_policy_fallback(model, initial_kwargs):
refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details)
raise MidStreamFallbackError(
message=refusal_error.message,
model=model,
llm_provider="anthropic",
original_exception=refusal_error,
is_pre_first_chunk=True,
)
if not has_generated_content and not retriable_pending_error and error_event is None:
buffered_lifecycle_chunks = (*buffered_lifecycle_chunks, chunk)
continue
@ -5186,8 +5279,13 @@ class Router:
kwargs=initial_kwargs,
metadata_variable_name="litellm_metadata",
)
# The content-policy dispatch branch matches on the trigger's own type, so a refusal's
# MidStreamFallbackError envelope is unwrapped here or the wrong fallback list is consulted.
fallback_trigger: Final[Exception] = (
e.original_exception if isinstance(e.original_exception, litellm.ContentPolicyViolationError) else e
)
fallback_response = await self.async_function_with_fallbacks_common_utils( # rebind-ok: set on success
e=e,
e=fallback_trigger,
disable_fallbacks=False,
fallbacks=fallbacks,
context_window_fallbacks=context_window_fallbacks,
@ -5243,6 +5341,11 @@ class Router:
# share, leaking primary-deployment metadata into the mid-stream
# fallback request. safe_deep_copy avoids deep-copying the full
# kwargs (which can hold non-deepcopyable logging handles/clients).
# The pre-routing hook stamps its tier selection into this bucket during the primary
# attempt; seeding it before the snapshot gives both the live kwargs and the copy a
# bucket, so the post-call carry-over below always has somewhere to read and write.
kwargs.setdefault("litellm_metadata", {}) # mutable-ok: shared bucket # rebind-ok: stamp must be readable here
fallback_kwargs: Final[dict[str, object]] = kwargs.copy() # mutable-ok: mutated below before re-entry
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
@ -5252,6 +5355,14 @@ class Router:
response: Final = await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs)
# The snapshot predates the pre-routing hook, so the tier it stamped into the live kwargs
# is carried over write-or-clear: a stale or caller-supplied selection left in the copy
# would key the mid-stream fallback lookup off a tier this attempt never routed to.
clear_pre_routing_selection(fallback_kwargs)
live_pre_routing_selection: Final = get_pre_routing_selection(kwargs)
if live_pre_routing_selection is not None:
record_pre_routing_selection(fallback_kwargs, live_pre_routing_selection)
if kwargs.get("stream") and hasattr(response, "__aiter__"):
return await self._aanthropic_messages_streaming_iterator(
response=cast("AsyncIterator[bytes]", response), # cast-ok: stream=True always returns a byte iterator
@ -6299,8 +6410,6 @@ class Router:
"responses",
"generate_content",
"generate_content_stream",
"vector_store_search",
"vector_store_create",
"ocr",
"search",
"video_generation",
@ -6324,6 +6433,8 @@ class Router:
return sync_wrapper
if call_type in (
"vector_store_search",
"vector_store_create",
"vector_store_retrieve",
"vector_store_list",
"vector_store_update",
@ -6335,11 +6446,16 @@ class Router:
client: object | None = None,
**kwargs,
):
if custom_llm_provider and "custom_llm_provider" not in kwargs:
kwargs["custom_llm_provider"] = custom_llm_provider
if kwargs.get("model"):
return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs)
return original_function(**kwargs)
provider_kwargs: Final = (
MappingProxyType({**kwargs, "custom_llm_provider": custom_llm_provider})
if custom_llm_provider and "custom_llm_provider" not in kwargs
else MappingProxyType(kwargs)
)
if provider_kwargs.get("model"):
return self._generic_api_call_with_fallbacks(original_function=original_function, **provider_kwargs)
if call_type == "vector_store_search":
return original_function(**MappingProxyType({**provider_kwargs, "router": self}))
return original_function(**provider_kwargs)
return vector_store_sync_wrapper
@ -6515,6 +6631,7 @@ class Router:
return await self._init_vector_store_api_endpoints(
original_function=original_function,
custom_llm_provider=custom_llm_provider,
call_type=call_type,
**kwargs,
)
elif call_type in ("afile_delete", "afile_content"):
@ -6555,6 +6672,7 @@ class Router:
self,
original_function: Callable,
custom_llm_provider: str | None = None,
call_type: str | None = None,
**kwargs,
):
"""
@ -6573,6 +6691,13 @@ class Router:
**kwargs,
)
# For search, pass the router so provider transforms can resolve
# router-managed embedding models (e.g. S3 Vectors query embeddings).
# The merge also overrides any client-supplied `router` key.
if call_type == "avector_store_search":
search_kwargs: Final = MappingProxyType({**kwargs, "router": self})
return await original_function(**search_kwargs)
# Otherwise, call the original function directly
return await original_function(**kwargs)
@ -6807,6 +6932,9 @@ class Router:
original_exception: Final = e
fallback_model_group = None
original_model_group: Final[str | None] = kwargs.get("model")
# A pre-routing hook (complexity / auto / adaptive / quality routers) picks a tier
# behind the router name, and fallbacks are configured per tier, not per router.
lookup_groups: Final[tuple[str, ...]] = fallback_lookup_groups(kwargs, model_group)
fallback_failure_exception_str = ""
if disable_fallbacks is True or original_model_group is None:
@ -6851,15 +6979,15 @@ class Router:
]
# Get external fallbacks — handle both standard and non-standard formats
external_fallback_group: list | None = None
if fallbacks is not None and model_group is not None:
if fallbacks is not None and lookup_groups:
if _check_non_standard_fallback_format(fallbacks=fallbacks):
# Non-standard formats (e.g. ["claude-3-haiku"] or
# [{"model": "...", "messages": [...]}]) are passed through directly
external_fallback_group = fallbacks
else:
external_fallback_group, generic_idx = get_fallback_model_group(
external_fallback_group, generic_idx = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks,
model_group=cast(str, model_group),
lookup_groups=lookup_groups,
)
if external_fallback_group is None and generic_idx is not None:
external_fallback_group = fallbacks[generic_idx]["*"]
@ -6917,9 +7045,9 @@ class Router:
if isinstance(e, litellm.ContextWindowExceededError):
if context_window_fallbacks is not None:
context_window_fallback_model_group: Final[list[str] | None] = (
self._get_fallback_model_group_from_fallbacks(
self._get_fallback_model_group_for_lookup_groups(
fallbacks=context_window_fallbacks,
model_group=model_group,
lookup_groups=lookup_groups,
)
)
if context_window_fallback_model_group is None:
@ -6950,9 +7078,9 @@ class Router:
elif isinstance(e, litellm.ContentPolicyViolationError):
if content_policy_fallbacks is not None:
content_policy_fallback_model_group: Final[list[str] | None] = (
self._get_fallback_model_group_from_fallbacks(
self._get_fallback_model_group_for_lookup_groups(
fallbacks=content_policy_fallbacks,
model_group=model_group,
lookup_groups=lookup_groups,
)
)
if content_policy_fallback_model_group is None:
@ -6979,14 +7107,14 @@ class Router:
if litellm.expose_router_debug_in_errors:
e.message += f"\n{error_message}"
if fallbacks is not None and model_group is not None:
if fallbacks is not None and lookup_groups:
verbose_router_logger.debug("inside model fallbacks: %s", mask_sensitive_structure(fallbacks))
(
fallback_model_group,
generic_fallback_idx,
) = get_fallback_model_group(
) = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks, # if fallbacks = [{"gpt-3.5-turbo": ["claude-3-haiku"]}]
model_group=cast(str, model_group),
lookup_groups=lookup_groups,
)
## if none, check for generic fallback
if fallback_model_group is None and generic_fallback_idx is not None:
@ -6995,12 +7123,12 @@ class Router:
if fallback_model_group is None:
masked_fallbacks: Final = mask_sensitive_structure(fallbacks)
verbose_router_logger.info(
"No fallback model group found for original model_group=%s. Fallbacks=%s",
model_group,
"No fallback model group found for lookup_groups=%s. Fallbacks=%s",
" -> ".join(lookup_groups),
masked_fallbacks,
)
if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors:
original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}"
original_exception.message += f"No fallback model group found for lookup_groups={' -> '.join(lookup_groups)}. Fallbacks={masked_fallbacks}"
raise original_exception
input_kwargs.update(
@ -7046,6 +7174,7 @@ class Router:
If it fails after num_retries, fall back to another model group
"""
model_group: Final[str | None] = kwargs.get("model")
clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary
if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets):
_fallback_metadata_key: Final = _get_router_metadata_variable_name(
function_name=getattr(kwargs.get("original_function"), "__name__", None)
@ -7471,6 +7600,24 @@ class Router:
break
return fallback_model_group
def _get_fallback_model_group_for_lookup_groups(
self,
fallbacks: list[dict[str, list[str]]], # mutable-ok: mirrors the sibling resolver's contract
lookup_groups: tuple[str, ...],
) -> list[str] | None: # mutable-ok: mirrors the sibling resolver's contract
"""First lookup group whose exact-key chain resolves (tier first, then requested group)."""
return next(
(
resolved
for resolved in (
self._get_fallback_model_group_from_fallbacks(fallbacks=fallbacks, model_group=group)
for group in lookup_groups
)
if resolved is not None
),
None,
)
def _get_first_default_fallback(self) -> str | None:
"""
Returns the first model from the default_fallbacks list, if it exists.
@ -7886,6 +8033,31 @@ class Router:
return True
return False
def _has_content_policy_fallback(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
"""
Whether a content-policy fallback would resolve for this request, keyed the same way
async_function_with_fallbacks_common_utils resolves it: the tier a pre-routing hook
selected wins over the requested group. Raising without this returning True would turn
a deliverable response into an error the fallback chain cannot recover from.
"""
content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks)
if content_policy_fallbacks is not None:
return (
self._get_fallback_model_group_for_lookup_groups(
fallbacks=content_policy_fallbacks,
lookup_groups=fallback_lookup_groups(kwargs, model_group),
)
is not None
)
if self._has_default_fallbacks():
return True
verbose_router_logger.debug(
"No content-policy fallback available. Returning original response. model=%s, content_policy_fallbacks=%s",
model_group,
content_policy_fallbacks,
)
return False
def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool:
"""
Determines if a content policy error should be raised.
@ -7898,27 +8070,26 @@ class Router:
if response.choices[0].finish_reason != "content_filter":
return False
content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks)
return self._has_content_policy_fallback(model, kwargs)
### ONLY RAISE ERROR IF CP FALLBACK AVAILABLE ###
if content_policy_fallbacks is not None:
fallback_model_group = None
for item in content_policy_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
if list(item.keys())[0] == model:
fallback_model_group = item[model]
break
if fallback_model_group is not None:
return True
elif self._has_default_fallbacks(): # default fallbacks set
return True
verbose_router_logger.debug(
"Content Policy Error occurred. No available fallbacks. Returning original response. model=%s, content_policy_fallbacks=%s",
model,
content_policy_fallbacks,
def _should_raise_anthropic_refusal_error(
self, model: str, original_generic_function: Callable, response: object, kwargs: Mapping[str, Any]
) -> bool:
"""
The /v1/messages twin of _should_raise_content_policy_error: an Anthropic safeguard
refusal (stop_reason "refusal" carrying stop_details) re-enters the fallback chain only
when a content-policy fallback is configured; a plain refusal without stop_details, or
any response with nothing configured, is returned to the client unchanged.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
get_safeguard_refusal_stop_details,
)
return False
if getattr(original_generic_function, "__name__", "") != "anthropic_messages":
return False
if get_safeguard_refusal_stop_details(response) is None:
return False
return self._has_content_policy_fallback(model, kwargs)
def _get_healthy_deployments(self, model: str, parent_otel_span: Span | None):
_all_deployments: list = []
@ -8352,6 +8523,19 @@ class Router:
return deployment
except Exception as e:
if self.ignore_invalid_deployments:
if isinstance(e, litellm.BadRequestError):
self._provider_unresolved_deployments = (
*self._provider_unresolved_deployments,
partial(
self._create_deployment,
deployment_info=deployment_info,
_model_name=_model_name,
_litellm_params=_litellm_params,
_model_info=_model_info,
declared_id=declared_id,
duplicate_ids=duplicate_ids,
),
)
verbose_router_logger.exception(
"Error creating deployment: %s, ignoring and continuing with other deployments.", e
)
@ -8781,6 +8965,7 @@ class Router:
self.quality_routers = {}
self.complexity_routers = {}
self.auto_routers = {}
self._provider_unresolved_deployments = ()
self._invalidate_model_group_info_cache()
self._invalidate_access_groups_cache()
# we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
@ -9403,8 +9588,12 @@ class Router:
"""Re-assert this router's deployments onto a freshly fetched catalog.
Reads ``model_list`` at call time, so only deployments the router still
serves are restored.
serves are restored, plus any config deployment the fresh catalog now resolves.
"""
provider_unresolved: Final = self._provider_unresolved_deployments
self._provider_unresolved_deployments = ()
for create_deployment in provider_unresolved:
create_deployment()
for entry in tuple(self.model_list):
try:
deployment = entry if isinstance(entry, Deployment) else Deployment(**entry)
@ -9607,6 +9796,26 @@ class Router:
coerce_token_limit(model_info.get("max_output_tokens")),
)
def get_configured_display_name(self, model_name: str) -> "str | None":
"""
Return the display_name explicitly configured in a concrete deployment's
model_info for model_name, via O(1) index lookup.
Returns None for wildcard-expanded or unknown names, and treats a
non-string or empty configured value as absent rather than failing the
listing. Like get_configured_token_limits, this never triggers pattern
matching or deep copies, so it is safe to call per listed model on the
/v1/models hot path.
"""
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
if deployment is None:
return None
display_name: Final = deployment.model_info.get("display_name")
if isinstance(display_name, str) and display_name.strip():
return display_name
return None
def get_deployment_credentials_with_provider(
self, model_id: str, team_id: str | None = None
) -> dict[str, Any] | None:
@ -11192,27 +11401,6 @@ class Router:
"""
Update the router settings.
"""
# only the following settings are allowed to be configured
_allowed_settings: Final = [
"routing_strategy_args",
"routing_strategy",
"routing_groups",
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"max_retries",
"retry_after",
"fallbacks",
"context_window_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
"tag_routing_prefix",
]
_int_settings: Final = [
"timeout",
"num_retries",
@ -11225,13 +11413,15 @@ class Router:
rebuild_routing_groups = False
relink_lar1_from_args = False
for var in kwargs:
if var in _allowed_settings:
if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS:
if var in _int_settings:
_casted_value = int(kwargs[var])
setattr(self, var, _casted_value)
elif var == "routing_groups":
self._routing_groups_input = kwargs[var]
rebuild_routing_groups = True
elif var == "optional_pre_call_checks":
self.set_optional_pre_call_checks(kwargs[var])
elif var == "retry_policy":
value = kwargs[var]
if isinstance(value, dict):
@ -12087,6 +12277,7 @@ class Router:
if pre_routing_hook_response is not None:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
record_pre_routing_selection(request_kwargs, model)
if pre_routing_hook_response.litellm_params:
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
@ -12202,6 +12393,7 @@ class Router:
if pre_routing_hook_response is not None:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
record_pre_routing_selection(request_kwargs, model)
if pre_routing_hook_response.litellm_params:
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs

Some files were not shown because too many files have changed in this diff Show more