scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into devin_ai_1788201394-veo31-pricing-tiers

Browse source
This commit is contained in:
Devin AI 2026-09-01 13:03:19 +00:00
commit 40738355b4
294 changed files with 19444 additions and 3468 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.github/actions/cache-cargo-build/action.yml vendored
View file

@ -4,17 +4,16 @@ description: >-
so only the first job on a given Cargo.lock compiles the bridge from scratch.
litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
in release mode before it can produce a wheel. `uv sync` therefore pays a full
build in every job that installs the workspace: measured at 2m40s per unit shard
on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
it, because the uv cache holds wheels uv downloads rather than wheels it builds,
and a path dependency whose source moves every commit could never hit that cache
anyway. Cargo rebuilds only what changed when its target directory survives, so a
warm job pays for the bridge crate alone.
in the dev profile for editable installs. `uv sync` therefore pays a full build
in every job that installs the workspace. Nothing caught it, because the uv cache
holds wheels uv downloads rather than wheels it builds, and a path dependency
whose source moves every commit could never hit that cache anyway. Cargo rebuilds
only what changed when its target directory survives, so a warm job pays for the
bridge crate alone.
The key namespace is separate from test-rust.yml's. Both cache the same directory,
but that workflow fills it with debug and clippy artifacts, which a release build
cannot reuse, and a shared key would let whichever ran first deny the other a save.
The key namespace is separate from test-rust.yml's check and release caches. They
cache the same directory for different workloads, and a shared key would let
whichever ran first deny the others a save.
runs:
using: composite
@ -26,6 +25,6 @@ runs:
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-release-
${{ runner.os }}-maturin-dev-

77
.github/workflows/test-redis-compat.yml vendored Normal file
View file

@ -0,0 +1,77 @@
name: "Unit Tests: Redis Client Version Compatibility"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
paths:
- "litellm/_redis.py"
- "litellm/_redis_credential_provider.py"
- "tests/test_litellm/test_redis.py"
- "tests/test_litellm/caching/test_redis_connection_pool.py"
- ".github/workflows/test-redis-compat.yml"
- "pyproject.toml"
- "uv.lock"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
redis-compat:
name: "redis-py ${{ matrix.redis-version }}"
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
# newer legs prove the inspect.signature introspection in litellm/_redis.py
# keeps extracting kwargs on the redis-py releases people actually run now.
# Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
# specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
# for the 6.x line.
redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Pin redis-py to the matrix version
env:
REDIS_VERSION: ${{ matrix.redis-version }}
run: |
uv pip install "redis==${REDIS_VERSION:?}"
uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
- name: Run redis unit tests
run: |
uv run --no-sync pytest \
tests/test_litellm/test_redis.py \
tests/test_litellm/caching/test_redis_connection_pool.py \
--tb=short -vv \
--reruns 2 \
--reruns-delay 1 \
--durations=20

1
.github/workflows/test-unit.yml vendored
View file

@ -103,6 +103,7 @@ jobs:
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/endpoints
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories

15
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apk add --no-cache \
bash \
gcc \
python3 \
python3-dev \
python-3.13 \
python-3.13-dev \
rust \
openssl \
openssl-dev \
@ -51,6 +51,7 @@ RUN apk add --no-cache \
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
@ -65,7 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3
--python python3.13
# Copy full source tree
COPY . .
@ -86,7 +87,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
@ -101,7 +102,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}" \

12
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
COPY . .
@ -57,7 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
@ -71,7 +71,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

22
basedpyright-code-budget.json
View file

@ -1,9 +1,9 @@
{
"reportAny": {
"limit": 16171
"limit": 14765
},
"reportArgumentType": {
"limit": 2226
"limit": 2216
},
"reportAssignmentType": {
"limit": 319
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 5199
"limit": 4493
},
"reportFunctionMemberAccess": {
"limit": 7
@ -42,7 +42,7 @@
"limit": 12
},
"reportIndexIssue": {
"limit": 35
"limit": 25
},
"reportInvalidTypeForm": {
"limit": 34
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5611
"limit": 5607
},
"reportMissingTypeArgument": {
"limit": 15350
"limit": 15310
},
"reportMissingTypeStubs": {
"limit": 40
@ -99,19 +99,19 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44368
"limit": 44364
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38468
"limit": 38368
},
"reportUnknownParameterType": {
"limit": 19665
"limit": 19633
},
"reportUnknownVariableType": {
"limit": 30066
"limit": 29908
},
"reportUnnecessaryCast": {
"limit": 111
@ -141,6 +141,6 @@
"limit": 543
},
"reportUnusedVariable": {
"limit": 139
"limit": 137
}
}

15
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apk add --no-cache \
bash \
gcc \
python3 \
python3-dev \
python-3.13 \
python-3.13-dev \
openssl \
openssl-dev \
nodejs \
@ -49,6 +49,7 @@ RUN apk add --no-cache \
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
@ -63,7 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3
--python python3.13
# Copy full source tree
COPY . .
@ -84,7 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
@ -98,7 +99,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}" \

17
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
python3-dev \
python-3.13 \
python-3.13-dev \
gcc \
rust \
bash \
@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}" \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache
@ -69,7 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3
--python python3.13
# Copy full source tree
COPY . .
@ -96,7 +97,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3 \
--python python3.13 \
--no-sources-package litellm-proxy-extras; \
else \
uv sync --frozen --no-default-groups --no-editable \
@ -105,7 +106,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3; \
--python python3.13; \
fi
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@ -124,7 +125,7 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
done
# Copy only what runtime needs. The application is installed inside the venv;

26
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -5,7 +5,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Tuple, cast
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -87,7 +87,7 @@ class CheckBatchCost:
return
self.batch_processed_support_confirmed = True
async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> dict[str, str | None]:
"""
Look up user email and key alias by user_id for enriching the S3 callback metadata.
Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None).
@ -97,8 +97,10 @@ class CheckBatchCost:
if not user_id:
return {}
try:
user_row = await self.prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_id}
user_row: prisma_models.LiteLLM_UserTable | None = (
await self.prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_id}
)
)
if user_row is None:
return {}
@ -115,8 +117,10 @@ class CheckBatchCost:
if not api_key:
return None
try:
key_row = await self.prisma_client.db.litellm_verificationtoken.find_unique(
where={"token": api_key}
key_row: prisma_models.LiteLLM_VerificationToken | None = (
await self.prisma_client.db.litellm_verificationtoken.find_unique(
where={"token": api_key}
)
)
return getattr(key_row, "key_alias", None) if key_row is not None else None
except Exception as e:
@ -128,8 +132,10 @@ class CheckBatchCost:
if not team_id:
return None
try:
team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": team_id}
team_row: prisma_models.LiteLLM_TeamTable | None = (
await self.prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": team_id}
)
)
return getattr(team_row, "team_alias", None) if team_row is not None else None
except Exception as e:
@ -138,7 +144,7 @@ class CheckBatchCost:
async def _build_creator_attribution_metadata(
self, job: "LiteLLM_ManagedObjectTable", batch_id: str
) -> Dict[str, Any]:
) -> dict[str, object]:
"""
Rebuild the spend-tracking metadata for the key, team, and tags that created the
batch so the batch-cost spend log is attributed the same way a non-batch request
@ -152,7 +158,7 @@ class CheckBatchCost:
team_id = getattr(job, "team_id", None)
request_tags = getattr(job, "request_tags", None)
metadata: Dict[str, Any] = {
metadata: dict[str, object] = {
"user_api_key_user_id": job.created_by,
"user_api_key": api_key,
"user_api_key_team_id": team_id,

23
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -182,6 +182,10 @@ class _ManagedObjectTableActions(Protocol):
async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
class _SchedulerWithJobLookup(Protocol):
def get_job(self, job_id: str) -> object: ...
class _CursorPageArgs(TypedDict, total=False):
cursor: Mapping[str, str]
skip: int
@ -853,7 +857,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_ids.append(file_id)
return file_ids
def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, Any]]]) -> List[str]:
def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, object]]]) -> List[str]:
"""
Gets file ids from responses API input.
@ -878,7 +882,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Check for direct input_file type
if item.get("type") == "input_file":
file_id = item.get("file_id")
if file_id:
if isinstance(file_id, str) and file_id:
file_ids.append(file_id)
# Check for input_file in content array
@ -887,7 +891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
file_id = content_item.get("file_id")
if file_id:
if isinstance(file_id, str) and file_id:
file_ids.append(file_id)
return file_ids
@ -1227,7 +1231,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Handle both output_file_id and error_file_id
for file_attr in ["output_file_id", "error_file_id"]:
file_id_value = getattr(response, file_attr, None)
file_id_value: str | None = getattr(response, file_attr, None)
if file_id_value and model_id:
decoded_output_file_id = _is_base64_encoded_unified_file_id(file_id_value)
if decoded_output_file_id and "llm_output_file_id," in decoded_output_file_id:
@ -1496,7 +1500,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
import litellm.proxy.proxy_server as proxy_server_module
# Check if the scheduler has the batch cost checking job registered
scheduler = getattr(proxy_server_module, "scheduler", None)
scheduler: Final[_SchedulerWithJobLookup | None] = getattr(proxy_server_module, "scheduler", None)
if scheduler is None:
return False
@ -1542,7 +1546,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
MAX_MATCHES_TO_RETURN = 10
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
batches = await _managed_object_table(self.prisma_client).find_many(
where={
"file_purpose": "batch",
"batch_processed": False,
@ -1552,11 +1556,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
order={"created_at": "desc"},
)
referencing_batches = []
referencing_batches: Final[list[dict[str, object]]] = []
for batch in batches:
try:
# Parse the batch file_object to check for file references
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
decoded_file_object = _decode_json_blob(batch.file_object)
batch_data: Mapping[str, object] = (
decoded_file_object if isinstance(decoded_file_object, Mapping) else {}
)
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id

12
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \
--python python3
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
COPY . .
@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \
--python python3
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

1
gateway/routes/allowlist.py
View file

@ -86,6 +86,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/comprehendmedical",
"/cohere/",
"/gemini/",
"/gigachat/",
"/google/",
"/vertex_ai/",
"/vertex-ai/",

21
litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql Normal file
View file

@ -0,0 +1,21 @@
DO $$
BEGIN
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id'
) THEN
ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id";
END IF;
END $$;
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key';
DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction";
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction"
ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL;
DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx";
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx"
ON "LiteLLM_ShadowEvalJob"("target_type", "target_id");

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "router_names" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "router_name" TEXT;

11
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1529,14 +1529,16 @@ model LiteLLM_AutoRouterSession {
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
group_id String // legs of one job share this; the API's job id
api_key_id String // hashed virtual key whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
target_type String @default("key") // key | team | user
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
router_name String // first (often only) auto-router under evaluation; router_names is the full set
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
created_at DateTime @default(now())
created_by String?
ends_at DateTime
@ -1544,7 +1546,7 @@ model LiteLLM_ShadowEvalJob {
stopped_by String? // operator who stopped it early; null when it ended on its own
@@index([group_id])
@@index([api_key_id])
@@index([target_type, target_id])
@@index([created_at])
}
@ -1554,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt {
job_id String
request_id String // the judged real request
outcome String // real | shadow | tie | error
router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router
tier String? // router's tier for the prompt, when classified
real_model String?
shadow_model String?

7
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -512,6 +512,13 @@ class ProxyExtrasDBManager:
try:
import psycopg
except ImportError:
logger.warning(
"psycopg is not installed; skipping the LiteLLM_SpendLogs "
"partition check. If this table is partitioned (see "
"db_scripts/partition_spend_logs.sql), schema reconciliation "
"will try to rewrite its primary key and fail. Install the "
"litellm[extra_proxy] extra, which now includes psycopg."
)
return False
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)

9
litellm-rust/Cargo.toml
View file

@ -30,3 +30,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
base64 = "0.22"
[profile.release]
opt-level = 3
lto = "thin"
codegen-units = 1
panic = "unwind"
debug = false
incremental = false
strip = "symbols"

3
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -10,7 +10,8 @@ name = "_native"
crate-type = ["cdylib"]
[features]
default = ["extension-module"]
default = ["abi3"]
abi3 = ["pyo3/abi3-py310"]
extension-module = ["pyo3/extension-module"]
[dependencies]

3
litellm/__init__.py
View file

@ -7,6 +7,9 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os

15
litellm/_logging.py
View file

@ -264,13 +264,17 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
class LevelRoutingStreamHandler(logging.StreamHandler):
"""Writes records below WARNING to stdout and WARNING and above to stderr.
"""Writes records below WARNING and invalid-key warnings to stdout, others to stderr.
Collectors that derive severity from the stream report every stderr line as an error.
Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them.
"""
def emit(self, record: logging.LogRecord) -> None:
preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr
is_stdout_record: Final = record.levelno < logging.WARNING or (
record.levelno == logging.WARNING and record.name == verbose_proxy_stdout_logger.name
)
preferred: Final = sys.stdout if is_stdout_record else sys.stderr
if preferred is None or getattr(preferred, "closed", False):
self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record
else:
@ -508,6 +512,9 @@ else:
handler.setFormatter(formatter)
verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler
# writes its WARNING records to stdout. It has no handler or level of its own.
verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout")
verbose_router_logger = logging.getLogger("LiteLLM Router")
verbose_logger = logging.getLogger("LiteLLM")
@ -520,6 +527,7 @@ verbose_logger.addHandler(handler)
# handlers (JSON mode, uvicorn log config, a host app's root handler).
verbose_router_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
verbose_logger.addFilter(_stdout_truncation_filter)
@ -683,6 +691,7 @@ def _turn_on_json():
- Adds a JSON formatter to all loggers
"""
handler: Final = LevelRoutingStreamHandler()
handler.setLevel(numeric_level)
handler.setFormatter(JsonFormatter())
_initialize_loggers_with_handler(handler)
# Set up exception handlers
@ -700,12 +709,14 @@ def _disable_debugging():
verbose_logger.disabled = True
verbose_router_logger.disabled = True
verbose_proxy_logger.disabled = True
verbose_proxy_stdout_logger.disabled = True
def _enable_debugging():
verbose_logger.disabled = False
verbose_router_logger.disabled = False
verbose_proxy_logger.disabled = False
verbose_proxy_stdout_logger.disabled = False
def print_verbose(print_statement):

125
litellm/_redis.py
View file

@ -13,6 +13,7 @@ import json
# s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
import os
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from urllib.parse import urlsplit, urlunsplit
@ -38,9 +39,25 @@ from ._logging import verbose_logger
AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default"
def _get_redis_kwargs():
arg_spec: Final = inspect.getfullargspec(redis.Redis)
def _unwrapped_init_args(cls: type) -> frozenset[str]:
"""Every parameter on a single class's own ``__init__``, decorator-unwrapped.
Unlike ``_init_arg_names`` below, this does not walk the MRO: ``redis.Redis``
and ``redis.RedisCluster`` (sync and async) each declare every real
constructor parameter directly on their own ``__init__``, so MRO-walking is
unnecessary — and it actively breaks the several tests here that mock the
class with ``patch(..., autospec=True)``, since ``inspect.getmro`` needs a
real ``__mro__`` that an autospec'd stand-in for a class does not provide.
Still unwraps first: redis-py >= 7.4 decorates these ``__init__``s with
``@deprecated_args`` too, which the same class of bug as ``_init_arg_names``
would otherwise silently empty this allowlist through (see its docstring).
"""
spec: Final = inspect.getfullargspec(inspect.unwrap(cls.__init__))
return frozenset(spec.args + spec.kwonlyargs)
def _get_redis_kwargs():
# Only allow primitive arguments
exclude_args: Final = {
"self",
@ -60,7 +77,7 @@ def _get_redis_kwargs():
"azure_client_secret",
}
available_args: Final = {x for x in arg_spec.args if x not in exclude_args} | include_args
available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args
return available_args
@ -120,15 +137,23 @@ def _get_redis_url_kwargs(client: type | None = None) -> tuple[str, ...]:
return tuple(x for x in _init_arg_names(connection_cls) if x not in exclude_args) + include_args
def _get_redis_cluster_kwargs(client=None):
def _get_redis_cluster_kwargs(client: type | None = None):
"""Config kwargs the target cluster client's constructor actually accepts.
Defaults to the sync ``redis.RedisCluster``, but the async cluster client
(``redis.asyncio.cluster.RedisCluster``) declares connection settings such as
``decode_responses`` on its own constructor, where the sync class takes them
through ``**kwargs`` and so never names them in its signature. Introspecting
only the sync class regardless of which client is actually built silently
drops those for every async cluster caller.
"""
if client is None:
client = redis.Redis.from_url
arg_spec: Final = inspect.getfullargspec(redis.RedisCluster)
client = redis.RedisCluster
# Only allow primitive arguments
exclude_args: Final = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"}
available_args = {x for x in arg_spec.args if x not in exclude_args}
available_args = {x for x in _unwrapped_init_args(client) if x not in exclude_args}
available_args |= {
"password",
"username",
@ -161,6 +186,79 @@ def _get_redis_env_kwarg_mapping():
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment}
def _str_to_bool(value: str) -> bool:
return value.lower() in ("true", "1", "yes")
def _coerce_redis_kwargs_types(
redis_kwargs: Mapping[str, object],
client: type | tuple[type, ...] = redis.Redis,
) -> dict[str, object]: # mutable-ok: a caller mutates the returned kwargs before constructing its client
"""Coerces string values to the numeric/boolean type ``client``'s constructor
declares for that parameter. ``client`` may be a tuple of client classes; a
parameter's type is taken from the first signature that declares it, which
lets cluster callers coerce cluster-only kwargs such as
``cluster_error_retry_attempts`` alongside the shared connection kwargs.
Environment variables are always strings, and Helm ``--set`` stringifies values
too, so a config value like ``health_check_interval`` or ``socket_timeout``
can arrive as ``"30"``/``"5.5"`` rather than a real number. redis-py's own
connection-health-check arithmetic (``loop.time() + self.health_check_interval``)
then raises ``TypeError`` on every Redis operation instead of connecting.
``max_connections``, ``socket_timeout``, and ``socket_connect_timeout`` use an
explicit target type rather than the parameter's own signature default: redis-py
8.x changed the timeout defaults from ``None`` to int ``5``, so inferring the
type from the default would make a fractional ``"5.5"`` fail ``int()`` and get
silently dropped on 8.x while working on older versions. ``socket_keepalive``
is explicit too: its signature default is ``None``, which carries no type to
infer from, and leaving it a string makes ``"false"`` truthy.
"""
signatures: Final = tuple(inspect.signature(c) for c in (client if isinstance(client, tuple) else (client,)))
explicit_param_types: Final = MappingProxyType(
{
"max_connections": int,
"socket_timeout": float,
"socket_connect_timeout": float,
"socket_keepalive": bool,
}
)
result: Final = dict(redis_kwargs) # mutable-ok: per-key try/except coercion below needs to drop individual keys
for key, value in redis_kwargs.items():
if not isinstance(value, str):
continue
param = next((sig.parameters[key] for sig in signatures if key in sig.parameters), None)
if param is None:
continue
explicit_type = explicit_param_types.get(key)
if explicit_type is bool:
result[key] = _str_to_bool(value)
continue
if explicit_type is not None:
try:
result[key] = explicit_type(value)
except (ValueError, TypeError):
del result[key]
continue
default: object = param.default # pyright: ignore[reportAny] # inspect.Parameter.default is stubbed as Any
if default is inspect.Parameter.empty:
continue
# bool must be checked before int, since bool subclasses int
if isinstance(default, bool):
result[key] = _str_to_bool(value)
elif isinstance(default, int):
try:
result[key] = int(value)
except (ValueError, TypeError):
del result[key]
elif isinstance(default, float):
try:
result[key] = float(value)
except (ValueError, TypeError):
del result[key]
return result
def _redis_kwargs_from_environment():
mapping: Final = _get_redis_env_kwarg_mapping()
@ -505,7 +603,12 @@ def _get_redis_client_logic(**env_overrides):
raise ValueError("Either 'host' or 'url' must be specified for redis.")
# litellm.print_verbose(f"redis_kwargs: {redis_kwargs}")
return redis_kwargs
coercion_client: Final = (
(redis.Redis, redis.RedisCluster, async_redis.RedisCluster)
if redis_kwargs.get("startup_nodes")
else redis.Redis
)
return _coerce_redis_kwargs_types(redis_kwargs, client=coercion_client)
def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
@ -657,7 +760,9 @@ def get_redis_client(**env_overrides):
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
return _init_redis_sentinel(redis_kwargs)
return redis.Redis(**redis_kwargs)
return redis.Redis( # pyright: ignore[reportCallIssue] # object-valued kwargs match no overload statically
**redis_kwargs, # pyright: ignore[reportArgumentType] # allow-listed and coerced against this signature
)
def get_redis_async_client(
@ -669,7 +774,7 @@ def get_redis_async_client(
if "startup_nodes" in redis_kwargs:
from redis.cluster import ClusterNode
args = _get_redis_cluster_kwargs()
args = _get_redis_cluster_kwargs(async_redis.RedisCluster)
cluster_kwargs: Final = {}
for arg in redis_kwargs:
if arg in args:

57
litellm/caching/redis_cache.py
View file

@ -18,7 +18,7 @@ import time
from collections.abc import Awaitable, Callable, Sequence
from contextvars import ContextVar
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -58,6 +58,26 @@ else:
Span = Any
class _AsyncRedisCommands(Protocol):
"""Async redis commands this cache issues.
redis-py's type stubs omit these methods on RedisCluster, so the union returned by
init_async_client() is untyped at every call site without this protocol.
"""
def ping(self) -> Awaitable[bool]: ...
def delete(self, *names: str) -> Awaitable[int]: ...
def ttl(self, name: str) -> Awaitable[int]: ...
def rpush(self, name: str, *values: str | bytes | float) -> Awaitable[int]: ...
def lpop(self, name: str, count: int | None = None) -> Awaitable[object]: ...
def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
def _get_call_stack_info(num_frames: int = 2) -> str:
"""
Get the function names from the previous 1-2 functions in the call stack.
@ -429,6 +449,9 @@ class RedisCache(BaseCache):
self.redis_async_client = redis_async_client
return redis_async_client
def _async_commands(self) -> _AsyncRedisCommands:
return self.init_async_client()
def check_and_fix_namespace(self, key: str) -> str:
"""
Make sure each key starts with the given namespace
@ -1055,19 +1078,17 @@ class RedisCache(BaseCache):
await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
self.redis_batch_writing_buffer = []
def _get_cache_logic(self, cached_response: Any):
def _get_cache_logic(self, cached_response: bytes | str | None):
"""
Common 'get_cache_logic' across sync + async redis client implementations
"""
if cached_response is None:
return cached_response
# cached_response is in `b{} convert it to ModelResponse
cached_response = cached_response.decode("utf-8") # Convert bytes to string
return None
decoded: Final = cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response
try:
cached_response = json.loads(cached_response) # Convert string to dictionary
return json.loads(decoded)
except Exception:
cached_response = ast.literal_eval(cached_response)
return cached_response
return ast.literal_eval(decoded)
def get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
try:
@ -1314,8 +1335,7 @@ class RedisCache(BaseCache):
raise e
async def ping(self) -> bool:
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `ping`
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
start_time: Final = time.time()
print_verbose("Pinging Async Redis Cache")
try:
@ -1349,8 +1369,7 @@ class RedisCache(BaseCache):
@_redis_circuit_breaker_guard
async def delete_cache_keys(self, keys):
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
keys = [self.check_and_fix_namespace(key=key) for key in keys]
# keys is a list, unpack it so it gets passed as individual elements to delete
await _redis_client.delete(*keys)
@ -1415,8 +1434,7 @@ class RedisCache(BaseCache):
@_redis_circuit_breaker_guard
async def async_delete_cache(self, key: str):
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
key = self.check_and_fix_namespace(key=key)
# keys is str
return await _redis_client.delete(key)
@ -1523,8 +1541,7 @@ class RedisCache(BaseCache):
Redis ref: https://redis.io/docs/latest/commands/ttl/
"""
try:
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `ttl`
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
key = self.check_and_fix_namespace(key=key)
ttl: Final = await _redis_client.ttl(key)
if ttl <= -1: # -1 means the key does not exist, -2 key does not exist
@ -1554,7 +1571,7 @@ class RedisCache(BaseCache):
Returns:
int: The length of the list after the push operation
"""
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
key = self.check_and_fix_namespace(key=key)
start_time: Final = time.time()
try:
@ -1621,7 +1638,7 @@ class RedisCache(BaseCache):
if len(rpush_list) == 0:
return []
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
start_time: Final = time.time()
try:
@ -1678,7 +1695,7 @@ class RedisCache(BaseCache):
parent_otel_span: Span | None = None,
**kwargs,
) -> Any | list[Any]:
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
key = self.check_and_fix_namespace(key=key)
start_time: Final = time.time()
print_verbose(f"LPOP from Redis list: key: {key}, count: {count}")
@ -1810,7 +1827,7 @@ class RedisCache(BaseCache):
if len(lpop_list) == 0:
return []
_redis_client: Final[Any] = self.init_async_client()
_redis_client: Final = self._async_commands()
start_time: Final = time.time()
try:

7
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -212,7 +212,8 @@ def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _Ch
LiteLLMCompletionResponsesConfig,
)
is_custom: Final = item.get("type") == "custom_tool_call"
item_type: Final[object] = item.get("type")
is_custom: Final = item_type == "custom_tool_call"
arguments: Final = (item.get("input") if is_custom else item.get("arguments")) or ""
name: Final = item.get("name") or ("custom_tool" if is_custom else "")
function_chunk: Final = ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments)
@ -222,7 +223,7 @@ def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _Ch
function=function_chunk,
index=index,
)
raw_provider_fields: Final = item.get("provider_specific_fields")
raw_provider_fields: Final[object] = item.get("provider_specific_fields")
if isinstance(raw_provider_fields, dict):
provider_specific_fields = raw_provider_fields
elif raw_provider_fields and hasattr(raw_provider_fields, "__dict__"):
@ -507,7 +508,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _merge_responses_api_request_into_request_data(
self,
request_data: dict[str, Any],
request_data: dict[str, object],
responses_api_request: "ResponsesAPIOptionalRequestParams",
instructions: str | None,
) -> None:

23
litellm/constants.py
View file

@ -484,6 +484,22 @@ FIREWORKS_AI_80_B: Final = int(os.getenv("FIREWORKS_AI_80_B", 80))
#### Logging callback constants ####
REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM"
MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = int(os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50))
# Backpressure + lifetime bounds for the /v1/messages streaming relay (see
# BaseAnthropicMessagesStreamingIterator.async_sse_wrapper). The relay queue is
# bounded so a slow client throttles the upstream pump instead of letting it
# buffer the whole response in memory; the detached-drain cap bounds how many
# post-disconnect drains may run concurrently so client behavior can't create
# unbounded worker state.
ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = int(
os.getenv("ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "1024")
)
# Setting this to 0 disables detached draining entirely: every post-disconnect
# pump bills whatever partial output it has already collected and aborts the
# upstream stream immediately, instead of continuing to drain for the real
# terminal usage.
ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int(
os.getenv("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", "100")
)
LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0
LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000))
LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0))
@ -806,6 +822,7 @@ openai_compatible_endpoints: Final[list] = [
"https://api.meta.ai/v1",
"https://api.cognition.ai/v1",
"https://api.scx.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
]
@ -1410,6 +1427,12 @@ DEFAULT_SOFT_BUDGET: Final = float(
) # by default all litellm proxy keys have a soft budget of 50.0
# makes it clear this is a rate limit error for a litellm virtual key
RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash"
# Prefix of the 401 raised when a submitted virtual key is not shaped like one.
INVALID_VIRTUAL_KEY_ERROR_MESSAGE: Final = "LiteLLM Virtual Key expected"
# Attribute stamped on that 401 at its raise site so log routing recognises it by
# provenance. Message text is caller-influenceable on other 401s, so it must not
# be used to classify.
INVALID_VIRTUAL_KEY_ERROR_MARKER: Final = "_litellm_invalid_virtual_key_error"
# Python garbage collection threshold configuration
# Format: "gen0,gen1,gen2" e.g., "1000,50,50"

12
litellm/cost_calculator.py
View file

@ -2268,6 +2268,10 @@ def batch_cost_calculator(
return total_prompt_cost, total_completion_cost
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
field_names: Final = list(type(prompt_tokens_details).model_fields)
if getattr(prompt_tokens_details, "cache_write_tokens", None) is None:
@ -2293,7 +2297,7 @@ class BaseTokenUsageProcessor:
for usage in usage_objects:
# Handle direct attributes by checking what exists in the model
for attr in dir(usage):
if not attr.startswith("_") and not callable(getattr(usage, attr)):
if not attr.startswith("_") and not callable(_attribute_value(usage, attr)):
current_val = getattr(combined, attr, 0)
new_val = getattr(usage, attr, 0)
if (
@ -2313,7 +2317,7 @@ class BaseTokenUsageProcessor:
if (
hasattr(usage.prompt_tokens_details, attr)
and not attr.startswith("_")
and not callable(getattr(usage.prompt_tokens_details, attr))
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
):
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
@ -2332,7 +2336,9 @@ class BaseTokenUsageProcessor:
# Check what keys exist in the model's completion_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in type(usage.completion_tokens_details).model_fields:
if not attr.startswith("_") and not callable(getattr(usage.completion_tokens_details, attr)):
if not attr.startswith("_") and not callable(
_attribute_value(usage.completion_tokens_details, attr)
):
current_val = getattr(combined.completion_tokens_details, attr, 0) or 0
new_val = getattr(usage.completion_tokens_details, attr, 0) or 0
if isinstance(new_val, (int, float)):

2
litellm/endpoints/speech/speech_to_completion_bridge/handler.py
View file

@ -115,9 +115,11 @@ class SpeechToCompletionBridgeHandler:
**request_data,
)
requested_response_format: Final = optional_params.get("response_format")
if isinstance(result, ModelResponse):
return self.transformation_handler.transform_response(
model_response=result,
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
)
else:
raise Exception(f"Unmapped response type. Got type: {type(result)}")

126
litellm/endpoints/speech/speech_to_completion_bridge/transformation.py
View file

@ -1,10 +1,14 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
if TYPE_CHECKING:
from litellm import Logging as LiteLLMLoggingObj
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent
from litellm.types.utils import ModelResponse
@ -16,7 +20,64 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None:
return response_cost if isinstance(response_cost, float) else None
GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16"
GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm"
GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT})
class ChatAudioParam(TypedDict):
voice: ReadOnly[str]
format: ReadOnly[NotRequired[str]]
class SpeechToCompletionBridgeTransformationHandler:
def _validate_response_format(
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
) -> None:
if not self._is_gemini_tts_model(model):
return
response_format: Final = optional_params.get("response_format")
if not isinstance(response_format, str) or response_format in GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS:
return
from litellm.exceptions import BadRequestError
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
raise BadRequestError(
message=(
f"Gemini TTS only produces raw PCM16 audio, so response_format='{response_format}'"
f" is not supported. Supported response formats: {supported}."
),
model=model,
llm_provider=custom_llm_provider,
)
def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType(
{
param: value
for param, value in optional_params.items()
if param in OPENAI_CHAT_COMPLETION_PARAMS and param != "response_format"
}
)
def _chat_audio_format(self, model: str, optional_params: Mapping[str, object]) -> str | None:
if self._is_gemini_tts_model(model):
return GEMINI_TTS_CHAT_AUDIO_FORMAT
response_format: Final = optional_params.get("response_format")
return response_format if isinstance(response_format, str) else None
def _chat_audio_param(
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
) -> ChatAudioParam | None:
if not isinstance(voice, str):
return None
audio_format: Final = self._chat_audio_format(model, optional_params)
if audio_format is None:
voice_only: Final[ChatAudioParam] = {"voice": voice}
return voice_only
audio: Final[ChatAudioParam] = {"voice": voice, "format": audio_format}
return audio
def transform_request(
self,
model: str,
@ -28,36 +89,20 @@ class SpeechToCompletionBridgeTransformationHandler:
litellm_logging_obj: "LiteLLMLoggingObj",
custom_llm_provider: str,
) -> dict:
passed_optional_params: Final = {}
for op in optional_params:
if op in OPENAI_CHAT_COMPLETION_PARAMS:
passed_optional_params[op] = optional_params[op]
if voice is not None:
if isinstance(voice, str):
passed_optional_params["audio"] = {"voice": voice}
if "response_format" in optional_params:
passed_optional_params["audio"]["format"] = optional_params["response_format"]
return_kwargs = {
self._validate_response_format(model, custom_llm_provider, optional_params)
user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input}
return_kwargs: Final = {
"model": model,
"messages": [
{
"role": "user",
"content": input,
}
],
"messages": [user_message],
"modalities": ["audio"],
**passed_optional_params,
**self._chat_completion_params(optional_params),
"audio": self._chat_audio_param(model, voice, optional_params),
**litellm_params,
"headers": headers,
"litellm_logging_obj": litellm_logging_obj,
"custom_llm_provider": custom_llm_provider,
}
# filter out None values
return_kwargs = {k: v for k, v in return_kwargs.items() if v is not None}
return return_kwargs
return {k: v for k, v in return_kwargs.items() if v is not None}
def _convert_pcm16_to_wav(self, pcm_data: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes:
"""
@ -103,7 +148,14 @@ class SpeechToCompletionBridgeTransformationHandler:
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
def transform_response(self, model_response: "ModelResponse") -> "HttpxBinaryResponseContent":
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
return decoded_audio, "audio/pcm"
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
def transform_response(
self, model_response: "ModelResponse", response_format: str | None
) -> "HttpxBinaryResponseContent":
import base64
import httpx
@ -114,23 +166,17 @@ class SpeechToCompletionBridgeTransformationHandler:
audio_part: Final = cast(Choices, model_response.choices[0]).message.audio
if audio_part is None:
raise ValueError("No audio part found in the response")
audio_content: Final = audio_part.data
decoded_audio: Final = base64.b64decode(audio_part.data)
# Decode base64 to get binary content
binary_data = base64.b64decode(audio_content)
# Check if this is a Gemini TTS model that returns raw PCM16 data
model: Final = getattr(model_response, "model", "")
headers: Final = {}
if self._is_gemini_tts_model(model):
# Convert PCM16 to WAV format for proper audio file playback
binary_data = self._convert_pcm16_to_wav(binary_data)
headers["Content-Type"] = "audio/wav"
else:
headers["Content-Type"] = "audio/mpeg"
# Create an httpx.Response object
response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
content, content_type = (
self._gemini_tts_response_body(decoded_audio, response_format)
if self._is_gemini_tts_model(model)
else (decoded_audio, "audio/mpeg")
)
response: Final = httpx.Response(
status_code=200, content=content, headers=MappingProxyType({"Content-Type": content_type})
)
binary_response: Final = HttpxBinaryResponseContent(response)
binary_response.set_response_cost(_completion_response_cost(model_response))
return binary_response

3
litellm/google_genai/adapters/transformation.py
View file

@ -108,7 +108,6 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
def __init__(self, completion_stream: object):
self.sent_first_chunk = False
# State tracking for accumulating partial tool calls
self.accumulated_tool_calls = dict[int, _ToolCallAccumulator]()
self._returned_response = False
super().__init__(completion_stream)
@ -723,7 +722,7 @@ class GoogleGenAIAdapter:
)
for tool_call in tool_calls:
if not hasattr(tool_call, "function"):
if not hasattr(tool_call, "function") or isinstance(tool_call, ChatCompletionDeltaCustomToolCall):
continue
# 3. Use `index` as the primary key for accumulation

72
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -3,10 +3,12 @@ Arize Phoenix prompt manager that integrates with LiteLLM's prompt management sy
Fetches prompt versions from Arize Phoenix and provides workspace-based access control.
"""
from typing import Any, Final
from collections.abc import Mapping, Sequence
from typing import Any, Final, cast
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import (
@ -20,6 +22,31 @@ from litellm.types.utils import StandardCallbackDynamicParams
from .arize_phoenix_client import ArizePhoenixClient
class ArizePhoenixContentPart(TypedDict, total=False):
type: ReadOnly[str]
text: ReadOnly[str]
class ArizePhoenixTemplateMessage(TypedDict, total=False):
role: ReadOnly[str]
content: ReadOnly[Sequence[ArizePhoenixContentPart]]
class ArizePhoenixTemplateBody(TypedDict, total=False):
messages: ReadOnly[Sequence[ArizePhoenixTemplateMessage]]
class ArizePhoenixPromptMetadata(TypedDict):
model_name: ReadOnly[str | None]
model_provider: ReadOnly[str | None]
description: ReadOnly[str]
template_type: ReadOnly[str | None]
template_format: ReadOnly[str]
invocation_parameters: ReadOnly[Mapping[str, Mapping[str, object]]]
temperature: ReadOnly[float | None]
max_tokens: ReadOnly[int | None]
class ArizePhoenixPromptTemplate:
"""
Represents a prompt template loaded from Arize Phoenix.
@ -28,10 +55,10 @@ class ArizePhoenixPromptTemplate:
def __init__(
self,
template_id: str,
messages: list[dict[str, Any]],
metadata: dict[str, Any],
messages: Sequence[ArizePhoenixTemplateMessage],
metadata: ArizePhoenixPromptMetadata,
model: str | None = None,
):
) -> None:
self.template_id = template_id
self.messages = messages
self.metadata = metadata
@ -43,7 +70,7 @@ class ArizePhoenixPromptTemplate:
self.description = metadata.get("description", "")
self.template_format = metadata.get("template_format", "MUSTACHE")
def __repr__(self):
def __repr__(self) -> str:
return f"ArizePhoenixPromptTemplate(id='{self.template_id}', model='{self.model}')"
@ -109,7 +136,7 @@ class ArizePhoenixTemplateManager:
def _parse_prompt_data(self, data: dict[str, Any], prompt_version_id: str) -> ArizePhoenixPromptTemplate:
"""Parse Arize Phoenix prompt data and extract messages and metadata."""
template_data: Final = data.get("template", {})
template_data: Final[ArizePhoenixTemplateBody] = data.get("template", {})
messages: Final = template_data.get("messages", [])
# Extract invocation parameters
@ -129,7 +156,7 @@ class ArizePhoenixTemplateManager:
break
# Build metadata dictionary
metadata: Final = {
metadata: Final[ArizePhoenixPromptMetadata] = {
"model_name": data.get("model_name"),
"model_provider": data.get("model_provider"),
"description": data.get("description", ""),
@ -146,7 +173,9 @@ class ArizePhoenixTemplateManager:
metadata=metadata,
)
def render_template(self, template_id: str, variables: dict[str, Any] | None = None) -> list[AllMessageValues]:
def render_template(
self, template_id: str, variables: Mapping[str, object] | None = None
) -> list[AllMessageValues]:
"""Render a template with the given variables and return formatted messages."""
if template_id not in self.prompts:
raise ValueError(f"Template '{template_id}' not found")
@ -174,7 +203,9 @@ class ArizePhoenixTemplateManager:
# Combine rendered content
final_content = " ".join(rendered_content_parts)
rendered_messages.append({"role": role, "content": final_content})
rendered_messages.append(
cast("AllMessageValues", {"role": role, "content": final_content}) # cast-ok: Phoenix roles are OpenAI
)
return rendered_messages
@ -243,8 +274,8 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
def get_prompt_template(
self,
prompt_id: str,
prompt_variables: dict[str, Any] | None = None,
) -> tuple[list[AllMessageValues], dict[str, Any]]:
prompt_variables: Mapping[str, object] | None = None,
) -> tuple[list[AllMessageValues], dict[str, object]]:
"""
Get a prompt template and render it with variables.
@ -263,7 +294,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
rendered_messages: Final = self.prompt_manager.render_template(prompt_id, prompt_variables or {})
# Extract metadata
metadata: Final = {
metadata: Final[dict[str, object]] = {
"model": template.model,
"temperature": template.temperature,
"max_tokens": template.max_tokens,
@ -271,7 +302,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
# Add additional invocation parameters
invocation_params: Final = template.invocation_parameters
provider_params = {}
provider_params: Mapping[str, object] = {}
if "openai" in invocation_params:
provider_params = invocation_params["openai"]
@ -289,12 +320,12 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
self,
user_id: str | None,
messages: list[AllMessageValues],
function_call: dict[str, Any] | str | None = None,
litellm_params: dict[str, Any] | None = None,
function_call: dict[str, object] | str | None = None,
litellm_params: dict[str, object] | None = None,
prompt_id: str | None = None,
prompt_variables: dict[str, Any] | None = None,
prompt_variables: dict[str, object] | None = None,
**kwargs,
) -> tuple[list[AllMessageValues], dict[str, Any] | None]:
) -> tuple[list[AllMessageValues], dict[str, object] | None]:
"""
Pre-call hook that processes the prompt template before making the LLM call.
"""
@ -335,9 +366,9 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
except Exception as e:
# Log error but don't fail the call
import litellm
from litellm._logging import verbose_proxy_logger
litellm._logging.verbose_proxy_logger.error("Error in Arize Phoenix prompt pre_call_hook: %s", e)
verbose_proxy_logger.error("Error in Arize Phoenix prompt pre_call_hook: %s", e)
return messages, litellm_params
def get_available_prompts(self) -> list[str]:
@ -393,7 +424,8 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
rendered_messages, prompt_metadata = self.get_prompt_template(prompt_id, prompt_variables)
# Extract model from metadata (if specified)
template_model: Final = prompt_metadata.get("model")
raw_template_model: Final = prompt_metadata.get("model")
template_model: Final = raw_template_model if isinstance(raw_template_model, str) else None
# Extract optional parameters from metadata
optional_params: Final = {}

47
litellm/integrations/custom_logger.py
View file

@ -31,6 +31,9 @@ if TYPE_CHECKING:
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.mcp import (
MCPPostCallResponseObject,
@ -39,7 +42,7 @@ if TYPE_CHECKING:
)
from litellm.types.router import PreRoutingHookResponse
Span = _Span | Any
Span = _Span
else:
Span = Any
LiteLLMLoggingObj = Any
@ -268,7 +271,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
) -> list[dict]:
return healthy_deployments
async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None:
async def async_pre_call_deployment_hook(
self, kwargs: dict[str, object], call_type: CallTypes | None
) -> dict | None:
"""
Allow modifying the request just before it's sent to the deployment.
@ -344,9 +349,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_post_call_streaming_deployment_hook(
self,
request_data: dict,
response_chunk: Any,
response_chunk: object,
call_type: CallTypes | None,
) -> Any | None:
) -> object | None:
"""
Allow modifying streaming chunks just before they're returned to the user.
@ -378,7 +383,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
def translate_completion_output_params_streaming(
self, completion_stream: Any
self, completion_stream: object
) -> AdapterCompletionStreamWrapper | None:
"""
Translates the streaming chunk, from the OpenAI format to the custom format.
@ -418,9 +423,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
response: object,
request_headers: dict[str, str] | None = None,
litellm_call_info: dict[str, Any] | None = None,
litellm_call_info: dict[str, object] | None = None,
) -> dict[str, str] | None:
"""
Called after an LLM API call (success or failure) to allow injecting custom HTTP response headers.
@ -471,11 +476,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
) -> Any:
pass
async def async_logging_hook(self, kwargs: dict, result: Any, call_type: str) -> tuple[dict, Any]:
async def async_logging_hook(self, kwargs: dict, result: object, call_type: str) -> tuple[dict, object]:
"""For masking logged request/response. Return a modified version of the request/result."""
return kwargs, result
def logging_hook(self, kwargs: dict, result: Any, call_type: str) -> tuple[dict, Any]:
def logging_hook(self, kwargs: dict, result: object, call_type: str) -> tuple[dict, object]:
"""For masking logged request/response. Return a modified version of the request/result."""
return kwargs, result
@ -581,7 +586,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_should_run_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
@ -642,8 +647,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
tools: dict,
model: str,
messages: list[dict],
response: Any,
anthropic_messages_provider_config: Any,
response: object,
anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None",
anthropic_messages_optional_request_params: dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
@ -711,8 +716,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
tools: dict,
model: str,
messages: list[dict],
response: Any,
anthropic_messages_provider_config: Any,
response: object,
anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None",
anthropic_messages_optional_request_params: dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
@ -728,7 +733,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_post_agentic_loop_response_hook(
self,
response: Any,
response: object,
plan: AgenticLoopPlan,
kwargs: dict,
) -> Any:
@ -767,7 +772,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
@ -785,12 +790,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
tools: dict,
model: str,
messages: list[dict],
response: Any,
response: object,
optional_params: dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: dict,
) -> Any:
) -> object:
"""
Hook to execute chat completion agentic loop based on context from should_run hook.
"""
@ -800,7 +805,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
tools: dict,
model: str,
messages: list[dict],
response: Any,
response: object,
optional_params: dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
@ -1056,7 +1061,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
def _redact_base64(
self,
value: Any,
value: object,
depth: int = 0,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
) -> object:
@ -1079,7 +1084,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
return value
def _should_keep_content(self, content: Any) -> bool:
def _should_keep_content(self, content: object) -> bool:
"""Return True if this content item should be retained."""
if not isinstance(content, dict):
return True

59
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -2,10 +2,12 @@
GitLab prompt manager with configurable prompts folder.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, TypeVar
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -24,6 +26,19 @@ from litellm.types.utils import StandardCallbackDynamicParams
GITLAB_PREFIX: Final = "gitlab::"
_ResponseT = TypeVar("_ResponseT")
class GitLabCachedPrompt(TypedDict):
id: ReadOnly[str]
path: ReadOnly[str]
content: ReadOnly[str]
metadata: ReadOnly[Mapping[str, object]]
model: ReadOnly[str | None]
temperature: ReadOnly[float | None]
max_tokens: ReadOnly[int | None]
optional_params: ReadOnly[Mapping[str, object]]
def encode_prompt_id(raw_id: str) -> str:
"""Convert GitLab path IDs like 'invoice/extract' → 'gitlab::invoice::extract'"""
@ -206,7 +221,7 @@ class GitLabTemplateManager:
result[key] = value.strip("\"'")
return result
def render_template(self, template_id: str, variables: dict[str, Any] | None = None) -> str:
def render_template(self, template_id: str, variables: Mapping[str, object] | None = None) -> str:
if template_id not in self.prompts:
raise ValueError(f"Template '{template_id}' not found")
template: Final = self.prompts[template_id]
@ -313,7 +328,7 @@ class GitLabPromptManager(CustomPromptManagement):
def get_prompt_template(
self,
prompt_id: str,
prompt_variables: dict[str, Any] | None = None,
prompt_variables: Mapping[str, object] | None = None,
*,
ref: str | None = None,
) -> tuple[str, dict[str, Any]]:
@ -338,13 +353,13 @@ class GitLabPromptManager(CustomPromptManagement):
self,
user_id: str | None,
messages: list[AllMessageValues],
function_call: dict[str, Any] | str | None = None,
litellm_params: dict[str, Any] | None = None,
function_call: Mapping[str, object] | str | None = None,
litellm_params: dict[str, object] | None = None,
prompt_id: str | None = None,
prompt_variables: dict[str, Any] | None = None,
prompt_variables: Mapping[str, object] | None = None,
prompt_version: str | None = None,
**kwargs,
) -> tuple[list[AllMessageValues], dict[str, Any] | None]:
) -> tuple[list[AllMessageValues], dict[str, object] | None]:
if not prompt_id:
return messages, litellm_params
try:
@ -377,9 +392,9 @@ class GitLabPromptManager(CustomPromptManagement):
return final_messages, litellm_params
except Exception as e:
import litellm
from litellm._logging import verbose_proxy_logger
litellm._logging.verbose_proxy_logger.error("Error in GitLab prompt pre_call_hook: %s", e)
verbose_proxy_logger.error("Error in GitLab prompt pre_call_hook: %s", e)
return messages, litellm_params
def _parse_prompt_to_messages(self, prompt_content: str) -> list[AllMessageValues]:
@ -435,14 +450,14 @@ class GitLabPromptManager(CustomPromptManagement):
def post_call_hook(
self,
user_id: str | None,
response: Any,
response: _ResponseT,
input_messages: list[AllMessageValues],
function_call: dict[str, Any] | str | None = None,
litellm_params: dict[str, Any] | None = None,
function_call: Mapping[str, object] | str | None = None,
litellm_params: Mapping[str, object] | None = None,
prompt_id: str | None = None,
prompt_variables: dict[str, Any] | None = None,
prompt_variables: Mapping[str, object] | None = None,
**kwargs,
) -> Any:
) -> _ResponseT:
return response
def get_available_prompts(self) -> list[str]:
@ -498,7 +513,7 @@ class GitLabPromptManager(CustomPromptManagement):
messages: Final = self._parse_prompt_to_messages(rendered_prompt)
template_model: Final = prompt_metadata.get("model")
optional_params: Final[dict[str, Any]] = {}
optional_params: Final[dict[str, object]] = {}
for param in [
"temperature",
"max_tokens",
@ -658,14 +673,14 @@ class GitLabPromptCache:
self.template_manager: GitLabTemplateManager = self.prompt_manager.prompt_manager
# In-memory stores
self._by_file: dict[str, dict[str, Any]] = {}
self._by_id: dict[str, dict[str, Any]] = {}
self._by_file: dict[str, GitLabCachedPrompt] = {}
self._by_id: dict[str, GitLabCachedPrompt] = {}
# -------------------------
# Public API
# -------------------------
def load_all(self, *, recursive: bool = True) -> dict[str, dict[str, Any]]:
def load_all(self, *, recursive: bool = True) -> dict[str, GitLabCachedPrompt]:
"""
Scan GitLab for all .prompt files under prompts_path, load and parse each,
and return the mapping of repo file path -> JSON-like dict.
@ -695,7 +710,7 @@ class GitLabPromptCache:
return self._by_id
def reload(self, *, recursive: bool = True) -> dict[str, dict[str, Any]]:
def reload(self, *, recursive: bool = True) -> dict[str, GitLabCachedPrompt]:
"""Clear the cache and re-load from GitLab."""
self._by_file.clear()
self._by_id.clear()
@ -709,11 +724,11 @@ class GitLabPromptCache:
"""Return the template IDs (relative to prompts_path, without extension) currently cached."""
return list(self._by_id.keys())
def get_by_file(self, file_path: str) -> dict[str, Any] | None:
def get_by_file(self, file_path: str) -> GitLabCachedPrompt | None:
"""Get a cached prompt JSON by repo file path."""
return self._by_file.get(file_path)
def get_by_id(self, prompt_id: str) -> dict[str, Any] | None:
def get_by_id(self, prompt_id: str) -> GitLabCachedPrompt | None:
"""Get a cached prompt JSON by prompt ID (relative to prompts_path)."""
if prompt_id in self._by_id:
return self._by_id[prompt_id]
@ -728,7 +743,7 @@ class GitLabPromptCache:
# Internals
# -------------------------
def _template_to_json(self, prompt_id: str, tmpl: GitLabPromptTemplate) -> dict[str, Any]:
def _template_to_json(self, prompt_id: str, tmpl: GitLabPromptTemplate) -> GitLabCachedPrompt:
"""
Normalize a GitLabPromptTemplate into a JSON-like dict that is easy to serialize.
"""

2
litellm/integrations/otel/mappers/genai.py
View file

@ -62,6 +62,8 @@ class GenAIMapper:
GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens,
GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens,
Error.TYPE: lambda d: d.error.error_type if d.error else None,
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,

22
litellm/integrations/otel/model/payloads.py
View file

@ -95,6 +95,22 @@ class LLMUsage:
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
cache_creation_input_tokens: int | None = None
cache_read_input_tokens: int | None = None
@classmethod
def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage:
# Cache token counts only exist on the raw provider usage object under metadata
metadata: Final[Mapping[str, object]] = payload.get("metadata") or {}
raw_usage: Final = metadata.get("usage_object")
usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {}
return cls(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")),
cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")),
)
@dataclass(frozen=True)
@ -363,11 +379,7 @@ class LLMCallSpanData:
response_model=context.response_model,
response_id=as_str(response.get("id")),
request_params=LLMRequestParams.from_model_parameters(params),
usage=LLMUsage(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
),
usage=LLMUsage.from_standard_logging_payload(payload),
finish_reasons=finish_reasons,
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),

2
litellm/integrations/otel/model/semconv.py
View file

@ -110,6 +110,8 @@ class GenAI:
# usage
USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens"
USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens"
# content (opt-in, gated by capture mode)
INPUT_MESSAGES: Final = "gen_ai.input.messages"
OUTPUT_MESSAGES: Final = "gen_ai.output.messages"

62
litellm/integrations/posthog.py
View file

@ -12,7 +12,10 @@ For batching specific details see CustomBatchLogger class
import asyncio
import atexit
import os
from typing import Any, Final
from collections.abc import Mapping, Sequence
from typing import Final
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -34,6 +37,21 @@ from litellm.types.integrations.posthog import (
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
class PostHogBatchPayload(TypedDict):
api_key: ReadOnly[str]
batch: ReadOnly[Sequence[PostHogEventPayload]]
class PostHogLiteLLMParams(TypedDict, total=False):
metadata: ReadOnly[Mapping[str, object]]
class PostHogLogKwargs(TypedDict, total=False):
standard_logging_object: ReadOnly[StandardLoggingPayload]
standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams]
litellm_params: ReadOnly[PostHogLiteLLMParams]
class PostHogLogger(CustomBatchLogger):
def __init__(self, **kwargs):
"""
@ -137,7 +155,7 @@ class PostHogLogger(CustomBatchLogger):
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
def create_posthog_event_payload(self, kwargs: dict[str, Any]) -> PostHogEventPayload:
def create_posthog_event_payload(self, kwargs: PostHogLogKwargs) -> PostHogEventPayload:
"""
Helper function to create a PostHog event payload for logging
@ -171,11 +189,11 @@ class PostHogLogger(CustomBatchLogger):
def _create_posthog_properties(
self,
standard_logging_object: StandardLoggingPayload,
kwargs: dict[str, Any],
kwargs: PostHogLogKwargs,
event_name: str,
) -> dict[str, Any]:
) -> dict[str, object]:
"""Create PostHog properties following LLM Analytics spec"""
properties: Final = {}
properties: Final[dict[str, object]] = {}
# Core model information
properties["$ai_model"] = self._safe_get(standard_logging_object, "model", "")
@ -211,16 +229,19 @@ class PostHogLogger(CustomBatchLogger):
properties["$ai_error"] = error_str
# Add trace properties
self._add_trace_properties(properties, kwargs)
self._add_trace_properties(properties, standard_logging_object, kwargs)
# Add custom metadata fields
self._add_custom_metadata_properties(properties, kwargs)
return properties
def _add_trace_properties(self, properties: dict[str, Any], kwargs: dict[str, Any]):
standard_logging_object: Final = self._safe_get(kwargs, "standard_logging_object", {})
def _add_trace_properties(
self,
properties: dict[str, object],
standard_logging_object: StandardLoggingPayload,
kwargs: PostHogLogKwargs,
) -> None:
trace_id: Final = self._safe_get(standard_logging_object, "trace_id", self._safe_uuid())
properties["$ai_trace_id"] = trace_id
@ -232,7 +253,7 @@ class PostHogLogger(CustomBatchLogger):
if parent_id:
properties["$ai_parent_id"] = parent_id
def _add_custom_metadata_properties(self, properties: dict[str, Any], kwargs: dict[str, Any]):
def _add_custom_metadata_properties(self, properties: dict[str, object], kwargs: PostHogLogKwargs) -> None:
"""Add custom metadata fields to PostHog properties"""
metadata: Final = self._extract_metadata(kwargs)
if not isinstance(metadata, dict):
@ -277,7 +298,7 @@ class PostHogLogger(CustomBatchLogger):
if key not in litellm_internal_fields:
properties[key] = value
def _get_distinct_id(self, standard_logging_object: StandardLoggingPayload, kwargs: dict[str, Any]) -> str:
def _get_distinct_id(self, standard_logging_object: StandardLoggingPayload, kwargs: PostHogLogKwargs) -> str:
metadata: Final = self._extract_metadata(kwargs)
user_id: Final = self._safe_get(metadata, "user_id")
if user_id:
@ -291,7 +312,7 @@ class PostHogLogger(CustomBatchLogger):
return self._safe_uuid()
def _get_credentials_for_request(self, kwargs: dict[str, Any]) -> tuple[str | None, str | None]:
def _get_credentials_for_request(self, kwargs: PostHogLogKwargs) -> tuple[str | None, str | None]:
"""
Get PostHog credentials for this request.
@ -334,7 +355,7 @@ class PostHogLogger(CustomBatchLogger):
verbose_logger.debug("[POSTHOG MOCK] Mock mode enabled - API calls will be intercepted")
# Group events by credentials for batch sending
batches_by_credentials: Final[dict[tuple[str, str], list]] = {}
batches_by_credentials: Final[dict[tuple[str, str], list[PostHogEventPayload]]] = {}
for item in self.log_queue:
key = (item["api_key"], item["api_url"])
if key not in batches_by_credentials:
@ -380,18 +401,19 @@ class PostHogLogger(CustomBatchLogger):
verbose_logger.error("PostHog: Failed to initialize async components: %s", e)
raise
def _extract_metadata(self, kwargs: dict[str, Any]) -> dict[str, Any]:
litellm_params: Final = kwargs.get("litellm_params", {}) or {}
return litellm_params.get("metadata", {}) or {}
def _extract_metadata(self, kwargs: PostHogLogKwargs) -> Mapping[str, object]:
litellm_params: Final[PostHogLiteLLMParams] = kwargs.get("litellm_params", {}) or {}
metadata: Final[Mapping[str, object]] = litellm_params.get("metadata", {}) or {}
return metadata
def _safe_uuid(self) -> str:
return str(uuid.uuid4())
def _create_posthog_payload(self, events: list, api_key: str) -> dict[str, Any]:
def _create_posthog_payload(self, events: Sequence[PostHogEventPayload], api_key: str) -> PostHogBatchPayload:
return {"api_key": api_key, "batch": events}
def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
if obj is None or not hasattr(obj, "get"):
def _safe_get(self, obj: Mapping[str, object] | None, key: str, default: object = None) -> object:
if not isinstance(obj, Mapping):
return default
return obj.get(key, default)
@ -412,7 +434,7 @@ class PostHogLogger(CustomBatchLogger):
try:
# Group events by credentials (same logic as async_send_batch)
batches_by_credentials: Final[dict[tuple[str, str], list]] = {}
batches_by_credentials: Final[dict[tuple[str, str], list[PostHogEventPayload]]] = {}
for item in self.log_queue:
key = (item["api_key"], item["api_url"])
if key not in batches_by_credentials:

193
litellm/integrations/shadow_eval_logger.py
View file

@ -1,8 +1,11 @@
"""Shadow Eval Logger: samples a shadowed key's successful LLM requests (chat completions,
Anthropic Messages, and Responses API surfaces, each normalized to chat shape), duplicates
each against the job's other arm in a detached task (the auto-router for a forward job, the
fixed baseline model for a reverse one), blind-judges real vs shadow, and appends one
``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write.
each through every shadow arm in one detached task (each candidate auto-router for a
forward job, the fixed baseline model for a reverse one), blind-judges real vs each arm,
and appends one ``LiteLLM_ShadowEvalAttempt`` row per arm (verdict or error) as the
feature's only hot-path write. A multi-router job's arms therefore score the identical
sampled requests against the identical real responses, which is what makes their win
rates comparable head-to-head.
Counts, status, and spend derive from those rows at read time, so nothing can disagree
across pods or stop races; the hook reads active jobs through a short-TTL cache."""
@ -498,12 +501,16 @@ def _decision_classifier_cost(metadata: Mapping[str, object]) -> float:
return float(raw) if isinstance(raw, (int, float)) else 0.0
def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
"""Whether the router under evaluation served this request, which is what decides
the direction it belongs to. A forward job skips its own router's traffic, since
duplicating it would compare the router to itself: guaranteed ties, judge spend for
zero information. A reverse job samples exactly that traffic and nothing else."""
return _routing_decision(request_metadata).get("router_model_name") == router_name
def _direction_admits(request_metadata: Mapping[str, object], job: "ActiveShadowEvalJob") -> bool:
"""Whether this request belongs to the job's direction. A forward job skips traffic
any of its candidate routers served: duplicating a router's own request compares it
to itself (guaranteed ties), and judging a sibling against another candidate's live
response would score candidates against each other instead of against the incumbent.
A reverse job samples exactly its one router's traffic and nothing else."""
routed_by: Final = _routing_decision(request_metadata).get("router_model_name")
if job.direction == "reverse":
return routed_by == job.router_name
return routed_by not in job.arm_router_names
@dataclass(frozen=True, slots=True)
@ -546,6 +553,7 @@ class ActiveShadowEvalJob(BaseModel):
id: str
router_name: str
router_names: tuple[str, ...] = ()
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
shadow_percentage: float
@ -567,12 +575,25 @@ class ActiveShadowEvalJob(BaseModel):
raise ValueError("baseline_model is set for exactly the reverse jobs")
return self
@model_validator(mode="after")
def _reverse_evaluates_one_router(self) -> "ActiveShadowEvalJob":
"""A reverse row naming several routers is unsamplable (there is no one traffic
slice they share) and fails closed."""
if self.direction == "reverse" and len(self.arm_router_names) > 1:
raise ValueError("a reverse job evaluates exactly one router")
return self
@property
def shadow_target(self) -> str:
"""The model the duplicated arm calls: the router itself for a forward job, the
fixed baseline for a reverse one. Total because the validator above pins
def arm_router_names(self) -> tuple[str, ...]:
"""The job's full router set; rows from before router_names existed hold it in
router_name alone. The one place that reading lives on the sampling side."""
return self.router_names or (self.router_name,)
def arm_target(self, arm_router: str) -> str:
"""The model one duplicated arm calls: the candidate router itself for a forward
job, the fixed baseline for a reverse one. Total because the validator above pins
baseline_model to reverse jobs and only those."""
return self.baseline_model or self.router_name
return self.baseline_model or arm_router
def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
@ -592,7 +613,12 @@ _JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs"
class ShadowEvalLogger(CustomLogger):
"""Fires blind pairwise shadow evaluations for keys with an active shadow-eval job."""
"""Fires blind pairwise shadow evaluations for targets with an active shadow-eval job.
A job targets a virtual key, a team, or a user; a request qualifies for a job when
any of its resolved identities (key hash, team id, user id) matches the job's
target, so team and user jobs cover JWT-authenticated traffic, which carries no
key hash at all."""
def __init__(
self,
@ -617,10 +643,10 @@ class ShadowEvalLogger(CustomLogger):
# generation; the refill absorbs written rows and resets.
self._job_starts: dict[str, int] = {} # mutable-ok: per-generation counter
async def _active_jobs(self) -> Mapping[str, tuple[ActiveShadowEvalJob, ...]]:
"""Active jobs by api_key_id, cache-first. A key holds at most one job per
direction, so the value is a collection. A DB fault returns empty without
caching, so sampling pauses for that request and the next one retries."""
async def _active_jobs(self) -> Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]:
"""Active jobs by (target_type, target_id), cache-first. A target holds at most
one job per direction, so the value is a collection. A DB fault returns empty
without caching, so sampling pauses for that request and the next one retries."""
cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY)
if cached is not None:
return cached # pyright: ignore[reportReturnType] # cache stores exactly this mapping shape
@ -652,10 +678,10 @@ class ShadowEvalLogger(CustomLogger):
)
for row in grouped or []
}
by_key: Final = tuple(
by_target: Final = tuple(
sorted(
(
(str(record.api_key_id), job)
((str(record.target_type), str(record.target_id)), job)
for record in records or []
if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None
),
@ -663,7 +689,7 @@ class ShadowEvalLogger(CustomLogger):
)
)
jobs: Final = MappingProxyType(
{key: tuple(job for _, job in group) for key, group in groupby(by_key, key=itemgetter(0))}
{target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))}
)
await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs)
self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill
@ -691,7 +717,7 @@ class ShadowEvalLogger(CustomLogger):
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse")
or not _direction_admits(request_metadata, job)
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
@ -720,8 +746,18 @@ class ShadowEvalLogger(CustomLogger):
if should_redact_message_logging(dict(kwargs)): # mutable-ok: predicate takes a plain dict
return
metadata: Final = payload.get("metadata") or _EMPTY_METADATA
api_key_hash: Final = metadata.get("user_api_key_hash")
if not api_key_hash:
# Each identity the request resolved to is a candidate target; JWT-auth
# requests carry no key hash but do carry a team and user.
targets: Final = tuple(
(target_type, str(value))
for target_type, value in (
("key", metadata.get("user_api_key_hash")),
("team", metadata.get("user_api_key_team_id")),
("user", metadata.get("user_api_key_user_id")),
)
if value
)
if not targets:
return
request_id: Final = payload.get("id") or ""
if not request_id:
@ -731,8 +767,11 @@ class ShadowEvalLogger(CustomLogger):
return # only surfaces this table can normalize are comparable; unknown types fail closed
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
active_jobs: Final = await self._active_jobs()
eligible: Final = self._sampled_jobs(
(await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
tuple(job for target in targets for job in active_jobs.get(target, ())),
request_metadata,
request_id,
)
if not eligible:
return
@ -755,7 +794,10 @@ class ShadowEvalLogger(CustomLogger):
if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
self._record_funnel(job.id, "shed")
continue
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1
# One start writes one attempt row per arm, and max_turns is a row
# ceiling, so admission must pre-count every arm or a multi-router
# job overshoots the valve N-fold within a cache generation.
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + len(job.arm_router_names)
self._inflight_shadow_tasks += 1
asyncio.create_task(
self._run_shadow_eval(
@ -794,32 +836,74 @@ class ShadowEvalLogger(CustomLogger):
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""Budget gate -> shadow call -> blind judge -> one attempt row, and every exit
in exactly one coverage bucket: the gates that decline to spend on an admitted
sample (no DB to record into, an over-budget key, an unverifiable or exhausted
eval budget) count it withheld, so eligible traffic still reconciles as
not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits
above the dispatch so no provider spend happens without a place to record the
outcome, and the budget read lives here rather than in the success hook."""
"""Budget gates once per sampled request, then every router arm in turn: shadow
call -> blind judge -> one attempt row stamped with the arm. The gates that
decline to spend on an admitted sample (no DB to record into, an over-budget key,
an unverifiable or exhausted eval budget) count the REQUEST withheld before any
arm runs, so funnel counters stay per-request and a leg's eligible traffic still
reconciles as not_sampled + unjudgeable + shed + withheld + sampled requests,
where each sampled request writes one attempt row per arm. A budget crossed
mid-loop lets the remaining arms overshoot by one round, the same class of
overshoot as the samples already in flight when the cap is crossed. The prisma
gate sits above the dispatch so no provider spend happens without a place to
record the outcome, and the budget read lives here rather than in the success
hook."""
prisma: Final = self._prisma_provider()
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
for arm_router in job.arm_router_names:
await self._run_shadow_arm(
prisma=prisma,
job=job,
arm_router=arm_router,
request_id=request_id,
messages=messages,
real_text=real_text,
real_model=real_model,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=parent_metadata,
)
async def _run_shadow_arm(
self,
prisma: "PrismaClient",
job: ActiveShadowEvalJob,
arm_router: str,
request_id: str,
messages: Sequence[Mapping[str, object]],
real_text: str,
real_model: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
control_tier: str | None,
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""One arm's pipeline: shadow call -> blind judge -> one attempt row, every exit
recording this arm's outcome, so one arm's fault never silences a sibling arm."""
try:
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
shadow: Final = await self._call_router_shadow(
job.arm_target(arm_router), messages, shadow_params, parent_metadata
)
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
@ -827,6 +911,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=f"pipeline error: {e}",
real_cost=real_cost,
@ -840,6 +925,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=shadow.error,
shadow_cost=shadow.cost,
@ -864,6 +950,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=verdict.error,
shadow=shadow,
@ -880,6 +967,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome=verdict.preference,
shadow=shadow,
real_model=real_model,
@ -898,6 +986,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=f"pipeline error: {e}",
shadow=shadow,
@ -915,6 +1004,7 @@ class ShadowEvalLogger(CustomLogger):
request_id: str,
control_tier: str | None,
*,
router_name: str,
outcome: str,
real_cost: float,
real_classifier_cost: float,
@ -937,6 +1027,7 @@ class ShadowEvalLogger(CustomLogger):
data={ # mutable-ok: Prisma payload
"job_id": job.id,
"request_id": request_id,
"router_name": router_name,
"outcome": outcome,
"tier": control_tier if job.direction == "reverse" else (shadow.tier if shadow else None),
"real_model": real_model or None,
@ -1056,7 +1147,7 @@ class ShadowEvalLogger(CustomLogger):
)
_EMPTY_JOBS: Final[Mapping[str, tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
def _default_prisma_provider() -> "PrismaClient | None":

50
litellm/integrations/websearch_interception/handler.py
View file

@ -416,15 +416,25 @@ class WebSearchInterceptionLogger(CustomLogger):
if not tools:
return None
if call_type in (CallTypes.responses, CallTypes.aresponses):
return self._convert_responses_tools(kwargs=kwargs, tools=tools)
# Check if any tool is a web search tool (native or already LiteLLM standard)
has_websearch: Final = any(is_web_search_tool(t) for t in tools)
is_responses_call: Final = call_type in (CallTypes.responses, CallTypes.aresponses)
has_websearch: Final = (
any(is_web_search_tool_responses(tool) for tool in tools)
if is_responses_call
else any(is_web_search_tool(tool) for tool in tools)
)
if not has_websearch:
return None
if self.search_tool_name:
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
llm_router = None
self._select_search_tool_from_router(llm_router=llm_router)
if is_responses_call:
return self._convert_responses_tools(kwargs=kwargs, tools=tools)
verbose_logger.debug("WebSearchInterception: Converting native web_search tools to LiteLLM standard")
# If the client sent an Anthropic-native web_search_* tool, mark the
@ -1631,9 +1641,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return None
def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None":
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools: Final = list(getattr(llm_router, "search_tools") or [])
search_tools: Final = list(getattr(llm_router, "search_tools", []) or [])
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
def _select_search_tool_from_list(
@ -1643,20 +1651,26 @@ class WebSearchInterceptionLogger(CustomLogger):
) -> "_SearchToolConfig | None":
if self.search_tool_name:
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
if matching_tools:
search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
"WebSearchInterception: Found search tool '%s' from %s with provider '%s'",
self.search_tool_name,
source,
search_provider,
if not matching_tools:
raise ValueError(f"Configured search tool '{self.search_tool_name}' was not found")
selected_tool: Final = matching_tools[0]
litellm_params: Final = selected_tool.get("litellm_params")
selected_search_provider: Final = (
litellm_params.get("search_provider") if isinstance(litellm_params, Mapping) else None
)
if not isinstance(selected_search_provider, str) or not selected_search_provider.strip():
raise ValueError(
f"Configured search tool '{self.search_tool_name}' does not define a valid search provider"
)
return matching_tools[0]
verbose_logger.debug(
"WebSearchInterception: Search tool '%s' not found in %s, falling back to first available or perplexity",
"WebSearchInterception: Found search tool '%s' from %s with provider '%s'",
self.search_tool_name,
source,
selected_search_provider,
)
return selected_tool
if search_tools:
first_tool: Final = search_tools[0]

80
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -7,7 +7,13 @@ import os
from dataclasses import dataclass
from typing import Final
from litellm.types.files import get_file_mime_type_from_extension
from litellm.types.files import (
AUDIO_FILE_TYPES,
FILE_EXTENSIONS,
FILE_MIME_TYPES,
FileType,
get_file_mime_type_from_extension,
)
from litellm.types.utils import FileTypes
@ -323,3 +329,75 @@ def calculate_request_duration(file: FileTypes) -> float | None:
except Exception:
# Silently fail if duration extraction fails
return None
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"
def _speech_media_type_for_response_format(response_format: str) -> str | None:
file_type: Final = next(
(candidate for candidate, extensions in FILE_EXTENSIONS.items() if response_format.lower() in extensions),
None,
)
if file_type is None or file_type not in AUDIO_FILE_TYPES:
return None
return FILE_MIME_TYPES[file_type]
def resolve_speech_media_type(upstream_content_type: str | None, response_format: str | None) -> str:
upstream_media_type: Final = (upstream_content_type or "").split(";", 1)[0].strip().lower()
if upstream_media_type.startswith("audio/"):
return upstream_media_type
requested_media_type: Final = (
None if response_format is None else _speech_media_type_for_response_format(response_format)
)
return requested_media_type or DEFAULT_SPEECH_MEDIA_TYPE
_OGG_OPUS_HEAD_WINDOW: Final = 64
_ADTS_SYNC_AND_LAYER_MASK: Final = 0xF6
_ADTS_SYNC_AND_LAYER: Final = 0xF0
_ADTS_SAMPLE_RATE_INDEX_LIMIT: Final = 13
_MPEG_SYNC_MASK: Final = 0xE0
_MPEG_LAYER_MASK: Final = 0x06
_MPEG_RESERVED_VERSION: Final = 0x01
_MPEG_INVALID_BITRATE_INDEX: Final = 0x0F
_MPEG_RESERVED_SAMPLE_RATE_INDEX: Final = 0x03
def _adts_aac_frame_media_type(header: bytes) -> str | None:
sample_rate_index: Final = (header[2] >> 2) & 0x0F
return FILE_MIME_TYPES[FileType.AAC] if sample_rate_index < _ADTS_SAMPLE_RATE_INDEX_LIMIT else None
def _mpeg_audio_frame_media_type(header: bytes) -> str | None:
version: Final = (header[1] >> 3) & 0x03
layer: Final = header[1] & _MPEG_LAYER_MASK
bitrate_index: Final = header[2] >> 4
sample_rate_index: Final = (header[2] >> 2) & 0x03
if (
(header[1] & _MPEG_SYNC_MASK) != _MPEG_SYNC_MASK
or version == _MPEG_RESERVED_VERSION
or layer == 0
or bitrate_index == _MPEG_INVALID_BITRATE_INDEX
or sample_rate_index == _MPEG_RESERVED_SAMPLE_RATE_INDEX
):
return None
return FILE_MIME_TYPES[FileType.MP3]
def speech_media_type_from_audio_bytes(audio: bytes) -> str | None:
if audio[:4] == b"RIFF" and audio[8:12] == b"WAVE":
return FILE_MIME_TYPES[FileType.WAV]
if audio[:4] == b"fLaC":
return FILE_MIME_TYPES[FileType.FLAC]
if audio[:4] == b"OggS":
is_opus: Final = b"OpusHead" in audio[:_OGG_OPUS_HEAD_WINDOW]
return FILE_MIME_TYPES[FileType.OPUS if is_opus else FileType.OGG]
if audio[:3] == b"ID3":
return FILE_MIME_TYPES[FileType.MP3]
if len(audio) < 3 or audio[0] != 0xFF:
return None
if (audio[1] & _ADTS_SYNC_AND_LAYER_MASK) == _ADTS_SYNC_AND_LAYER:
return _adts_aac_frame_media_type(audio)
return _mpeg_audio_frame_media_type(audio)

3
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -50,6 +50,9 @@ OPTIONAL_KWARGS_KEYS: Final = (
"vertex_ai_project",
"vertex_ai_location",
"vertex_ai_credentials",
"gigachat_scope",
"gigachat_auth_url",
"gigachat_access_token",
"tpm",
"rpm",
"itpm",

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -369,6 +369,9 @@ def get_llm_provider(
elif endpoint == "https://api.meta.ai/v1":
custom_llm_provider = "meta"
dynamic_api_key = get_secret_str("META_API_KEY")
elif endpoint == "https://gigachat.devices.sberbank.ru/api/v1":
custom_llm_provider = "gigachat"
dynamic_api_key = get_secret_str("GIGACHAT_API_KEY")
elif (json_provider := JSONProviderRegistry.get_by_base_url(endpoint)) is not None:
custom_llm_provider = json_provider.slug
dynamic_api_key = api_key if api_key is not None else get_secret_str(json_provider.api_key_env)
@ -867,6 +870,9 @@ def _get_openai_compatible_provider_info(
# Manus is OpenAI compatible for responses API
api_base = api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im"
dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
elif custom_llm_provider == "gigachat":
api_base = api_base or get_secret_str("GIGACHAT_API_BASE") or "https://gigachat.devices.sberbank.ru/api/v1"
dynamic_api_key = api_key or get_secret_str("GIGACHAT_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(f"api base needs to be a string. api_base={api_base}")

8
litellm/litellm_core_utils/litellm_logging.py
View file

@ -2141,6 +2141,9 @@ class Logging(LiteLLMLoggingBaseClass):
logging_result: Final = self.normalize_logging_result(result=result)
if isinstance(result, Response) and isinstance(logging_result, (ModelResponse, EmbeddingResponse)):
result = logging_result
if standard_logging_object is None and result is not None and self.stream is not True:
if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance(
logging_result, (dict, list)
@ -6152,7 +6155,10 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
print(json.dumps(payload, indent=4), flush=True) # noqa: T201
try:
print(json.dumps(payload, indent=4, default=str), flush=True) # noqa: T201
except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging
verbose_logger.exception("Error serializing standard logging payload for debug output: %s", e)
def get_standard_logging_metadata(

44
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -3,7 +3,7 @@ Helper utilities for tracking the cost of built-in tools.
"""
from collections.abc import Mapping
from typing import Any, Final, Literal
from typing import Final, Literal
import litellm
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
@ -16,6 +16,7 @@ from litellm.types.llms.openai import (
WebSearchOptions,
)
from litellm.types.utils import (
ChatCompletionAnnotation,
Message,
ModelInfo,
ModelResponse,
@ -49,7 +50,7 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_cost_for_built_in_tools(
model: str,
response_object: Any,
response_object: object,
usage: Usage | None = None,
custom_llm_provider: str | None = None,
standard_built_in_tools_params: StandardBuiltInToolsParams | None = None,
@ -201,8 +202,7 @@ class StandardBuiltInToolCostTracking:
model_info: Final = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
file_search_raw: Final[Any] = standard_built_in_tools_params.get("file_search", {})
file_search_usage: Final[FileSearchTool | None] = FileSearchTool(**file_search_raw) if file_search_raw else None
file_search_usage: Final[FileSearchTool | None] = standard_built_in_tools_params.get("file_search") or None
# Convert model_info to dict and extract usage parameters
model_info_dict: Final = dict(model_info) if model_info is not None else None
@ -245,7 +245,7 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def _extract_file_search_params(
file_search_usage: Any,
file_search_usage: object,
) -> tuple[float | None, float | None]:
"""Extract and convert file search parameters safely."""
storage_gb = None
@ -335,7 +335,7 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def _extract_token_counts(
computer_use_usage: Any,
computer_use_usage: object,
) -> tuple[int | None, int | None]:
"""Extract and convert token counts safely."""
input_tokens = None
@ -351,9 +351,9 @@ class StandardBuiltInToolCostTracking:
return input_tokens, output_tokens
@staticmethod
def _safe_convert_to_int(value: Any) -> int | None:
def _safe_convert_to_int(value: object) -> int | None:
"""Safely convert a value to int."""
if value is not None:
if isinstance(value, (int, float, str)):
try:
return int(value)
except (TypeError, ValueError):
@ -381,7 +381,7 @@ class StandardBuiltInToolCostTracking:
return usage.model_copy(update={"server_tool_use": server_tool_use})
@staticmethod
def response_object_includes_web_search_call(response_object: Any, usage: Usage | None = None) -> bool:
def response_object_includes_web_search_call(response_object: object, usage: Usage | None = None) -> bool:
"""
Check if the response object includes a web search call.
@ -446,7 +446,7 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def response_object_includes_file_search_call(
response_object: Any,
response_object: object,
) -> bool:
"""
Check if the response object includes a file search call.
@ -477,11 +477,11 @@ class StandardBuiltInToolCostTracking:
message: Message | None = getattr(choice, "message", None)
if message is None:
continue
if annotations := getattr(message, "annotations", None):
if len(annotations) > 0:
for annotation in annotations:
if annotation.get("type", None) == annotation_type:
return True
annotations: list[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
if annotations:
for annotation in annotations:
if annotation.get("type", None) == annotation_type:
return True
return False
@staticmethod
@ -522,10 +522,8 @@ class StandardBuiltInToolCostTracking:
if model_info is None:
return 0.0
search_context_raw: Final[Any] = model_info.get("search_context_cost_per_query", {})
search_context_pricing: Final[SearchContextCostPerQuery] = (
SearchContextCostPerQuery(**search_context_raw) if search_context_raw else SearchContextCostPerQuery()
)
search_context_raw: Final = model_info.get("search_context_cost_per_query")
search_context_pricing: Final[SearchContextCostPerQuery] = search_context_raw or SearchContextCostPerQuery()
if web_search_options.get("search_context_size", None) == "low":
return search_context_pricing.get("search_context_size_low", 0.0)
elif web_search_options.get("search_context_size", None) == "medium":
@ -545,10 +543,8 @@ class StandardBuiltInToolCostTracking:
"""
if model_info is None:
return 0.0
search_context_raw: Final[Any] = model_info.get("search_context_cost_per_query", {}) or {}
search_context_pricing: Final[SearchContextCostPerQuery] = (
SearchContextCostPerQuery(**search_context_raw) if search_context_raw else SearchContextCostPerQuery()
)
search_context_raw: Final = model_info.get("search_context_cost_per_query")
search_context_pricing: Final[SearchContextCostPerQuery] = search_context_raw or SearchContextCostPerQuery()
return search_context_pricing.get("search_context_size_medium", 0.0)
@staticmethod
@ -714,7 +710,7 @@ class StandardBuiltInToolCostTracking:
response_object: ModelResponse,
) -> bool:
for _choice in response_object.choices:
message = getattr(_choice, "message", None)
message: Message | None = getattr(_choice, "message", None)
if (
message is not None
and hasattr(message, "annotations")

2
litellm/litellm_core_utils/model_response_utils.py
View file

@ -144,7 +144,6 @@ def _is_choice_non_empty(choice: StreamingChoices) -> bool:
# Check model_extra for dynamically added fields on the choice
choice_extra_fields: Final[Mapping[str, object]] = choice.model_extra or {}
for extra_field_name, extra_field_value in choice_extra_fields.items():
# Skip certain structural fields that are just default/None placeholders
if extra_field_name == "index" and extra_field_value == 0:
continue
if extra_field_name in {"finish_reason", "logprobs"} and extra_field_value is None:
@ -192,7 +191,6 @@ def _is_delta_non_empty(delta: Delta) -> bool:
# Check model_extra for dynamically added fields (this is where Pydantic stores them)
delta_extra_fields: Final[Mapping[str, object]] = delta.model_extra or {}
for extra_field_value in delta_extra_fields.values():
# Even structural fields are meaningful if they have actual content
if _has_meaningful_content(extra_field_value):
return True

55
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -205,6 +205,41 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
_IMAGE_SCAN_MAX_DEPTH: Final = 4
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
recursion; an Anthropic tool_result nests its image parts exactly one level down."""
frontier = parts # rebind-ok: depth-bounded frontier walk
for _ in range(_IMAGE_SCAN_MAX_DEPTH):
if any(isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_PART_TYPES for part in frontier):
return True
frontier = tuple( # rebind-ok: depth-bounded frontier walk
nested
for part in frontier
if isinstance(part, Mapping)
for content in (part.get("content"),)
if isinstance(content, list)
for nested in content
)
if not frontier:
return False
return False
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
"""Whether any message carries an image content part, across the dialects that reach
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
return any(
isinstance(content, list) and _content_parts_contain_image(content)
for message in messages
for content in (message.get("content"),)
)
def _audio_or_image_in_message_content(message: AllMessageValues) -> bool:
"""
Checks if message content contains an image or audio
@ -520,10 +555,10 @@ def update_messages_with_model_file_ids(
def update_responses_input_with_model_file_ids(
input: Any,
input: object,
model_id: str | None = None,
model_file_id_mapping: dict[str, dict[str, str]] | None = None,
) -> str | list[dict[str, Any]]:
) -> object:
"""
Updates responses API input with provider-specific file IDs.
File IDs are always inside the content array, not as direct input_file items.
@ -604,8 +639,8 @@ def update_responses_input_with_model_file_ids(
def _decode_vector_store_ids_in_tools(
tools: list[dict[str, Any]] | None,
) -> list[dict[str, Any]] | None:
tools: list[dict[str, object]] | None,
) -> list[dict[str, object]] | None:
"""
Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to
provider-native IDs. Non-unified IDs are passed through unchanged.
@ -657,10 +692,10 @@ def _decode_vector_store_ids_in_tools(
def update_responses_tools_with_model_file_ids(
tools: list[dict[str, Any]] | None,
tools: list[dict[str, object]] | None,
model_id: str | None = None,
model_file_id_mapping: dict[str, dict[str, str]] | None = None,
) -> list[dict[str, Any]] | None:
) -> list[dict[str, object]] | None:
"""
Updates responses API tools with provider-specific file IDs.
@ -853,7 +888,7 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
# ---------------------------------------------------------------------------
def _estimate_json_bytes(obj: Any) -> int:
def _estimate_json_bytes(obj: object) -> int:
"""Estimate the JSON-serialised byte size of ``obj`` without materialising
JSON. Walks iteratively (no recursion stack risk).
@ -1944,7 +1979,7 @@ def drop_tool_reference_parts_from_tool_messages(
return [_drop_tool_reference_parts(message) for message in messages] # mutable-ok: pipelines mutate message lists
def _attempt_json_repair(s: str) -> Any | None:
def _attempt_json_repair(s: str) -> object | None:
"""
Attempt to repair truncated JSON produced by LLM tool calls.
@ -2060,7 +2095,7 @@ def parse_tool_call_arguments(
raise ValueError(error_message) from original_error
def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
"""
Split a string that contains one or more concatenated JSON objects into
a list of parsed dicts.
@ -2096,7 +2131,7 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
return []
decoder: Final = json.JSONDecoder()
results: Final[list[dict[str, Any]]] = []
results: Final[list[dict[str, object]]] = []
idx = 0
length: Final = len(raw)

71
litellm/litellm_core_utils/token_counter.py
View file

@ -4,8 +4,9 @@ import base64
import io
import struct
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Any, Final, Literal, cast
from typing import Final, Literal, cast
import httpx
import tiktoken
import litellm
@ -171,6 +172,10 @@ def calculate_tiles_needed(
return total_tiles
def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
return struct.unpack(fmt, buffer)
def get_image_type(image_data: bytes) -> str | None:
"""take an image (really only the first ~100 bytes max are needed)
and return 'png' 'gif' 'jpeg' 'webp' 'heic' or None. method added to
@ -210,9 +215,9 @@ def get_image_dimensions(
if data.startswith(("http://", "https://")):
try:
client: Final = _get_httpx_client()
response: Final = safe_get(client, data)
response: Final[httpx.Response] = safe_get(client, data)
max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
content_length: Final = response.headers.get("Content-Length")
content_length: Final[str | None] = response.headers.get("Content-Length")
if content_length is not None and int(content_length) > max_bytes:
pass # skip download; img_data stays None
else:
@ -229,10 +234,10 @@ def get_image_dimensions(
img_type: Final = get_image_type(img_data)
if img_type == "png":
w, h = struct.unpack(">LL", img_data[16:24])
w, h = _unpack_ints(">LL", img_data[16:24])
return w, h
elif img_type == "gif":
w, h = struct.unpack("<HH", img_data[6:10])
w, h = _unpack_ints("<HH", img_data[6:10])
return w, h
elif img_type == "jpeg":
with io.BytesIO(img_data) as fhandle:
@ -245,25 +250,25 @@ def get_image_dimensions(
while ord(byte) == 0xFF:
byte = fhandle.read(1)
ftype = ord(byte)
size = struct.unpack(">H", fhandle.read(2))[0] - 2
size = _unpack_ints(">H", fhandle.read(2))[0] - 2
fhandle.seek(1, 1)
h, w = struct.unpack(">HH", fhandle.read(4))
h, w = _unpack_ints(">HH", fhandle.read(4))
return w, h
elif img_type == "webp":
# For WebP, the dimensions are stored at different offsets depending on the format
# Check for VP8X (extended format)
if img_data[12:16] == b"VP8X":
w = struct.unpack("<I", img_data[24:27] + b"\x00")[0] + 1
h = struct.unpack("<I", img_data[27:30] + b"\x00")[0] + 1
w = _unpack_ints("<I", img_data[24:27] + b"\x00")[0] + 1
h = _unpack_ints("<I", img_data[27:30] + b"\x00")[0] + 1
return w, h
# Check for VP8 (lossy format)
elif img_data[12:16] == b"VP8 ":
w = struct.unpack("<H", img_data[26:28])[0] & 0x3FFF
h = struct.unpack("<H", img_data[28:30])[0] & 0x3FFF
w = _unpack_ints("<H", img_data[26:28])[0] & 0x3FFF
h = _unpack_ints("<H", img_data[28:30])[0] & 0x3FFF
return w, h
# Check for VP8L (lossless format)
elif img_data[12:16] == b"VP8L":
bits: Final = struct.unpack("<I", img_data[21:25])[0]
bits: Final = _unpack_ints("<I", img_data[21:25])[0]
w = (bits & 0x3FFF) + 1
h = ((bits >> 14) & 0x3FFF) + 1
return w, h
@ -420,8 +425,8 @@ def token_counter(
def _count_function_call_tokens(
key: str,
value: Any,
message: Mapping[str, Any],
value: object,
message: Mapping[str, object],
count_function: TokenCounterFunction,
) -> int:
"""
@ -587,7 +592,7 @@ def _fix_model_name(model: str) -> str:
def _count_image_tokens(
image_url: Any,
image_url: object,
use_default_image_token_count: bool,
) -> int:
"""
@ -627,7 +632,7 @@ def _count_image_tokens(
raise ValueError(f"Invalid image_url type: {type(image_url).__name__}. Expected str or dict with 'url' field.")
def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
def _validate_anthropic_content(content: Mapping[str, object]) -> type:
"""
Validate and determine which Anthropic TypedDict applies.
@ -642,7 +647,7 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
"tool_result": AnthropicMessagesToolResultParam,
}
expected_cls: Final = mapping.get(content_type)
expected_cls: Final = mapping.get(content_type) if isinstance(content_type, str) else None
if expected_cls is None:
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
@ -693,8 +698,28 @@ def _count_document_tokens(
)
def _count_file_tokens(
file_value: object,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
) -> int:
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
if not isinstance(file_value, Mapping):
return 0
filename: Final = file_value.get("filename")
file_data: Final = file_value.get("file_data")
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
if not isinstance(file_data, str) or not file_data:
return name_tokens
return name_tokens + calculate_img_tokens(
data=file_data,
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
def _count_anthropic_content(
content: Mapping[str, Any],
content: Mapping[str, object],
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
@ -709,7 +734,7 @@ def _count_anthropic_content(
avoiding hardcoded field names.
"""
typeddict_cls: Final = _validate_anthropic_content(content)
type_hints: Final = getattr(typeddict_cls, "__annotations__", {})
type_hints: Final[Mapping[str, object]] = getattr(typeddict_cls, "__annotations__", {})
tokens = 0
# Fields to skip (metadata/identifiers that don't contribute to prompt tokens)
@ -778,6 +803,12 @@ def _count_content_list(
use_default_image_token_count,
default_token_count,
)
elif c["type"] == "file":
num_tokens += _count_file_tokens(
c.get("file"),
count_function,
use_default_image_token_count,
)
elif c["type"] in ("tool_use", "tool_result"):
num_tokens += _count_anthropic_content(
c,
@ -807,7 +838,7 @@ def _count_content_list(
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)."
)
return num_tokens
except Exception as e:

50
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -100,16 +100,6 @@ InputWriteBackTarget = (
)
class _SSEDelta(TypedDict, total=False):
type: ReadOnly[str]
text: ReadOnly[str]
stop_reason: ReadOnly[str | None]
class _SSEEventData(TypedDict, total=False):
delta: ReadOnly[_SSEDelta]
def _as_str_mapping(value: Mapping[str, object]) -> Mapping[str, object]:
return value
@ -157,6 +147,16 @@ class ExtractedInput:
EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=())
class _AnthropicSSEDelta(TypedDict, total=False):
type: ReadOnly[str]
text: ReadOnly[str]
stop_reason: ReadOnly[str | None]
class _AnthropicSSEEvent(TypedDict, total=False):
delta: ReadOnly[_AnthropicSSEDelta]
class AnthropicMessagesHandler(BaseTranslation):
"""Process Anthropic messages with guardrails.
@ -859,12 +859,28 @@ class AnthropicMessagesHandler(BaseTranslation):
@staticmethod
def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
"""Normalize an Anthropic image block into strings a guardrail can read.
base64 becomes a data URI so the format travels with the payload, which is what
the OpenAI path already puts in this field. A file source yields nothing: those
bytes live behind the Files API and this extractor has no client to fetch them.
"""
source: Final = block.get("source")
if not isinstance(source, Mapping):
return ()
# Could be base64 or url
source_type: Final = source.get("type")
if source_type == "url":
url: Final = source.get("url")
return (url,) if isinstance(url, str) and url else ()
data: Final = source.get("data")
return (data,) if data else ()
if not isinstance(data, str) or not data:
return ()
media_type: Final = source.get("media_type")
if isinstance(media_type, str) and media_type:
return (f"data:{media_type};base64,{data}",)
return (data,)
async def _apply_guardrail_responses_to_input(
self,
@ -1231,8 +1247,8 @@ class AnthropicMessagesHandler(BaseTranslation):
# Only process content_block_delta events
if event_type == "content_block_delta" and data_line:
try:
data: _SSEEventData = json.loads(data_line)
delta = data.get("delta", {})
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
if delta.get("type") == "text_delta":
text += delta.get("text", "")
except json.JSONDecodeError:
@ -1294,9 +1310,9 @@ class AnthropicMessagesHandler(BaseTranslation):
# Check for message_delta event with stop_reason
if event_type == "message_delta" and data_line:
try:
data: _SSEEventData = json.loads(data_line)
delta = data.get("delta", {})
stop_reason = delta.get("stop_reason")
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
stop_reason: str | None = delta.get("stop_reason")
if stop_reason is not None:
return True
except json.JSONDecodeError:

34
litellm/llms/anthropic/chat/handler.py
View file

@ -66,6 +66,10 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.chat.transformation import BaseConfig
def _loads_stream_chunk(payload: str) -> dict[str, object]:
return json.loads(payload)
async def make_call(
client: AsyncHTTPHandler | None,
api_base: str,
@ -78,7 +82,7 @@ async def make_call(
json_mode: bool,
speed: str | None = None,
tool_name_reverse_map: dict[str, str] | None = None,
) -> tuple[Any, httpx.Headers]:
) -> tuple["ModelResponseIterator", httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@ -93,7 +97,7 @@ async def make_call(
)
except httpx.HTTPStatusError as e:
error_headers = getattr(e, "headers", None)
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AnthropicError(
@ -138,7 +142,7 @@ def make_sync_call(
json_mode: bool,
speed: str | None = None,
tool_name_reverse_map: dict[str, str] | None = None,
) -> tuple[Any, httpx.Headers]:
) -> tuple["ModelResponseIterator", httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@ -153,7 +157,7 @@ def make_sync_call(
)
except httpx.HTTPStatusError as e:
error_headers = getattr(e, "headers", None)
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AnthropicError(
@ -292,7 +296,7 @@ class AnthropicChatCompletion(BaseLLM):
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
error_text = getattr(e, "text", str(e))
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
if error_response and hasattr(error_response, "text"):
@ -593,7 +597,7 @@ class AnthropicChatCompletion(BaseLLM):
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
error_text = getattr(e, "text", str(e))
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
if error_response and hasattr(error_response, "text"):
@ -664,10 +668,10 @@ class ModelResponseIterator:
# Accumulate web_search_tool_result blocks for multi-turn reconstruction
# See: https://github.com/BerriAI/litellm/issues/17737
self.web_search_results: list[dict[str, Any]] = []
self.web_search_results: list[dict[str, object]] = []
# Accumulate compaction blocks for multi-turn reconstruction
self.compaction_blocks: list[dict[str, Any]] = []
self.compaction_blocks: list[dict[str, object]] = []
# Accumulate streamed thinking text so final usage can split reasoning
# tokens from regular output tokens.
@ -727,7 +731,7 @@ class ModelResponseIterator:
str,
ChatCompletionToolCallChunk | None,
list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
dict[str, Any],
dict[str, object],
str | None,
]:
"""
@ -735,7 +739,7 @@ class ModelResponseIterator:
"""
text = ""
tool_use: ChatCompletionToolCallChunk | None = None
provider_specific_fields: Final = {}
provider_specific_fields: Final[dict[str, object]] = {}
reasoning_content: str | None = None
content_block: Final = ContentBlockDelta(**chunk)
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = []
@ -809,8 +813,8 @@ class ModelResponseIterator:
def _handle_redacted_thinking_content(
self,
content_block_start: ContentBlockStart,
provider_specific_fields: dict[str, Any],
) -> tuple[list[ChatCompletionRedactedThinkingBlock], dict[str, Any]]:
provider_specific_fields: dict[str, object],
) -> tuple[list[ChatCompletionRedactedThinkingBlock], dict[str, object]]:
"""
Handle the redacted thinking content
"""
@ -878,7 +882,7 @@ class ModelResponseIterator:
tool_use: ChatCompletionToolCallChunk | None = None
finish_reason = ""
usage: Usage | None = None
provider_specific_fields: dict[str, Any] = {}
provider_specific_fields: dict[str, object] = {}
reasoning_content: str | None = None
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
@ -1212,7 +1216,7 @@ class ModelResponseIterator:
# Try to parse as valid JSON first
try:
data_json: Final = json.loads(data_str)
data_json: Final = _loads_stream_chunk(data_str)
return self.chunk_parser(chunk=data_json)
except json.JSONDecodeError:
# Switch to accumulation mode and start accumulating
@ -1330,7 +1334,7 @@ class ModelResponseIterator:
str_line = str_line[index:]
if str_line.startswith("data:"):
data_json: Final = json.loads(str_line[5:])
data_json: Final = _loads_stream_chunk(str_line[5:])
return self.chunk_parser(chunk=data_json)
else:
return ModelResponseStream(id=self.response_id)

12
litellm/llms/anthropic/common_utils.py
View file

@ -865,13 +865,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
f"Failed to fetch models from Anthropic. Status code: {response.status_code}, Response: {response.text}"
)
models: Final = response.json()["data"]
models: Final[Sequence[Mapping[str, str]]] = response.json()["data"]
litellm_model_names: Final = []
for model in models:
stripped_model_name = model["id"]
litellm_model_name = "anthropic/" + stripped_model_name
litellm_model_names.append(litellm_model_name)
litellm_model_names: Final = ["anthropic/" + model["id"] for model in models]
return litellm_model_names
def get_token_counter(self) -> BaseTokenCounter | None:
@ -1077,7 +1073,7 @@ def strip_empty_content_blocks_from_anthropic_messages(
return out
def _is_empty_text_block(block: Any) -> bool:
def _is_empty_text_block(block: object) -> bool:
if not isinstance(block, dict) or block.get("type") != "text":
return False
text: Final = block.get("text")
@ -1131,7 +1127,7 @@ def normalize_anthropic_tool_use_id(raw_id: str) -> str:
return sanitized or "tool_use_id"
def _sanitize_tool_use_id_content_block(block: Any) -> Any:
def _sanitize_tool_use_id_content_block(block: object) -> object:
if not isinstance(block, dict):
return block
block_type: Final = block.get("type")

64
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -18,6 +18,24 @@ TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LE
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
def _optional_attr(source: object, name: str) -> object:
return getattr(source, name, None)
def _as_string_mapping(value: object) -> Mapping[str, object] | None:
if isinstance(value, Mapping):
return value
return None
def _thought_signature(provider_specific_fields: object) -> str | None:
fields: Final = _as_string_mapping(provider_specific_fields)
if fields is None:
return None
signature: Final = fields.get("thought_signature")
return signature if isinstance(signature, str) else None
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
{"name", "type", "input_schema", "description", "cache_control", "strict"}
)
@ -56,7 +74,7 @@ def truncate_tool_name(name: str) -> str:
def create_tool_name_mapping(
tools: list[dict[str, Any]],
tools: Sequence[Mapping[str, object]],
) -> dict[str, str]:
"""
Create a mapping of truncated tool names to original names.
@ -70,6 +88,8 @@ def create_tool_name_mapping(
mapping: Final[dict[str, str]] = {}
for tool in tools:
original_name = tool.get("name", "")
if not isinstance(original_name, str):
continue
truncated_name = truncate_tool_name(original_name)
if truncated_name != original_name:
mapping[truncated_name] = original_name
@ -286,44 +306,44 @@ class LiteLLMAnthropicMessagesAdapter:
### FOR [BETA] `/v1/messages` endpoint support
def _extract_signature_from_tool_call(self, tool_call: Any) -> str | None:
def _extract_signature_from_tool_call(self, tool_call: object) -> str | None:
"""
Extract signature from a tool call's provider_specific_fields.
Only checks provider_specific_fields, not thinking blocks.
"""
signature = None
fields: Final = _optional_attr(tool_call, "provider_specific_fields")
if fields:
return _thought_signature(fields)
if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields:
if "thought_signature" in tool_call.provider_specific_fields:
signature = tool_call.provider_specific_fields["thought_signature"]
elif hasattr(tool_call.function, "provider_specific_fields") and tool_call.function.provider_specific_fields:
if "thought_signature" in tool_call.function.provider_specific_fields:
signature = tool_call.function.provider_specific_fields["thought_signature"]
function_fields: Final = _optional_attr(_optional_attr(tool_call, "function"), "provider_specific_fields")
if function_fields:
return _thought_signature(function_fields)
return signature
return None
def _extract_signature_from_tool_use_content(self, content: dict[str, Any]) -> str | None:
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
"""
Extract signature from a tool_use content block's provider_specific_fields.
"""
provider_specific_fields: Final = content.get("provider_specific_fields", {})
provider_specific_fields: Final = _as_string_mapping(content.get("provider_specific_fields", {}))
if provider_specific_fields:
return provider_specific_fields.get("signature")
signature: Final = provider_specific_fields.get("signature")
return signature if isinstance(signature, str) else None
return None
def _add_cache_control_if_applicable(
self,
source: Any,
target: Any,
source: object,
target: object,
model: str | None,
) -> None:
"""
Extract cache_control from source and add to target if it should be preserved.
This method accepts Any type to support both regular dicts and TypedDict objects.
TypedDict objects (like ChatCompletionTextObject, ChatCompletionImageObject, etc.)
are dicts at runtime but have specific types at type-check time. Using Any allows
this method to work with both while maintaining runtime correctness.
This method accepts an unconstrained type to support both regular dicts and
TypedDict objects. TypedDict objects (like ChatCompletionTextObject,
ChatCompletionImageObject, etc.) are dicts at runtime but have specific types at
type-check time, so the widest parameter type works with both.
Args:
source: Dict or TypedDict containing potential cache_control field
@ -751,7 +771,7 @@ class LiteLLMAnthropicMessagesAdapter:
return new_tools, tool_name_mapping
def translate_anthropic_output_format_to_openai(self, output_format: Any) -> dict[str, object] | None:
def translate_anthropic_output_format_to_openai(self, output_format: object) -> dict[str, object] | None:
"""
Translate Anthropic's output_format to OpenAI's response_format.
@ -1366,7 +1386,7 @@ class LiteLLMAnthropicMessagesAdapter:
@classmethod
def _first_positive_prompt_tokens_detail_value(cls, usage: Usage, field_names: tuple[str, ...]) -> int:
prompt_tokens_details: Final = getattr(usage, "prompt_tokens_details", None)
prompt_tokens_details: Final = _optional_attr(usage, "prompt_tokens_details")
if prompt_tokens_details is None:
return 0
@ -1374,7 +1394,7 @@ class LiteLLMAnthropicMessagesAdapter:
if isinstance(prompt_tokens_details, dict):
value = cls._positive_int(prompt_tokens_details.get(field_name))
else:
value = cls._positive_int(getattr(prompt_tokens_details, field_name, None))
value = cls._positive_int(_optional_attr(prompt_tokens_details, field_name))
if value > 0:
return value
return 0

27
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -14,7 +14,7 @@ Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers:
import re
from collections.abc import Awaitable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeVar, Union, cast
from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeVar, Union, cast
from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
@ -232,7 +232,7 @@ async def _check_summary_model_access(
key_models: Final = list(getattr(user_api_key_auth, "models", None) or [])
team_id: Final[str | None] = getattr(user_api_key_auth, "team_id", None)
team_model_aliases: Final = getattr(user_api_key_auth, "team_model_aliases", None)
team_model_aliases: Final[dict[str, str] | None] = getattr(user_api_key_auth, "team_model_aliases", None)
team_models: Final = list(getattr(user_api_key_auth, "team_models", None) or [])
user_id: Final[str | None] = getattr(user_api_key_auth, "user_id", None)
project_id: Final[str | None] = getattr(user_api_key_auth, "project_id", None)
@ -443,7 +443,9 @@ async def _check_summary_model_budget(
)
return False
end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None)
end_user_model_max_budget: Final[dict[str, object] | None] = getattr(
user_api_key_auth, "end_user_model_max_budget", None
)
end_user_id: Final[str | None] = getattr(user_api_key_auth, "end_user_id", None)
if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None:
try:
@ -854,8 +856,8 @@ def _extract_summary_text(raw: str | None) -> str | None:
def _system_to_openai_message(
system: str | list[dict[str, Any]] | None,
) -> Mapping[str, object] | None:
system: str | list[dict[str, object]] | None,
) -> dict[str, object] | None:
"""Translate Anthropic-shaped ``system`` to an OpenAI system message.
Accepts a bare string or a list of Anthropic content blocks; returns
@ -866,10 +868,10 @@ def _system_to_openai_message(
if isinstance(system, str):
return {"role": "system", "content": system} if system else None
if isinstance(system, list):
parts: Final[tuple[str, ...]] = tuple(
parts: Final[list[object]] = [
block.get("text", "") for block in system if isinstance(block, dict) and block.get("type") == "text"
)
joined: Final = "\n\n".join(part for part in parts if part)
]
joined: Final = "\n\n".join(part for part in parts if isinstance(part, str) and part)
return {"role": "system", "content": joined} if joined else None
return None
@ -951,7 +953,7 @@ async def _call_summary_model(
summary_model: str,
summary_messages: Sequence[Mapping[str, object]],
metadata: Mapping[str, object],
llm_router: object,
llm_router: Optional["Router"],
allowed_model_region: str | None = None,
max_tokens: int = COMPACT_SUMMARY_MAX_TOKENS,
) -> Union["ModelResponse", "CustomStreamWrapper"]:
@ -1036,10 +1038,9 @@ def _extract_usage(response: object) -> tuple[int, int]:
usage: Final[object] = getattr(response, "usage", None)
if usage is None:
return 0, 0
return (
int(getattr(usage, "prompt_tokens", 0) or 0),
int(getattr(usage, "completion_tokens", 0) or 0),
)
prompt_tokens: Final[int | None] = getattr(usage, "prompt_tokens", 0)
completion_tokens: Final[int | None] = getattr(usage, "completion_tokens", 0)
return int(prompt_tokens or 0), int(completion_tokens or 0)
def apply_client_compaction_block_history(

260
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -8,6 +8,10 @@ import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm.constants import (
ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS,
ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE,
)
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
@ -21,6 +25,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()
_UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdlib strong-ref set for pump tasks
_DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains
INCOMPLETE_STREAM_ERROR_MESSAGE: Final = (
"Provider stream ended before emitting a message_stop event; "
"the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."
@ -133,6 +140,34 @@ def _is_terminal_stream_chunk(chunk: object) -> bool:
return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
def _try_claim_detached_drain_slot() -> bool:
"""Claim a detached-drain slot for the current task, bounding concurrency.
Returns True if a slot was claimed (the caller may keep draining upstream
for billing) or False if the cap is already reached (the caller should stop
and bill what it has). Only touched from the event loop, so the check +
insert need no lock.
"""
if len(_DETACHED_STREAM_DRAINS) >= ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS:
return False
current_task: Final = asyncio.current_task()
if current_task is not None:
_DETACHED_STREAM_DRAINS.add(current_task)
current_task.add_done_callback(_DETACHED_STREAM_DRAINS.discard)
return True
def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool:
"""After client detach the relay never reads the queue again, so drain it here.
The forwarded exception still sitting in the queue means the relay tore
down before re-raising it, so the proxy's failure handling never ran and
the caller must salvage spend itself.
"""
remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize()))
return any(item is exc for item in remaining)
def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
@ -414,17 +449,167 @@ class BaseAnthropicMessagesStreamingIterator:
async def async_sse_wrapper(
self,
completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | dict],
completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
) -> AsyncIterator[bytes]:
"""
Generic async SSE wrapper that converts streaming chunks to SSE format
and handles logging.
The upstream read runs in a detached background task (``_pump_upstream``)
so that a client disconnect tears down only this client-facing generator,
never the upstream drain + billing. The provider (e.g. Bedrock) keeps
generating and billing the full response regardless of the client, so
draining it to completion is what lets spend tracking see the real
terminal ``message_delta`` / ``message_stop`` usage instead of a
truncated placeholder count.
Chunks reach the client through a bounded queue. While the client is
connected the pump blocks on a full queue (racing the disconnect
signal), so a slow reader throttles the upstream read exactly as the old
direct ``yield`` did instead of letting the whole response buffer in
memory. Once the client goes away the pump stops enqueueing and only
keeps a single ``collected_chunks`` copy for billing, and the number of
such post-disconnect drains running at once is capped so client behavior
can't create unbounded worker state; over the cap the pump bills what it
has rather than draining further. Detached-drain lifetime is otherwise
bounded by the upstream stream/read timeout.
An upstream failure (Bedrock read / decode / chunk-conversion error)
that happens while the client is still connected is forwarded through
the queue and re-raised here, so the original provider exception (and
its status) reaches the proxy's failure handling unchanged rather than
being masked by a generic incomplete-stream event.
This method provides the common logic for both Anthropic and Bedrock implementations.
"""
collected_chunks: Final = []
saw_terminal_event = False
queue: Final[asyncio.Queue[bytes | None | BaseException]] = asyncio.Queue(
maxsize=ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE
)
client_detached: Final = asyncio.Event()
pump_task: Final = asyncio.create_task(self._pump_upstream_to_queue(completion_stream, queue, client_detached))
_UPSTREAM_PUMP_TASKS.add(pump_task)
pump_task.add_done_callback(_UPSTREAM_PUMP_TASKS.discard)
reached_end = False # rebind-ok: flipped once the relay consumes the end-of-stream sentinel
try:
while True:
item = await queue.get()
if item is None:
reached_end = True
break
if isinstance(item, BaseException):
raise item
yield item
finally:
client_detached.set()
if not reached_end:
self._dispatch_pending_deferred_logging()
def _dispatch_pending_deferred_logging(self) -> None:
"""Fire deferred billing that a torn-down response would otherwise drop.
When the pump finishes draining while the client is still connected it
stores the logging coroutine for ProxyLogging._fire_deferred_stream_logging,
which the proxy only fires on a normally completed response: a client
disconnect (GeneratorExit / CancelledError) re-raises past it. Without
this dispatch that window loses the spend row entirely.
"""
deferred_cb: Final = getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None)
deferred_args: Final = getattr(self.litellm_logging_obj, "_deferred_stream_complete_args", None)
if deferred_cb is None or deferred_args is None:
return
self.litellm_logging_obj._on_deferred_stream_complete = None
self.litellm_logging_obj._deferred_stream_complete_args = None
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=deferred_cb(*deferred_args))
async def _bill_collected_chunks(
self,
collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _handle_streaming_logging
*,
stream_teardown: bool,
) -> None:
from litellm._logging import verbose_proxy_logger
try:
await self._handle_streaming_logging(collected_chunks, stream_teardown=stream_teardown)
except Exception as exc: # noqa: BLE001 # billing is best-effort; never crash the pump
verbose_proxy_logger.warning(
"async_sse_wrapper billing failed after %d chunks: %s(%s)",
len(collected_chunks),
type(exc).__name__,
exc,
)
@staticmethod
async def _abort_upstream(
completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
) -> None:
"""Close the upstream provider stream so it stops generating and billing."""
from litellm._logging import verbose_proxy_logger
try:
await aclose_if_supported(completion_stream)
except Exception as exc: # noqa: BLE001 # abort is best-effort; log and continue
verbose_proxy_logger.warning(
"async_sse_wrapper failed to abort upstream stream: %s(%s)",
type(exc).__name__,
exc,
)
@staticmethod
async def _enqueue_for_client(
queue: "asyncio.Queue[bytes | None | BaseException]",
client_detached: "asyncio.Event",
item: bytes | None | BaseException,
) -> bool:
"""Deliver one item to the client, applying backpressure.
Returns True if the item was queued, False if the client disconnected
before there was room (the item is then dropped, since a gone client
can't receive it). Never blocks once the client has detached.
"""
if client_detached.is_set():
return False
try:
queue.put_nowait(item)
except asyncio.QueueFull:
pass
else:
return True
put_task: Final = asyncio.ensure_future(queue.put(item))
detached_task: Final = asyncio.ensure_future(client_detached.wait())
try:
await asyncio.wait(frozenset((put_task, detached_task)), return_when=asyncio.FIRST_COMPLETED)
finally:
if not detached_task.done():
detached_task.cancel()
if put_task.done() and not put_task.cancelled():
return True
put_task.cancel()
return False
async def _pump_upstream_to_queue(
self,
completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
queue: "asyncio.Queue[bytes | None | BaseException]",
client_detached: "asyncio.Event",
) -> None:
"""Drain the whole upstream into ``queue`` (backpressured) and bill once.
Runs detached so a client disconnect can't interrupt the upstream read;
see ``async_sse_wrapper`` for the full rationale. On a completed drain
the success billing (or deferred park) happens before the end-of-stream
sentinel is enqueued: the relay can only tear down after consuming the
sentinel, so its teardown can never outrun the park and get mistaken
for a client disconnect, and a sentinel the client never consumes falls
back to dispatching the parked billing here.
"""
from litellm._logging import verbose_proxy_logger
collected_chunks: Final[list[bytes]] = [] # mutable-ok: SSE billing buffer appended to across the drain
saw_terminal_event = False # rebind-ok: accumulates across the upstream loop
draining_detached = False # rebind-ok: set once this pump claims a detached-drain slot
try:
async for chunk in completion_stream:
if self.completion_start_time is None:
@ -432,17 +617,62 @@ class BaseAnthropicMessagesStreamingIterator:
saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk)
encoded_chunk = self._convert_chunk_to_sse_format(chunk)
collected_chunks.append(encoded_chunk)
yield encoded_chunk
except (GeneratorExit, asyncio.CancelledError):
# A client disconnect tears the generator down at the yield, so the
# post-loop logging below never runs and the tokens already streamed
# (and billed by the provider) would never reach spend tracking. See LIT-5839.
if collected_chunks:
await self._handle_streaming_logging(collected_chunks, stream_teardown=True)
raise
if not client_detached.is_set():
await self._enqueue_for_client(queue, client_detached, encoded_chunk)
continue
if not draining_detached:
if not _try_claim_detached_drain_slot():
verbose_proxy_logger.warning(
"async_sse_wrapper: detached-drain cap (%d) reached; billing %d partial "
"chunks and aborting the upstream stream to stop provider billing",
ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS,
len(collected_chunks),
)
await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
await self._abort_upstream(completion_stream)
return
draining_detached = True
except Exception as exc: # noqa: BLE001 # upstream errors are handled/forwarded by _handle_pump_upstream_error
await self._handle_pump_upstream_error(queue, client_detached, collected_chunks, exc)
return
if not saw_terminal_event:
yield _incomplete_stream_error_sse_event()
if client_detached.is_set():
await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
return
if not saw_terminal_event and not await self._enqueue_for_client(
queue, client_detached, _incomplete_stream_error_sse_event()
):
await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
return
await self._bill_collected_chunks(collected_chunks, stream_teardown=False)
if not await self._enqueue_for_client(queue, client_detached, None):
self._dispatch_pending_deferred_logging()
# Handle logging after all chunks are processed
await self._handle_streaming_logging(collected_chunks)
async def _handle_pump_upstream_error(
self,
queue: "asyncio.Queue[bytes | None | BaseException]",
client_detached: "asyncio.Event",
collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _bill_collected_chunks
exc: BaseException,
) -> None:
"""Forward a provider error to a still-connected client, else salvage partial spend.
Handing the original exception to the client-facing generator lets it
re-raise so the proxy's failure handling keeps the provider status and
owns logging (no success-bill). If the client already went away, or
disconnects before ever consuming the queued exception, no failure hook
runs, so bill the partial instead of dropping the request.
"""
from litellm._logging import verbose_proxy_logger
if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc):
await client_detached.wait()
if not _exception_left_unconsumed(queue, exc):
return
verbose_proxy_logger.warning(
"async_sse_wrapper upstream pump failed after client disconnect (%d chunks): %s(%s)",
len(collected_chunks),
type(exc).__name__,
exc,
)
await self._bill_collected_chunks(collected_chunks, stream_teardown=True)

42
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -179,14 +179,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
)
@staticmethod
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str:
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
"""Group a run of consecutive thinking blocks together; keep every other block alone."""
index, block = indexed_block
return "thinking" if block.get("type") == "thinking" else f"block:{index}"
@classmethod
def _assistant_group_to_input_item(
cls, group: tuple[Mapping[str, Any], ...]
cls, group: tuple[Mapping[str, object], ...]
) -> dict[str, Any] | None: # mutable-ok: API message payload
first: Final = group[0]
btype: Final = first.get("type")
@ -206,7 +206,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
def translate_messages_to_responses_input(
self,
messages: list[AllAnthropicPassThroughMessageValues],
) -> list[dict[str, Any]]:
) -> list[dict[str, object]]:
"""
Convert Anthropic messages list to Responses API `input` items.
@ -220,7 +220,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
assistant thinking -> reasoning
assistant tool_use -> function_call
"""
input_items: Final[list[dict[str, Any]]] = []
input_items: Final[list[dict[str, object]]] = []
for m in messages:
if m["role"] == "system":
@ -248,7 +248,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
}
)
elif isinstance(content, list):
user_parts: list[dict[str, Any]] = []
user_parts: list[Mapping[str, object]] = []
tool_image_parts: list[dict[str, Any]] = [] # mutable-ok: json content parts
for block in content:
if not isinstance(block, dict):
@ -379,9 +379,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
def translate_tools_to_responses_api(
self,
tools: list[AllAnthropicToolsValues],
) -> list[dict[str, Any]]:
) -> list[dict[str, object]]:
"""Convert Anthropic tool definitions to Responses API function tools."""
result: Final[list[dict[str, Any]]] = []
result: Final[list[dict[str, object]]] = []
for tool in tools:
tool_dict = cast(dict[str, Any], tool)
tool_type = tool_dict.get("type", "")
@ -392,7 +392,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
continue
# Responses turns strict mode on when `strict` is omitted, silently rewriting
# `required` to every property. Anthropic tools are non-strict unless asked.
func_tool: dict[str, Any] = {
func_tool: dict[str, object] = {
"type": "function",
"name": tool_name,
"strict": bool(tool_dict.get("strict")),
@ -407,7 +407,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
@staticmethod
def translate_tool_choice_to_responses_api(
tool_choice: AnthropicMessagesToolChoice,
) -> str | dict[str, Any]:
) -> str | dict[str, object]:
"""Convert Anthropic tool_choice to Responses API tool_choice."""
tc_type: Final = tool_choice.get("type")
if tc_type == "any":
@ -420,8 +420,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
@staticmethod
def translate_context_management_to_responses_api(
context_management: dict[str, Any],
) -> list[dict[str, Any]] | None:
context_management: dict[str, object],
) -> list[dict[str, object]] | None:
"""
Convert Anthropic context_management dict to OpenAI Responses API array format.
@ -435,13 +435,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if not isinstance(edits, list):
return None
result: Final[list[dict[str, Any]]] = []
result: Final[list[dict[str, object]]] = []
for edit in edits:
if not isinstance(edit, dict):
continue
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
entry: dict[str, Any] = {"type": "compaction"}
entry: dict[str, object] = {"type": "compaction"}
trigger = edit.get("trigger")
if isinstance(trigger, dict) and trigger.get("value") is not None:
entry["compact_threshold"] = int(trigger["value"])
@ -451,9 +451,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
@staticmethod
def translate_thinking_to_reasoning(
thinking: dict[str, Any],
output_config: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
thinking: dict[str, object],
output_config: dict[str, object] | None = None,
) -> dict[str, object] | None:
"""
Convert Anthropic thinking param to Responses API reasoning param.
@ -473,12 +473,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(output_config, dict) and output_config.get("effort"):
effort = output_config["effort"]
elif thinking_type == "enabled":
effort = reasoning_effort_from_thinking_budget(thinking.get("budget_tokens", 0))
raw_budget: Final = thinking.get("budget_tokens", 0)
budget_tokens: Final = int(raw_budget) if isinstance(raw_budget, (int, float)) else 0
effort = reasoning_effort_from_thinking_budget(budget_tokens)
else:
return None
auto_summary: Final = is_reasoning_auto_summary_enabled()
result: Final[dict[str, Any]] = {"effort": effort}
result: Final[dict[str, object]] = {"effort": effort}
summary: Final = thinking.get("summary")
if summary:
result["summary"] = summary
@ -570,7 +572,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
# output_format / output_config.format -> text format
# output_format: {"type": "json_schema", "schema": {...}}
# output_config: {"format": {"type": "json_schema", "schema": {...}}}
output_format: Any = anthropic_request.get("output_format")
output_format: object = anthropic_request.get("output_format")
output_config = anthropic_request.get("output_config")
if not isinstance(output_format, dict) and isinstance(output_config, dict):
output_format = output_config.get("format")
@ -620,7 +622,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
ResponseReasoningItem,
)
content: Final[list[dict[str, Any]]] = []
content: Final[list[dict[str, object]]] = []
stop_reason: AnthropicFinishReason = "end_turn"
for item in response.output:

59
litellm/llms/base_llm/managed_resources/base_managed_resource.py
View file

@ -5,7 +5,8 @@
import base64
import json
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar, cast
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Generic, Protocol, TypeVar, cast, runtime_checkable
from litellm import verbose_logger
from litellm.llms.base_llm.managed_resources.isolation import (
@ -38,6 +39,30 @@ else:
ResourceObjectType = TypeVar("ResourceObjectType")
@runtime_checkable
class _HasIdentifier(Protocol):
id: str
class _ManagedResourceRecord(Protocol[ResourceObjectType]):
unified_resource_id: str
resource_object: ResourceObjectType
def model_dump(self) -> dict[str, object]: ...
class _ManagedResourceTable(Protocol[ResourceObjectType]):
async def create(self, *, data: Mapping[str, object]) -> object: ...
async def find_first(self, *, where: Mapping[str, object]) -> _ManagedResourceRecord[ResourceObjectType] | None: ...
async def find_many(
self, *, where: Mapping[str, object], take: int, order: Mapping[str, str]
) -> list[_ManagedResourceRecord[ResourceObjectType]]: ...
async def delete(self, *, where: Mapping[str, object]) -> object: ...
class BaseManagedResource(ABC, Generic[ResourceObjectType]):
"""
Base class for managing resources with target_model_names support.
@ -64,6 +89,9 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
self.internal_usage_cache = internal_usage_cache
self.prisma_client = prisma_client
def _resource_table(self) -> _ManagedResourceTable[ResourceObjectType]:
return getattr(self.prisma_client.db, self.table_name)
# ============================================================================
# ABSTRACT METHODS
# ============================================================================
@ -137,7 +165,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
litellm_parent_otel_span: Span | None,
model_mappings: dict[str, str],
user_api_key_dict: UserAPIKeyAuth,
additional_db_fields: dict[str, Any] | None = None,
additional_db_fields: Mapping[str, object] | None = None,
) -> None:
"""
Store unified resource ID with model mappings in cache and database.
@ -153,7 +181,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
verbose_logger.info("Storing LiteLLM Managed %s with id=%s in cache", self.resource_type, unified_resource_id)
# Prepare cache data
cache_data: Final = {
cache_data: Final[dict[str, object]] = {
"unified_resource_id": unified_resource_id,
"resource_object": resource_object,
"model_mappings": model_mappings,
@ -176,7 +204,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
)
# Prepare database data
db_data: Final = {
db_data: Final[dict[str, object]] = {
"unified_resource_id": unified_resource_id,
"model_mappings": json.dumps(model_mappings),
"flat_model_resource_ids": list(model_mappings.values()),
@ -205,7 +233,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
db_data.update(additional_db_fields)
# Store in database
table: Final = getattr(self.prisma_client.db, self.table_name)
table: Final = self._resource_table()
result: Final = await table.create(data=db_data)
verbose_logger.debug(
@ -240,7 +268,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
return result
# Check database
table: Final = getattr(self.prisma_client.db, self.table_name)
table: Final = self._resource_table()
db_object: Final = await table.find_first(where={"unified_resource_id": unified_resource_id})
if db_object:
@ -264,7 +292,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
The deleted resource object or None if not found
"""
# Get old value from database
table: Final = getattr(self.prisma_client.db, self.table_name)
table: Final = self._resource_table()
initial_value: Final = await table.find_first(where={"unified_resource_id": unified_resource_id})
if initial_value is None:
@ -515,7 +543,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
user_api_key_dict: UserAPIKeyAuth,
limit: int | None = None,
after: str | None = None,
additional_filters: dict[str, Any] | None = None,
additional_filters: Mapping[str, object] | None = None,
) -> dict[str, Any]:
"""
List resources created by a user.
@ -533,7 +561,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
if owner_filter is None:
return build_list_page([])
where_clause: Final[dict[str, Any]] = {**owner_filter}
where_clause: Final[dict[str, object]] = {**owner_filter}
if after:
where_clause["id"] = {"gt": after}
@ -544,14 +572,14 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
# Fetch resources
fetch_limit: Final = limit or 20
table: Final = getattr(self.prisma_client.db, self.table_name)
table: Final = self._resource_table()
resources: Final = await table.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
resource_objects: Final[list[Any]] = []
resource_objects: Final[list[object]] = []
for resource in resources:
try:
# Stop once we have enough
@ -559,12 +587,13 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
break
# Parse resource object
resource_data = resource.resource_object
if isinstance(resource_data, str):
resource_data = json.loads(resource_data)
stored_resource = resource.resource_object
resource_data: object = (
json.loads(stored_resource) if isinstance(stored_resource, str) else stored_resource
)
# Set unified ID
if hasattr(resource_data, "id"):
if isinstance(resource_data, _HasIdentifier):
resource_data.id = resource.unified_resource_id
elif isinstance(resource_data, dict):
resource_data["id"] = resource.unified_resource_id

9
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1631,6 +1631,11 @@ class AmazonConverseConfig(BaseConfig):
bedrock_tool_config["toolChoice"] = tool_choice_values
self._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params)
config_block_entries: Final = tuple(
(config_name, config_class, inference_params.pop(config_name, None))
for config_name, config_class in self.get_config_blocks().items()
)
data: Final[CommonRequestObject] = {
"inferenceConfig": self._transform_inference_params(inference_params=inference_params),
}
@ -1641,9 +1646,7 @@ class AmazonConverseConfig(BaseConfig):
if system_content_blocks:
data["system"] = system_content_blocks
# Handle all config blocks
for config_name, config_class in self.get_config_blocks().items():
config_value = inference_params.pop(config_name, None)
for config_name, config_class, config_value in config_block_entries:
if config_value is not None:
data[config_name] = config_class(**config_value)

1
litellm/llms/bedrock/common_utils.py
View file

@ -1487,6 +1487,7 @@ class CommonBatchFilesUtils:
aws_role_name=optional_params.get("aws_role_name"),
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
aws_external_id=optional_params.get("aws_external_id"),
)
# Prepare the request data

1
litellm/llms/bedrock/files/handler.py
View file

@ -113,6 +113,7 @@ class BedrockFilesHandler(BaseAWSLLM):
aws_role_name=optional_params.get("aws_role_name"),
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
aws_external_id=optional_params.get("aws_external_id"),
)
# Create S3 client

3
litellm/llms/bedrock/files/transformation.py
View file

@ -146,6 +146,7 @@ class _BedrockS3RequestParams(BaseModel):
aws_role_name: str | None = None
aws_web_identity_token: str | None = None
aws_sts_endpoint: str | None = None
aws_external_id: str | None = None
s3_region_name: str | None = None
s3_endpoint_url: str | None = None
@ -1029,6 +1030,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
aws_role_name=optional_params.get("aws_role_name"),
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
aws_external_id=optional_params.get("aws_external_id"),
)
# Calculate SHA256 hash of the content (REQUIRED for S3)
@ -1290,6 +1292,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
aws_role_name=request_params.aws_role_name,
aws_web_identity_token=request_params.aws_web_identity_token,
aws_sts_endpoint=request_params.aws_sts_endpoint,
aws_external_id=request_params.aws_external_id,
)
empty_body_hash: Final = hashlib.sha256(b"").hexdigest()

70
litellm/llms/bedrock/realtime/handler.py
View file

@ -7,13 +7,18 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
import asyncio
import contextlib
import json
from collections.abc import AsyncIterator, Mapping
from typing import Final, Protocol
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import _redact_string, verbose_proxy_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM
@ -32,6 +37,17 @@ def _json_str(value: JsonValue) -> str | None:
return value if isinstance(value, str) else None
def _should_log_event(openai_message: Mapping[str, object]) -> bool:
logged_types: Final = (
litellm.logged_real_time_event_types
if litellm.logged_real_time_event_types is not None
else DefaultLoggedRealTimeEventTypes
)
if logged_types == "*":
return True
return openai_message.get("type") in logged_types
class RealtimeClientWebSocket(Protocol):
"""The client-facing websocket surface the realtime bridge talks to."""
@ -205,16 +221,22 @@ class BedrockRealtime(BaseAWSLLM):
)
)
bedrock_to_client_task: Final = asyncio.create_task(
self._forward_bedrock_to_client(
bedrock_stream,
websocket,
transformation_config,
model,
logging_obj,
session_state,
async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]:
return tuple(
[
event
async for event in self._forward_bedrock_to_client(
bedrock_stream,
websocket,
transformation_config,
model,
logging_obj,
session_state,
)
]
)
)
bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events())
# Wait for both tasks to complete
await asyncio.gather(
@ -223,6 +245,27 @@ class BedrockRealtime(BaseAWSLLM):
return_exceptions=True,
)
forwarded_logged_events: Final = (
bedrock_to_client_task.result()
if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None
else ()
)
logged_events: Final = (
*forwarded_logged_events,
*(
leftover_event
for leftover_event in transformation_config.leftover_usage_done_events()
if _should_log_event(leftover_event)
),
)
if logged_events:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
logging_obj.dispatch_success_handlers(
list(logged_events), # mutable-ok: realtime spend logging requires a list result
prefer_async_handlers=True,
)
)
except Exception as e:
verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e)
try:
@ -304,8 +347,8 @@ class BedrockRealtime(BaseAWSLLM):
model: str,
logging_obj: LiteLLMLogging,
session_state: RealtimeResponseTransformInput,
):
"""Forward messages from Bedrock stream to client WebSocket."""
) -> AsyncIterator[OpenAIRealtimeEvents]:
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging."""
try:
while True:
# Receive from Bedrock
@ -353,11 +396,14 @@ class BedrockRealtime(BaseAWSLLM):
)
# Send transformed messages to client
openai_messages = transformed_response.get("response", [])
response_value = transformed_response["response"]
openai_messages = response_value if isinstance(response_value, list) else (response_value,)
for openai_message in openai_messages:
message_json = json.dumps(openai_message)
await client_ws.send_text(message_json)
verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
if _should_log_event(openai_message):
yield openai_message
except Exception as e:
verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True)

190
litellm/llms/bedrock/realtime/transformation.py
View file

@ -7,7 +7,7 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format.
import base64
import json
import uuid as uuid_lib
from typing import Any, Final
from typing import Any, Final, cast
from pydantic import BaseModel
@ -20,29 +20,54 @@ from litellm.types.llms.openai import (
OpenAIRealtimeContentPartDone,
OpenAIRealtimeDoneEvent,
OpenAIRealtimeEvents,
OpenAIRealtimeInputAudioBufferSpeechEvent,
OpenAIRealtimeInputAudioTranscriptionCompleted,
OpenAIRealtimeInputAudioTranscriptionDelta,
OpenAIRealtimeOutputItemDone,
OpenAIRealtimeResponseAudioDone,
OpenAIRealtimeResponseContentPartAdded,
OpenAIRealtimeResponseDelta,
OpenAIRealtimeResponseDoneObject,
OpenAIRealtimeResponseTextDone,
OpenAIRealtimeResponseUsage,
OpenAIRealtimeStreamResponseBaseObject,
OpenAIRealtimeStreamResponseOutputItemAdded,
OpenAIRealtimeStreamSession,
OpenAIRealtimeStreamSessionEvents,
OpenAIRealtimeUsageTokenDetails,
)
from litellm.types.realtime import (
ALL_DELTA_TYPES,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
from litellm.utils import get_empty_usage
class BedrockContentEnd(BaseModel):
stopReason: str | None = None
class BedrockUsageTokenDetails(BaseModel):
speechTokens: int = 0
textTokens: int = 0
class BedrockUsageDetailsTotal(BaseModel):
input: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
output: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
class BedrockUsageDetails(BaseModel):
total: BedrockUsageDetailsTotal = BedrockUsageDetailsTotal()
class BedrockUsageEvent(BaseModel):
totalInputTokens: int = 0
totalOutputTokens: int = 0
totalTokens: int = 0
details: BedrockUsageDetails = BedrockUsageDetails()
TRIGGER_AUDIO_SAMPLE_RATE_HERTZ: Final = 16000
TRIGGER_AUDIO_BYTES_PER_SECOND: Final = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2
TRIGGER_LEADING_SILENCE: Final = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2)
@ -87,6 +112,15 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Text configuration
self.text_media_type = "text/plain"
# Response-stream state (Bedrock events carry no role on textOutput,
# so the USER/ASSISTANT split from contentStart is tracked here)
self._user_transcript_active = False
self._user_transcript_generation_stage: str | None = None
self._user_item_id: str | None = None
self._user_transcript_buffer = ""
self._cumulative_usage = BedrockUsageEvent()
self._reported_usage = BedrockUsageEvent()
def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
"""Validate environment - no special validation needed for Bedrock."""
return headers
@ -691,6 +725,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
role: Final = content_start.get("role")
if role != "ASSISTANT":
if role == "USER" and content_start.get("type") == "TEXT":
self._user_transcript_active = True
self._user_transcript_generation_stage = self._parse_generation_stage(
content_start.get("additionalModelFields")
)
return (
[],
current_response_id,
@ -700,6 +739,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
)
verbose_logger.debug("Handling ASSISTANT contentStart")
is_new_response: Final = current_response_id is None
# Initialize IDs if needed
if not current_response_id:
@ -715,7 +755,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
returned_messages: Final[list[OpenAIRealtimeEvents]] = []
# Send response.created
# Send response.created only once per response (a response can contain
# multiple content blocks, e.g. TEXT then AUDIO)
response_created: Final = OpenAIRealtimeStreamResponseBaseObject(
type="response.created",
event_id=f"event_{uuid.uuid4()}",
@ -727,7 +768,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
"conversation_id": current_conversation_id,
},
)
returned_messages.append(response_created)
if is_new_response:
returned_messages.append(response_created)
# Send response.output_item.added
output_item_added: Final = OpenAIRealtimeStreamResponseOutputItemAdded(
@ -767,6 +809,108 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
current_delta_type,
)
@staticmethod
def _parse_generation_stage(additional_model_fields: object) -> str | None:
if not isinstance(additional_model_fields, str):
return None
try:
parsed: Final = json.loads(additional_model_fields)
except json.JSONDecodeError:
return None
stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None
return stage if isinstance(stage, str) else None
def _current_user_item_id(self, new_utterance: bool = False) -> str:
"""Item id shared by all events of one user utterance (speech boundaries and transcript)."""
if new_utterance or self._user_item_id is None:
self._user_item_id = f"item_{uuid.uuid4()}"
return self._user_item_id
def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]:
"""Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events."""
verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End")
speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = {
"type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped",
"event_id": f"event_{uuid.uuid4()}",
"item_id": self._current_user_item_id(new_utterance=is_speech_start),
}
return (speech_event,)
def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None:
"""Record Bedrock's session-cumulative usage totals for the next response.done."""
verbose_logger.debug("Handling usageEvent")
self._cumulative_usage = usage_event
def _take_usage_delta(self) -> OpenAIRealtimeResponseUsage:
"""Usage for the response now completing: cumulative totals minus what prior response.done events reported."""
prior: Final = self._reported_usage
latest: Final = self._cumulative_usage
self._reported_usage = latest
input_details: Final[OpenAIRealtimeUsageTokenDetails] = {
"audio_tokens": latest.details.total.input.speechTokens - prior.details.total.input.speechTokens,
"text_tokens": latest.details.total.input.textTokens - prior.details.total.input.textTokens,
"cached_tokens": 0,
}
output_details: Final[OpenAIRealtimeUsageTokenDetails] = {
"audio_tokens": latest.details.total.output.speechTokens - prior.details.total.output.speechTokens,
"text_tokens": latest.details.total.output.textTokens - prior.details.total.output.textTokens,
}
usage_delta: Final[OpenAIRealtimeResponseUsage] = {
"input_tokens": latest.totalInputTokens - prior.totalInputTokens,
"output_tokens": latest.totalOutputTokens - prior.totalOutputTokens,
"total_tokens": latest.totalTokens - prior.totalTokens,
"input_token_details": input_details,
"output_token_details": output_details,
}
return usage_delta
def leftover_usage_done_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
"""Logged-only response.done for usage Bedrock reports after the final turn's contentEnd."""
if self._cumulative_usage == self._reported_usage:
return ()
usage: Final = self._take_usage_delta()
leftover_done: Final = OpenAIRealtimeDoneEvent(
type="response.done",
event_id=f"event_{uuid.uuid4()}",
response=OpenAIRealtimeResponseDoneObject(
object="realtime.response",
id=f"resp_{uuid.uuid4()}",
status="completed",
conversation_id=f"conv_{uuid.uuid4()}",
usage=dict(usage), # mutable-ok: OpenAIRealtimeResponseDoneObject types usage as plain dict
),
)
return (leftover_done,)
def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]:
"""Transform a USER-role Bedrock textOutput (ASR transcript) to an OpenAI transcription delta."""
verbose_logger.debug("Handling USER textOutput (ASR transcript)")
delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
"type": "conversation.item.input_audio_transcription.delta",
"event_id": f"event_{uuid.uuid4()}",
"item_id": self._current_user_item_id(),
"content_index": 0,
"delta": transcript,
}
if self._user_transcript_generation_stage != "SPECULATIVE":
self._user_transcript_buffer += transcript
return (delta_event,)
def user_transcript_completed_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
"""One completed event with the full transcript once the FINAL user content block ends."""
transcript: Final = self._user_transcript_buffer
if not transcript:
return ()
self._user_transcript_buffer = ""
completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {
"type": "conversation.item.input_audio_transcription.completed",
"event_id": f"event_{uuid.uuid4()}",
"item_id": self._current_user_item_id(),
"content_index": 0,
"transcript": transcript,
}
return (completed_event,)
def transform_text_output_event(
self,
event: dict,
@ -985,7 +1129,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if not current_response_id or not current_conversation_id:
return [], None, None, None
usage_obj: Final = get_empty_usage()
usage: Final = self._take_usage_delta()
response_done: Final = OpenAIRealtimeDoneEvent(
type="response.done",
event_id=f"event_{uuid.uuid4()}",
@ -995,11 +1139,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
status="completed",
output=[],
conversation_id=current_conversation_id,
usage={
"prompt_tokens": usage_obj.prompt_tokens,
"completion_tokens": usage_obj.completion_tokens,
"total_tokens": usage_obj.total_tokens,
},
usage=dict(usage),
),
)
@ -1042,8 +1182,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Create a function call arguments done event
# This is a custom event format that matches what clients expect
from typing import cast
function_call_event: Final[dict[str, Any]] = {
"type": "response.function_call_arguments.done",
"event_id": f"event_{uuid.uuid4()}",
@ -1194,18 +1332,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
returned_messages.extend(events)
elif "textOutput" in event:
events, current_delta_chunks = self.transform_text_output_event(
event,
current_output_item_id,
current_response_id,
current_delta_chunks,
)
returned_messages.extend(events)
if self._user_transcript_active:
returned_messages.extend(self.transform_user_transcript_event(event["textOutput"].get("content", "")))
else:
events, current_delta_chunks = self.transform_text_output_event(
event,
current_output_item_id,
current_response_id,
current_delta_chunks,
)
returned_messages.extend(events)
elif "audioOutput" in event:
events = self.transform_audio_output_event(event, current_output_item_id, current_response_id)
returned_messages.extend(events)
elif "contentEnd" in event and self._user_transcript_active:
self._user_transcript_active = False
self._user_transcript_generation_stage = None
returned_messages.extend(self.user_transcript_completed_events())
elif "contentEnd" in event:
events, current_delta_chunks = self.transform_content_end_event(
event,
@ -1224,6 +1370,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
) = self._response_done_events(current_response_id, current_conversation_id)
returned_messages.extend(done_events)
elif "userSpeechStart" in event or "userSpeechEnd" in event:
returned_messages.extend(self.transform_user_speech_event("userSpeechStart" in event))
elif "usageEvent" in event:
self.transform_usage_event(BedrockUsageEvent.model_validate(event["usageEvent"]))
elif "toolUse" in event:
events, tool_call_id, tool_name = self.transform_tool_use_event(
event, current_output_item_id, current_response_id

94
litellm/llms/gemini/common_utils.py
View file

@ -2,7 +2,7 @@ import base64
import datetime
import json
import math
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import Any, Final
import httpx
@ -128,24 +128,35 @@ def is_gemini_image_model(model: str) -> bool:
return "gemini" in base_model
def _parse_image_config_string(raw_image_config: str, model: str) -> object:
try:
return json.loads(raw_image_config)
except json.JSONDecodeError as exc:
raise litellm.UnsupportedParamsError(
model=model,
message="`imageConfig` must be valid JSON when provided as a string.",
) from exc
def map_openai_image_params_to_gemini(
params: dict[str, Any],
params: Mapping[str, object],
model: str,
supported_params: Sequence[str],
optional_params: dict[str, Any] | None = None,
optional_params: Mapping[str, object] | None = None,
parse_image_config_string: bool = False,
) -> dict[str, Any]:
optional_params = optional_params or {}
) -> dict[str, object]:
already_mapped: Final[Mapping[str, object]] = optional_params or {}
filtered_params: Final = {key: value for key, value in params.items() if key in supported_params}
mapped_params: Final[dict[str, Any]] = {}
mapped_params: Final[dict[str, object]] = {}
if "n" in filtered_params and "n" not in optional_params:
if "n" in filtered_params and "n" not in already_mapped:
mapped_params["sampleCount"] = filtered_params["n"]
if "size" in filtered_params and "size" not in optional_params:
size_param: Final = filtered_params.get("size")
if isinstance(size_param, str) and "size" not in already_mapped:
image_config: Final = map_openai_size_to_gemini_image_config(
filtered_params["size"],
size_param,
model,
)
if image_config is not None:
@ -156,33 +167,30 @@ def map_openai_image_params_to_gemini(
if "imageSize" in image_config:
mapped_params["imageSize"] = image_config["imageSize"]
image_config_param = filtered_params.get("imageConfig")
if isinstance(image_config_param, str) and parse_image_config_string:
try:
image_config_param = json.loads(image_config_param)
except json.JSONDecodeError as exc:
raise litellm.UnsupportedParamsError(
model=model,
message="`imageConfig` must be valid JSON when provided as a string.",
) from exc
raw_image_config: Final = filtered_params.get("imageConfig")
image_config_param: Final[object] = (
_parse_image_config_string(raw_image_config, model)
if isinstance(raw_image_config, str) and parse_image_config_string
else raw_image_config
)
if isinstance(image_config_param, dict):
mapped_params["imageConfig"] = image_config_param
for key, value in filtered_params.items():
if key not in ("n", "size", "imageConfig", "tools", "web_search_options") and key not in optional_params:
if key not in ("n", "size", "imageConfig", "tools", "web_search_options") and key not in already_mapped:
mapped_params[key] = value
return mapped_params
def _dedupe_gemini_search_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
def _dedupe_gemini_search_tools(tools: list[dict[str, object]]) -> list[dict[str, object]]:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
search_tool_keys: Final = VertexGeminiConfig._search_tool_keys()
seen_search_keys: Final[set[str]] = set()
deduped_tools: Final[list[dict[str, Any]]] = []
deduped_tools: Final[list[dict[str, object]]] = []
for tool in tools:
if not isinstance(tool, dict):
@ -203,7 +211,7 @@ def _dedupe_gemini_search_tools(tools: list[dict[str, Any]]) -> list[dict[str, A
return deduped_tools
def _has_gemini_search_tool(tools: list[Any]) -> bool:
def _has_gemini_search_tool(tools: list[object]) -> bool:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -213,9 +221,9 @@ def _has_gemini_search_tool(tools: list[Any]) -> bool:
def map_gemini_image_tools_params(
non_default_params: dict[str, Any],
mapped_params: dict[str, Any],
) -> dict[str, Any]:
non_default_params: Mapping[str, object],
mapped_params: Mapping[str, object],
) -> dict[str, object]:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -239,21 +247,24 @@ def map_gemini_image_tools_params(
gemini_config._drop_search_tools_mixed_with_functions(result)
if isinstance(result.get("tools"), list):
result["tools"] = _dedupe_gemini_search_tools(result["tools"])
resolved_tools: Final = result.get("tools")
if isinstance(resolved_tools, list):
result["tools"] = _dedupe_gemini_search_tools(resolved_tools)
return result
def get_gemini_image_web_search_requests(
response_data: dict[str, Any],
response_data: Mapping[str, object],
) -> int | None:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
grounding_metadata: Final[list[dict[str, Any]]] = []
for candidate in response_data.get("candidates", []):
raw_candidates: Final = response_data.get("candidates")
candidates: Final[list[object]] = raw_candidates if isinstance(raw_candidates, list) else []
grounding_metadata: Final[list[dict[str, object]]] = []
for candidate in candidates:
if not isinstance(candidate, dict):
continue
candidate_grounding = candidate.get("groundingMetadata")
@ -267,13 +278,14 @@ def get_gemini_image_web_search_requests(
def get_gemini_image_generation_config(
model: str,
optional_params: dict[str, Any],
) -> dict[str, Any]:
generation_config: Final[dict[str, Any]] = {"response_modalities": ["IMAGE", "TEXT"]}
optional_params: Mapping[str, object],
) -> dict[str, object]:
generation_config: Final[dict[str, object]] = {"response_modalities": ["IMAGE", "TEXT"]}
image_config: Final[dict[str, Any]] = {}
if isinstance(optional_params.get("imageConfig"), dict):
image_config.update(optional_params["imageConfig"])
raw_image_config: Final = optional_params.get("imageConfig")
image_config: Final[dict[str, object]] = {}
if isinstance(raw_image_config, dict):
image_config.update(raw_image_config)
if not supports_gemini_image_size(model):
image_config.pop("imageSize", None)
@ -398,7 +410,7 @@ class GeminiModelInfo(BaseLLMModelInfo):
f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}"
)
models: Final = response.json()["models"]
models: Final[list[dict[str, str]]] = response.json()["models"]
litellm_model_names: Final = self.process_model_name(models)
return litellm_model_names
@ -473,12 +485,12 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
async def count_tokens(
self,
model_to_use: str,
messages: list[dict[str, Any]] | None,
contents: list[dict[str, Any]] | None,
messages: list[dict[str, object]] | None,
contents: list[dict[str, object]] | None,
deployment: dict[str, Any] | None = None,
request_model: str = "",
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
tools: list[dict[str, object]] | None = None,
system: object | None = None,
) -> TokenCountResponse | None:
import copy

38
litellm/llms/gemini/files/transformation.py
View file

@ -5,11 +5,13 @@ For vertex ai, check out the vertex_ai/files/handler.py file.
"""
import time
from typing import Any, Final, Literal
from collections.abc import Mapping
from typing import Final, Literal, TypedDict
from urllib.parse import urlparse
import httpx
from openai.types.file_deleted import FileDeleted
from typing_extensions import ReadOnly, Required
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
@ -18,7 +20,6 @@ from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
LiteLLMLoggingObj,
)
from litellm.types.llms.gemini import GeminiCreateFilesResponseObject
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
@ -31,6 +32,25 @@ from litellm.types.utils import LlmProviders
from ..common_utils import GeminiModelInfo
class _GeminiFileMetadata(TypedDict, total=False):
name: ReadOnly[str]
uri: ReadOnly[Required[str]]
displayName: ReadOnly[Required[str]]
mimeType: ReadOnly[str]
sizeBytes: ReadOnly[Required[str]]
createTime: ReadOnly[Required[str]]
updateTime: ReadOnly[str]
expirationTime: ReadOnly[str]
sha256Hash: ReadOnly[str]
state: ReadOnly[str]
source: ReadOnly[str]
error: ReadOnly[Mapping[str, object]]
class _GeminiCreateFileResponse(TypedDict):
file: ReadOnly[_GeminiFileMetadata]
class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
def __init__(self):
pass
@ -41,14 +61,14 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
def validate_environment(
self,
headers: dict[Any, Any],
headers: dict[str, str],
model: str,
messages: list[AllMessageValues],
optional_params: dict[Any, Any],
litellm_params: dict[Any, Any],
optional_params: dict[str, object],
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[Any, Any]:
) -> dict[str, str]:
"""
Validate environment and add Gemini API key to headers.
Google AI Studio uses x-goog-api-key header for authentication.
@ -164,9 +184,9 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
Transform Gemini's file upload response into OpenAI-style FileObject
"""
try:
response_json: Final = raw_response.json()
response_json: Final[_GeminiCreateFileResponse] = raw_response.json()
response_object: Final = GeminiCreateFilesResponseObject(**response_json.get("file", {}))
response_object: Final = response_json["file"]
# Extract file information from Gemini response
@ -262,7 +282,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
"""
try:
verbose_logger.debug("Retrieve file response: %s", raw_response.text)
response_json: Final = raw_response.json()
response_json: Final[_GeminiFileMetadata] = raw_response.json()
verbose_logger.debug("Response JSON: %s", response_json)
# Map Gemini state to OpenAI status
gemini_state: Final = response_json.get("state", "STATE_UNSPECIFIED")

61
litellm/llms/gemini/realtime/transformation.py
View file

@ -7,6 +7,8 @@ from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Any, Final, cast
from typing_extensions import ReadOnly, Required, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
@ -96,6 +98,23 @@ def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
return VertexGeminiConfig()._map_audio_params({"voice": voice})
class _GeminiLiveSetupEnvelope(TypedDict, total=False):
setup: ReadOnly[BidiGenerateContentSetup]
class _OpenAIRealtimeClientEvent(TypedDict, total=False):
type: ReadOnly[str]
audio: ReadOnly[Required[str]]
session: ReadOnly[dict[str, object]]
item: ReadOnly[dict[str, object]]
def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup:
envelope: Final[_GeminiLiveSetupEnvelope] = json.loads(session_configuration_request)
empty_setup: Final[BidiGenerateContentSetup] = {}
return envelope.get("setup", empty_setup)
# Google bills Live transcription at an estimated 25 audio tokens/sec of input and
# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
@ -130,7 +149,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return True
@staticmethod
def _usage_detail_alias(details: Any, defaults: dict[str, int]) -> dict[str, Any]:
def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]:
if not isinstance(details, dict):
return dict(defaults)
return {
@ -139,7 +158,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
}
@staticmethod
def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, Any]) -> dict[str, Any]:
def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, Any]) -> dict[str, object]:
usage_dict.setdefault(
"input_token_details",
GeminiRealtimeConfig._usage_detail_alias(
@ -222,8 +241,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if not session_configuration_request:
return False
try:
setup: Final = json.loads(session_configuration_request).get("setup", {})
automatic_detection: Final = setup.get("realtimeInputConfig", {}).get("automaticActivityDetection", {})
setup: Final = _parse_setup(session_configuration_request)
automatic_detection: Final[object] = setup.get("realtimeInputConfig", {}).get(
"automaticActivityDetection", {}
)
return isinstance(automatic_detection, dict) and automatic_detection.get("disabled") is True
except (json.JSONDecodeError, TypeError, AttributeError):
return False
@ -406,7 +427,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[Any]) -> tuple[str, ...]:
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
@ -431,7 +452,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
def _handle_session_update(
self,
json_message: dict,
json_message: _OpenAIRealtimeClientEvent,
model: str,
session_configuration_request: str | None,
) -> list[str]:
@ -445,7 +466,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
with a 1007, tearing the session down). To carry tools/instructions, send
them on the first session.update before any conversation content.
"""
session_payload = json_message.get("session") or {}
empty_session: Final[dict[str, object]] = {}
session_payload = json_message.get("session") or empty_session
# Normalize GA-remapped fields (``output_modalities``,
# nested ``audio.input.transcription``,
# ``audio.input.turn_detection``) back to their flat beta keys so
@ -486,14 +508,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
verbose_logger.debug("Gemini Realtime: Ignoring session.update (setup already sent)")
return []
def _handle_conversation_item(self, json_message: dict) -> list[str]:
def _handle_conversation_item(self, json_message: _OpenAIRealtimeClientEvent) -> list[str]:
"""
Handle conversation.item.create for user text or function call output.
Converts OpenAI format to Gemini's clientContent (for user text) or
toolResponse (for function outputs).
"""
item: Final = json_message.get("item", {})
empty_item: Final[dict[str, object]] = {}
item: Final = json_message.get("item", empty_item)
item_type: Final = item.get("type")
if item_type == "function_call_output":
@ -524,7 +547,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
call_id,
)
function_response: Final[dict[str, Any]] = {"response": output_dict}
function_response: Final[dict[str, object]] = {"response": output_dict}
if self._include_function_response_id() and call_id:
function_response["id"] = call_id
if function_name:
@ -559,7 +582,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
) -> list[str]:
realtime_input_dict: BidiGenerateContentRealtimeInput = {}
try:
json_message: Final = json.loads(message)
json_message: Final[_OpenAIRealtimeClientEvent] = json.loads(message)
except json.JSONDecodeError:
if isinstance(message, bytes):
message_str = message.decode("utf-8", errors="replace")
@ -610,9 +633,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
session_configuration_request: str | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = json.loads(
session_configuration_request
).get("setup", {})
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
@ -663,7 +684,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
session_configuration_request_dict: BidiGenerateContentSetup = {}
if session_configuration_request is not None:
try:
session_configuration_request_dict = json.loads(session_configuration_request).get("setup", {})
session_configuration_request_dict = _parse_setup(session_configuration_request)
except json.JSONDecodeError:
session_configuration_request_dict = {}
generation_config: Final = session_configuration_request_dict.get("generationConfig", {})
@ -931,9 +952,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return events
@staticmethod
def get_nested_value(obj: dict, path: str) -> Any:
def get_nested_value(obj: dict, path: str) -> object | None:
keys: Final = path.split(".")
current = obj
current: object = obj
for key in keys:
if isinstance(current, dict) and key in current:
current = current[key]
@ -1011,9 +1032,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
current_response_id = f"resp_{uuid.uuid4()}"
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = json.loads(
session_configuration_request
).get("setup", {})
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
@ -1337,7 +1356,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
session_setup: BidiGenerateContentSetup = {}
if session_configuration_request is not None:
try:
session_setup = json.loads(session_configuration_request).get("setup", {})
session_setup = _parse_setup(session_configuration_request)
except (json.JSONDecodeError, TypeError):
session_setup = {}
tool_call_generation_config = session_setup.get("generationConfig", {}) or {}

6
litellm/llms/gigachat/__init__.py
View file

@ -15,9 +15,11 @@ API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview
from .chat.transformation import GigaChatConfig, GigaChatError
from .embedding.transformation import GigaChatEmbeddingConfig
from .passthrough.transformation import GigaChatPassthroughConfig
__all__ = [
__all__ = (
"GigaChatConfig",
"GigaChatEmbeddingConfig",
"GigaChatError",
]
"GigaChatPassthroughConfig",
)

92
litellm/llms/gigachat/authenticator.py
View file

@ -7,6 +7,7 @@ Based on official GigaChat SDK authentication flow.
import time
import uuid
from collections.abc import Mapping
from typing import Final
import httpx
@ -16,7 +17,7 @@ from litellm.caching.caching import InMemoryCache
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
_get_httpx_client,
_get_httpx_client, # pyright: ignore[reportPrivateUsage] # house cached-client factory has no public alias
get_async_httpx_client,
)
from litellm.secret_managers.main import get_secret_str
@ -63,6 +64,7 @@ def get_access_token(
credentials: str | None = None,
scope: str | None = None,
auth_url: str | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> str:
"""
Get valid access token, using cache if available.
@ -78,71 +80,88 @@ def get_access_token(
Raises:
GigaChatAuthError: If authentication fails
"""
credentials = credentials or _get_credentials()
if not credentials:
if not litellm_params:
litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default
access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN")
if access_token:
return access_token
effective_credentials: Final = credentials or _get_credentials()
if not effective_credentials:
raise GigaChatAuthError(
status_code=401,
message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.",
)
scope = scope or _get_scope()
auth_url = auth_url or _get_auth_url()
effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope()
effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url()
# Check cache
cache_key: Final = f"gigachat_token:{credentials[:16]}"
cache_key: Final = f"gigachat_token:{effective_credentials[:16]}"
cached: Final = _token_cache.get_cache(cache_key)
if cached:
token, expires_at = cached
_token, _expires_at = cached
# Check if token is still valid (with buffer)
if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS:
if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS:
verbose_logger.debug("Using cached GigaChat access token")
return token
return _token
# Request new token
token, expires_at = _request_token_sync(credentials, scope, auth_url)
new_token, new_expires_at = _request_token_sync(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str
# Cache token
ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
if ttl_seconds > 0:
_token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds)
if new_expires_at:
# Cache token
ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
if ttl_seconds > 0:
_token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds)
return token
return new_token
async def get_access_token_async(
credentials: str | None = None,
scope: str | None = None,
auth_url: str | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> str:
"""Async version of get_access_token."""
credentials = credentials or _get_credentials()
if not credentials:
if not litellm_params:
litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default
access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN")
if access_token:
return access_token
effective_credentials: Final = credentials or _get_credentials()
if not effective_credentials:
raise GigaChatAuthError(
status_code=401,
message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.",
)
scope = scope or _get_scope()
auth_url = auth_url or _get_auth_url()
effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope()
effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url()
# Check cache
cache_key: Final = f"gigachat_token:{credentials[:16]}"
cache_key: Final = f"gigachat_token:{effective_credentials[:16]}"
cached: Final = _token_cache.get_cache(cache_key)
if cached:
token, expires_at = cached
if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS:
_token, _expires_at = cached
if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS:
verbose_logger.debug("Using cached GigaChat access token")
return token
return _token
# Request new token
token, expires_at = await _request_token_async(credentials, scope, auth_url)
new_token, new_expires_at = await _request_token_async(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str
# Cache token
ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
if ttl_seconds > 0:
_token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds)
if new_expires_at:
# Cache token
ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
if ttl_seconds > 0:
_token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds)
return token
return new_token
def _request_token_sync(
@ -154,7 +173,7 @@ def _request_token_sync(
Request new access token from GigaChat OAuth endpoint (sync).
Returns:
Tuple of (access_token, expires_at_ms)
tuple of (access_token, expires_at_ms)
"""
headers: Final = {
"Authorization": f"Basic {credentials}",
@ -169,7 +188,7 @@ def _request_token_sync(
client: Final = _get_http_client()
response: Final = client.post(auth_url, headers=headers, data=data, timeout=30)
response.raise_for_status()
return _parse_token_response(response)
return _parse_token_response(response) # pyright: ignore[reportArgumentType] # httpx Response may be None at type level
except httpx.HTTPStatusError as e:
raise GigaChatAuthError(
status_code=e.response.status_code,
@ -204,7 +223,7 @@ async def _request_token_async(
)
response: Final = await client.post(auth_url, headers=headers, data=data, timeout=30)
response.raise_for_status()
return _parse_token_response(response)
return _parse_token_response(response) # pyright: ignore[reportArgumentType] # httpx Response may be None at type level
except httpx.HTTPStatusError as e:
raise GigaChatAuthError(
status_code=e.response.status_code,
@ -223,7 +242,7 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
access_token: Final = data.get("tok") or data.get("access_token")
expires_at = data.get("exp") or data.get("expires_at")
expires_at_raw: Final = data.get("exp") or data.get("expires_at")
if not access_token:
raise GigaChatAuthError(
@ -232,8 +251,11 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
)
# expires_at is in milliseconds
if isinstance(expires_at, str):
expires_at = int(expires_at)
expires_at: int # rebind-ok: conditionally assigned from str or int
if isinstance(expires_at_raw, str):
expires_at = int(expires_at_raw) # rebind-ok: conditionally assigned from str or int
else:
expires_at = expires_at_raw # pyright: ignore[reportAssignmentType] # raw value is int or str; converted above; rebind-ok: conditionally assigned from str or int
verbose_logger.debug("GigaChat access token obtained successfully")
return access_token, expires_at

4
litellm/llms/gigachat/chat/__init__.py
View file

@ -5,8 +5,8 @@ GigaChat Chat Module
from .streaming import GigaChatModelResponseIterator
from .transformation import GigaChatConfig, GigaChatError
__all__ = [
__all__ = (
"GigaChatConfig",
"GigaChatError",
"GigaChatModelResponseIterator",
]
)

68
litellm/llms/gigachat/chat/streaming.py
View file

@ -4,13 +4,15 @@ GigaChat Streaming Response Handler
import json
import uuid
from collections.abc import Mapping, Sequence
from typing import Any, Final
from litellm.llms.gigachat.utils import convert_usage
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.utils import GenericStreamingChunk
from litellm.types.utils import ChatCompletionUsageBlock, GenericStreamingChunk
class GigaChatModelResponseIterator:
@ -26,14 +28,9 @@ class GigaChatModelResponseIterator:
self.response_iterator = self.streaming_response
self.json_mode = json_mode
def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
def chunk_parser(self, chunk: Mapping[str, object]) -> GenericStreamingChunk:
"""Parse a single streaming chunk from GigaChat."""
text = ""
tool_use: ChatCompletionToolCallChunk | None = None
is_finished = False
finish_reason: str | None = None
choices: Final = chunk.get("choices", [])
choices: Sequence = chunk.get("choices") or () # mutable-ok: tuple literal as default
if not choices:
return GenericStreamingChunk(
text="",
@ -45,40 +42,63 @@ class GigaChatModelResponseIterator:
)
choice: Final = choices[0]
delta: Final = choice.get("delta", {})
finish_reason = choice.get("finish_reason")
delta: Mapping[str, object] = choice.get("delta") or {} # mutable-ok: empty dict default for get
chunk_finish_reason: Final = choice.get("finish_reason")
# Extract text content
text = delta.get("content", "") or ""
text: Final = delta.get("content", "") or ""
usage_block: ChatCompletionUsageBlock | None = None # rebind-ok: conditionally assigned after stop detection
tool_use: ChatCompletionToolCallChunk | None = None # rebind-ok: conditionally assigned on function_call
finish_reason: str | None = chunk_finish_reason
# Handle function_call in stream
if finish_reason == "function_call" and delta.get("function_call"):
func_call: Final = delta["function_call"]
args = func_call.get("arguments", {})
if isinstance(args, dict):
args = json.dumps(args, ensure_ascii=False)
raw_function_call: Final = delta.get("function_call")
if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call:
func_call: Final[Mapping[str, object]] = raw_function_call
args_raw: Final[object] = func_call.get("arguments") or {}
args_str: str # rebind-ok: conditionally assigned from dict or str
if isinstance(args_raw, dict):
args_str = json.dumps(args_raw, ensure_ascii=False) # rebind-ok: build from dict
else:
args_str = str(args_raw)
name_raw: Final = func_call.get("name")
tool_use = ChatCompletionToolCallChunk(
id=f"call_{uuid.uuid4().hex[:24]}",
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=func_call.get("name", ""),
arguments=args,
name=name_raw if isinstance(name_raw, str) else "",
arguments=args_str,
),
index=0,
)
finish_reason = "tool_calls"
if finish_reason is not None:
is_finished = True
usage_data: Final = chunk.get("usage") or {} # mutable-ok: empty dict default
if usage_data and isinstance(usage_data, dict):
validated_usage: Final = {k: int(v) for k, v in usage_data.items()}
usage = convert_usage(validated_usage)
_prompt_details: dict | None = (
usage.prompt_tokens_details.model_dump() if usage.prompt_tokens_details else None
) # rebind-ok: conditional
_completion_details: dict | None = (
usage.completion_tokens_details.model_dump() if usage.completion_tokens_details else None
) # rebind-ok: conditional
usage_block = ChatCompletionUsageBlock( # pyright: ignore[reportCallIssue] # TypedDict kwarg constructor
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
total_tokens=usage.total_tokens,
prompt_tokens_details=_prompt_details,
completion_tokens_details=_completion_details,
)
return GenericStreamingChunk(
text=text,
text=str(text),
tool_use=tool_use,
is_finished=is_finished,
is_finished=chunk_finish_reason is not None,
finish_reason=finish_reason or "",
usage=None,
usage=usage_block,
index=choice.get("index", 0),
)

155
litellm/llms/gigachat/chat/transformation.py
View file

@ -4,19 +4,22 @@ GigaChat Chat Transformation
Transforms OpenAI-format requests to GigaChat format and back.
"""
from __future__ import annotations
import json
import time
import uuid
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.gigachat.utils import convert_usage, get_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from litellm.types.utils import Choices, Message, ModelResponse
from ..authenticator import get_access_token
from ..file_handler import upload_file_sync
@ -30,9 +33,6 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
# GigaChat API endpoint
GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
def is_valid_json(value: str) -> bool:
"""Checks whether the value passed is a valid serialized JSON string"""
@ -90,30 +90,30 @@ class GigaChatConfig(BaseConfig):
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
"""Get complete API URL for chat completions."""
base: Final = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
base: Final = get_api_base(api_base)
return f"{base}/chat/completions"
def validate_environment(
self,
headers: dict,
headers: dict, # mutable-ok: mutates in place per GigaChat OAuth setup
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
) -> dict: # mutable-ok: base class contract returns dict for httpx
"""
Set up headers with OAuth token.
"""
# Get access token
credentials: Final = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY")
access_token: Final = get_access_token(credentials=credentials)
access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params)
# Store credentials for image uploads
self._current_credentials = credentials
@ -125,9 +125,9 @@ class GigaChatConfig(BaseConfig):
return headers
def get_supported_openai_params(self, model: str) -> list[str]:
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: base class contract returns list
"""Return list of supported OpenAI parameters."""
return [
return [ # mutable-ok: base class contract returns list
"stream",
"temperature",
"top_p",
@ -143,11 +143,11 @@ class GigaChatConfig(BaseConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
non_default_params: Mapping[str, object],
optional_params: dict, # mutable-ok: mutated in place per GigaChat mapping
model: str,
drop_params: bool,
) -> dict:
) -> dict: # mutable-ok: base class contract returns dict
"""Map OpenAI parameters to GigaChat parameters."""
for param, value in non_default_params.items():
if param == "stream":
@ -167,42 +167,50 @@ class GigaChatConfig(BaseConfig):
pass
elif param == "tools":
# Convert tools to functions format
optional_params["functions"] = self._convert_tools_to_functions(value)
if isinstance(value, Sequence):
optional_params["functions"] = self._convert_tools_to_functions(value)
elif param == "tool_choice":
# Map OpenAI tool_choice to GigaChat function_call
mapped_choice = self._map_tool_choice(value)
if mapped_choice is not None:
optional_params["function_call"] = mapped_choice
if isinstance(value, (str, Mapping)):
mapped_choice = self._map_tool_choice(value)
if mapped_choice is not None:
optional_params["function_call"] = mapped_choice
elif param == "functions":
optional_params["functions"] = value
elif param == "function_call":
optional_params["function_call"] = value
elif param == "response_format":
# Handle structured output via function calling
if value.get("type") == "json_schema":
if isinstance(value, Mapping) and value.get("type") == "json_schema":
json_schema = value.get("json_schema", {})
schema_name = json_schema.get("name", "structured_output")
schema = json_schema.get("schema", {})
function_def = {
function_def = { # mutable-ok: request payload for httpx
"name": schema_name,
"description": f"Output structured response: {schema_name}",
"parameters": schema,
}
if "functions" not in optional_params:
optional_params["functions"] = []
optional_params["functions"].append(function_def)
optional_params["function_call"] = {"name": schema_name}
existing_functions = optional_params.get("functions")
optional_params["functions"] = [
*(
existing_functions
if isinstance(existing_functions, Sequence) and not isinstance(existing_functions, str)
else ()
),
function_def,
]
optional_params["function_call"] = {"name": schema_name} # mutable-ok: request payload
optional_params["_structured_output"] = True
return optional_params
def _convert_tools_to_functions(self, tools: list[dict]) -> list[dict]:
def _convert_tools_to_functions(self, tools: Sequence) -> Sequence[dict]:
"""Convert OpenAI tools format to GigaChat functions format."""
functions: Final = []
functions: Final[list[dict]] = [] # mutable-ok: accumulator for building functions list
for tool in tools:
if tool.get("type") == "function":
if isinstance(tool, dict) and tool.get("type") == "function":
func = tool.get("function", {})
functions.append(
{
@ -213,7 +221,7 @@ class GigaChatConfig(BaseConfig):
)
return functions
def _map_tool_choice(self, tool_choice: str | dict) -> str | dict | None:
def _map_tool_choice(self, tool_choice: str | Mapping[str, object]) -> str | Mapping[str, object] | None:
"""
Map OpenAI tool_choice to GigaChat function_call format.
@ -246,8 +254,9 @@ class GigaChatConfig(BaseConfig):
# OpenAI format: {"type": "function", "function": {"name": "func_name"}}
# GigaChat format: {"name": "func_name"}
if tool_choice.get("type") == "function":
func_name: Final = tool_choice.get("function", {}).get("name")
if func_name:
function_spec: Final = tool_choice.get("function")
func_name: Final = function_spec.get("name") if isinstance(function_spec, Mapping) else None
if isinstance(func_name, str) and func_name:
return {"name": func_name}
# Default to None (don't set function_call)
@ -273,20 +282,51 @@ class GigaChatConfig(BaseConfig):
verbose_logger.error("Failed to upload image: %s", e)
return None
def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[str]]:
"""
Extract text and image attachments from a multimodal message content list.
Args:
content: List of content parts (OpenAI multimodal format)
Returns:
Tuple of (combined text, list of attachment file ids)
"""
texts: Final[list[str]] = [] # mutable-ok: accumulator
attachments: Final[list[str]] = [] # mutable-ok: accumulator
for part in content:
if isinstance(part, dict):
if part.get("type") == "text":
texts.append(part.get("text", ""))
elif part.get("type") == "image_url":
# Extract image URL and upload to GigaChat
image_url: object = part.get("image_url", {})
upload_url: str
if isinstance(image_url, str):
upload_url = image_url
else:
upload_url = str(image_url.get("url", "")) if isinstance(image_url, dict) else ""
if upload_url:
file_id = self._upload_image(upload_url)
if file_id:
attachments.append(file_id)
text: Final = "\n".join(texts) if texts else ""
return text, attachments
def transform_request(
self,
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
headers: Mapping[str, object],
) -> dict: # mutable-ok: request payload sent to httpx
"""Transform OpenAI request to GigaChat format."""
# Transform messages
giga_messages: Final = self._transform_messages(messages)
# Build request
request_data: Final = {
request_data: Final[dict[str, object]] = {
"model": model.replace("gigachat/", ""),
"messages": giga_messages,
}
@ -311,9 +351,9 @@ class GigaChatConfig(BaseConfig):
return request_data
def _transform_messages(self, messages: list[AllMessageValues]) -> list[dict]:
def _transform_messages(self, messages: Sequence[AllMessageValues]) -> Sequence[dict]:
"""Transform OpenAI messages to GigaChat format."""
transformed: Final = []
transformed: Final[list[dict]] = [] # mutable-ok: accumulator for building transformed messages
for i, msg in enumerate(messages):
message = dict(msg)
@ -341,24 +381,7 @@ class GigaChatConfig(BaseConfig):
# Handle list content (multimodal) - extract text and images
content = message.get("content")
if isinstance(content, list):
texts = []
attachments = []
for part in content:
if isinstance(part, dict):
if part.get("type") == "text":
texts.append(part.get("text", ""))
elif part.get("type") == "image_url":
# Extract image URL and upload to GigaChat
image_url = part.get("image_url", {})
if isinstance(image_url, str):
url = image_url
else:
url = image_url.get("url", "")
if url:
file_id = self._upload_image(url)
if file_id:
attachments.append(file_id)
message["content"] = "\n".join(texts) if texts else ""
message["content"], attachments = self._transform_list_content(content)
if attachments:
message["attachments"] = attachments
@ -393,7 +416,7 @@ class GigaChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: tiktoken.Encoding | None,
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -408,7 +431,7 @@ class GigaChatConfig(BaseConfig):
is_structured_output: Final = optional_params.get("_structured_output", False)
choices: Final = []
choices: Final[list[Choices]] = [] # mutable-ok: accumulator for building response choices
for choice in response_json.get("choices", []):
message_data = choice.get("message", {})
finish_reason = choice.get("finish_reason", "stop")
@ -462,11 +485,7 @@ class GigaChatConfig(BaseConfig):
# Build usage
usage_data: Final = response_json.get("usage", {})
usage: Final = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
)
usage: Final = convert_usage(usage_data)
model_response.id = response_json.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}")
model_response.created = response_json.get("created", int(time.time()))

22
litellm/llms/gigachat/embedding/transformation.py
View file

@ -5,6 +5,8 @@ Transforms OpenAI /v1/embeddings format to GigaChat format.
API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/reference/rest/post-embeddings
"""
from __future__ import annotations
import types
from typing import Final
@ -14,14 +16,12 @@ from litellm import LlmProviders
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.llms.gigachat.utils import get_api_base
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse
from ..authenticator import get_access_token
# GigaChat API endpoint
GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
class GigaChatEmbeddingError(BaseLLMException):
"""GigaChat Embedding API error."""
@ -78,9 +78,9 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
Returns provider info for GigaChat.
Returns:
Tuple of (custom_llm_provider, api_base, dynamic_api_key)
tuple of (custom_llm_provider, api_base, dynamic_api_key)
"""
api_base = api_base or GIGACHAT_BASE_URL
api_base = get_api_base(api_base)
return LlmProviders.GIGACHAT.value, api_base, api_key
def get_complete_url(
@ -93,7 +93,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
stream: bool | None = None,
) -> str:
"""Get the complete URL for embeddings endpoint."""
base: Final = api_base or GIGACHAT_BASE_URL
base: Final = get_api_base(api_base)
return f"{base}/embeddings"
def transform_embedding_request(
@ -114,14 +114,12 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
"""
# Normalize input to list
if isinstance(input, str):
input_list: list = [input]
elif isinstance(input, list):
input_list = input
input_list: list = [input] # rebind-ok: locally scoped conversion
else:
input_list = [input]
input_list = input
# Remove gigachat/ prefix from model if present
model = model.removeprefix("gigachat/")
model = model.removeprefix("gigachat/") # rebind-ok: parameter reassignment for normalization
return {
"model": model,
@ -191,7 +189,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
Set up headers with OAuth token for GigaChat.
"""
# Get access token via OAuth
access_token: Final = get_access_token(api_key)
access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params)
default_headers: Final = {
"Content-Type": "application/json",

15
litellm/llms/gigachat/file_handler.py
View file

@ -9,6 +9,7 @@ import base64
import hashlib
import re
import uuid
from collections.abc import Mapping
from typing import Final
from litellm._logging import verbose_logger
@ -16,13 +17,11 @@ from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
)
from litellm.llms.gigachat.utils import get_api_base
from litellm.types.utils import LlmProviders
from .authenticator import get_access_token, get_access_token_async
# GigaChat API endpoint
GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
# Simple in-memory cache for file IDs
_file_cache: Final[dict[str, str]] = {}
@ -82,6 +81,7 @@ def upload_file_sync(
image_url: str,
credentials: str | None = None,
api_base: str | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> str | None:
"""
Upload file to GigaChat and return file_id (sync).
@ -114,10 +114,10 @@ def upload_file_sync(
filename: Final = f"{uuid.uuid4()}.{ext}"
# Get access token
access_token: Final = get_access_token(credentials)
access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params)
# Upload to GigaChat
base_url: Final = api_base or GIGACHAT_BASE_URL
base_url: Final = get_api_base(api_base)
upload_url: Final = f"{base_url}/files"
client: Final = _get_httpx_client(params={"ssl_verify": False})
@ -147,6 +147,7 @@ async def upload_file_async(
image_url: str,
credentials: str | None = None,
api_base: str | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> str | None:
"""
Upload file to GigaChat and return file_id (async).
@ -179,10 +180,10 @@ async def upload_file_async(
filename: Final = f"{uuid.uuid4()}.{ext}"
# Get access token
access_token: Final = await get_access_token_async(credentials)
access_token: Final = await get_access_token_async(credentials=credentials, litellm_params=litellm_params)
# Upload to GigaChat
base_url: Final = api_base or GIGACHAT_BASE_URL
base_url: Final = get_api_base(api_base)
upload_url: Final = f"{base_url}/files"
client: Final = get_async_httpx_client(

7
litellm/llms/gigachat/passthrough/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
GigaChat passthrough Module
"""
from .transformation import GigaChatPassthroughConfig
__all__ = ("GigaChatPassthroughConfig",)

213
litellm/llms/gigachat/passthrough/transformation.py Normal file
View file

@ -0,0 +1,213 @@
from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.gigachat.authenticator import get_access_token
from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator
from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import EmbeddingResponse
if TYPE_CHECKING:
from httpx import URL, Response
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import CostResponseTypes
class GigaChatPassthroughConfig(BasePassthroughConfig):
def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
return request_data.get("stream", False)
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
endpoint: str,
request_query_params: Mapping[str, object] | None,
litellm_params: Mapping[str, object],
) -> tuple[URL, str]:
"""Get complete API URL for chat completions."""
base_target_url: Final = self.get_api_base(api_base)
if base_target_url is None:
raise Exception("GigaChat api base not found")
complete_url: Final = f"{base_target_url}/{endpoint.lstrip('/')}"
return (
httpx.URL(complete_url),
base_target_url,
)
def validate_environment(
self,
headers: dict, # mutable-ok: mutates in place to set OAuth headers
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict: # mutable-ok: base class contract returns dict for httpx
"""
Set up headers with OAuth token.
"""
# Get access token
access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params)
headers["Authorization"] = f"Bearer {access_token}" # rebind-ok: mutating for OAuth setup
headers["Content-Type"] = "application/json" # rebind-ok: mutating for OAuth setup
headers["Accept"] = "application/json" # rebind-ok: mutating for OAuth setup
return headers
def logging_non_streaming_response(
self,
model: str,
custom_llm_provider: str,
httpx_response: Response,
request_data: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
endpoint: str,
) -> CostResponseTypes | None:
from litellm import encoding
from litellm.types.utils import LlmProviders, ModelResponse
from litellm.utils import ProviderConfigManager
# cost tracking only for completions and embeddings
if "completions" in endpoint:
provider_chat_config: Final = ProviderConfigManager.get_provider_chat_config(
provider=LlmProviders(custom_llm_provider),
model=model,
)
if provider_chat_config is None:
raise ValueError(f"No provider config found for model: {model}")
raw_messages: Final = request_data.get("messages")
litellm_model_response: Final = provider_chat_config.transform_response(
model=model,
messages=list(raw_messages)
if isinstance(raw_messages, list)
else [], # mutable-ok: transform_response wants a list
raw_response=httpx_response,
model_response=ModelResponse(),
logging_obj=logging_obj,
optional_params={}, # mutable-ok: empty dict kwarg for transform_response
litellm_params={}, # mutable-ok: empty dict kwarg for transform_response
api_key="",
request_data=dict(request_data), # mutable-ok: transform_response wants a dict
encoding=encoding,
)
return litellm_model_response
if "embeddings" in endpoint:
provider_embedding_config: Final = ProviderConfigManager.get_provider_embedding_config(
provider=LlmProviders(custom_llm_provider),
model=model,
)
if provider_embedding_config is None:
raise ValueError(f"No provider config found for model: {model}")
litellm_embedding_response: Final[EmbeddingResponse] = (
provider_embedding_config.transform_embedding_response(
model=model,
raw_response=httpx_response,
model_response=EmbeddingResponse(),
logging_obj=logging_obj,
optional_params={}, # mutable-ok: empty dict kwarg for transform_embedding_response
api_key="",
request_data=dict(request_data), # mutable-ok: transform_embedding_response wants a dict
litellm_params={}, # mutable-ok: empty dict kwarg for transform_embedding_response
)
)
return litellm_embedding_response
return None
def handle_logging_collected_chunks(
self,
all_chunks: Sequence[str],
litellm_logging_obj: LiteLLMLoggingObj,
model: str,
custom_llm_provider: str,
endpoint: str,
) -> CostResponseTypes | None:
"""
1. Convert all_chunks to a ModelResponseStream
2. combine model_response_stream to model_response
3. Return the model_response
"""
from litellm.litellm_core_utils.streaming_handler import (
convert_generic_chunk_to_model_response_stream,
generic_chunk_has_all_required_fields,
)
from litellm.main import stream_chunk_builder
from litellm.types.utils import ModelResponseStream
all_translated_chunks: Final[list[object]] = [] # mutable-ok: accumulator
for chunk in all_chunks:
chunk = chunk.strip()
if not chunk or chunk == "[DONE]":
continue
chunk = chunk.removeprefix("data: ")
try:
message = json.loads(chunk)
except json.JSONDecodeError:
continue
gigachat_iterator = GigaChatModelResponseIterator(
streaming_response=None,
sync_stream=False,
)
translated_chunk = gigachat_iterator.chunk_parser(chunk=message)
if isinstance(translated_chunk, dict) and generic_chunk_has_all_required_fields( # pyright: ignore[reportUnnecessaryIsInstance] # runtime guard for patched chunk_parser
dict(translated_chunk)
):
chunk_obj = convert_generic_chunk_to_model_response_stream(
translated_chunk # pyright: ignore[reportArgumentType] # validated TypedDict
)
elif isinstance(translated_chunk, ModelResponseStream):
chunk_obj = translated_chunk
else:
continue
all_translated_chunks.append(chunk_obj)
if len(all_translated_chunks) > 0:
return stream_chunk_builder(
chunks=all_translated_chunks,
logging_obj=litellm_logging_obj,
)
return None
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
@staticmethod
def get_api_key(
api_key: str | None = None,
) -> str | None:
return api_key or get_secret_str("GIGACHAT_API_KEY")
@staticmethod
def get_base_model(model: str) -> str | None:
return model
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
return list(super().get_models(api_key, api_base))

26
litellm/llms/gigachat/utils.py Normal file
View file

@ -0,0 +1,26 @@
from collections.abc import Mapping
from typing import Final
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
# GigaChat API endpoint
GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
def convert_usage(usage_data: Mapping[str, int]) -> Usage:
precached_prompt_tokens: Final = usage_data.get("precached_prompt_tokens", 0)
prompt_tokens_details: Final = (
PromptTokensDetailsWrapper(cached_tokens=precached_prompt_tokens) if precached_prompt_tokens > 0 else None
)
return Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0) + precached_prompt_tokens,
completion_tokens=usage_data.get("completion_tokens", 0),
prompt_tokens_details=prompt_tokens_details,
total_tokens=usage_data.get("total_tokens", 0) + precached_prompt_tokens,
)
def get_api_base(api_base: str | None = None) -> str | None:
return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL

140
litellm/llms/litellm_proxy/skills/code_execution.py
View file

@ -13,55 +13,109 @@ Generated files are returned directly in the response - no separate storage need
import base64
import json
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from enum import Enum
from typing import Any, Final, Protocol
from typing import Any, Final, Protocol, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
class _ToolCallFunction(Protocol):
"""Function payload of an assistant tool call."""
name: str | None
arguments: str
class _ToolParameterSchema(TypedDict, total=False):
type: ReadOnly[str]
description: ReadOnly[str]
class _ToolCall(Protocol):
"""Tool call requested by the assistant on a chat completion choice."""
id: str
function: _ToolCallFunction
class _ToolArgumentSchema(TypedDict, total=False):
type: ReadOnly[str]
properties: ReadOnly[Mapping[str, _ToolParameterSchema]]
required: ReadOnly[Sequence[str]]
class _AssistantMessage(Protocol):
"""Assistant message carried by a chat completion choice."""
content: str | None
tool_calls: Sequence[_ToolCall] | None
class _OpenAIToolFunction(TypedDict, total=False):
name: ReadOnly[str]
description: ReadOnly[str]
parameters: ReadOnly[_ToolArgumentSchema]
class _CompletionChoice(Protocol):
"""Single choice of a chat completion response."""
finish_reason: str
message: _AssistantMessage
class _OpenAIToolSpec(TypedDict, total=False):
type: ReadOnly[str]
function: ReadOnly[_OpenAIToolFunction]
class _SandboxFile(TypedDict):
"""File generated inside the sandbox during a code execution run."""
class _AnthropicToolSpec(TypedDict, total=False):
name: ReadOnly[str]
description: ReadOnly[str]
input_schema: ReadOnly[_ToolArgumentSchema]
class _CodeExecutionArguments(TypedDict, total=False):
code: ReadOnly[str]
class _GeneratedFile(TypedDict, total=False):
name: ReadOnly[str]
mime_type: ReadOnly[str]
content_base64: ReadOnly[str]
size: ReadOnly[int]
class _SandboxGeneratedFile(TypedDict):
name: ReadOnly[str]
mime_type: ReadOnly[str]
content_base64: ReadOnly[str]
class _CodeExecutionArguments(TypedDict):
"""Arguments the model passes to the `litellm_code_execution` tool."""
class _SandboxExecutionResult(TypedDict):
success: ReadOnly[bool]
output: ReadOnly[str]
error: ReadOnly[str]
files: ReadOnly[Sequence[_SandboxGeneratedFile]]
code: NotRequired[ReadOnly[str]]
class _ExecutionResult(TypedDict, total=False):
iteration: ReadOnly[int]
success: ReadOnly[bool]
output: ReadOnly[str]
error: ReadOnly[str]
files: ReadOnly[Sequence[str]]
class _ToolCallFunction(Protocol):
name: str
arguments: str
class _ToolCall(Protocol):
id: str
function: _ToolCallFunction
class _AssistantMessage(Protocol):
content: str | None
tool_calls: Sequence[_ToolCall] | None
class _ResponseChoice(Protocol):
message: _AssistantMessage
finish_reason: str | None
class _CompletionResponse(Protocol):
choices: Sequence[_ResponseChoice]
class _CodeExecutionOutcome(TypedDict, total=False):
response: ReadOnly[_CompletionResponse | None]
files: ReadOnly[Sequence[_GeneratedFile]]
execution_results: ReadOnly[Sequence[_ExecutionResult]]
messages: ReadOnly[Sequence[dict[str, object]]]
max_iterations_reached: ReadOnly[bool]
def _parse_code_execution_arguments(serialized_arguments: str) -> _CodeExecutionArguments:
return json.loads(serialized_arguments)
class LiteLLMInternalTools(str, Enum):
@ -75,7 +129,7 @@ class LiteLLMInternalTools(str, Enum):
CODE_EXECUTION = "litellm_code_execution"
def get_litellm_code_execution_tool() -> dict[str, object]:
def get_litellm_code_execution_tool() -> _OpenAIToolSpec:
"""
Returns the litellm_code_execution tool definition in OpenAI format.
@ -96,7 +150,7 @@ def get_litellm_code_execution_tool() -> dict[str, object]:
}
def get_litellm_code_execution_tool_anthropic() -> dict[str, object]:
def get_litellm_code_execution_tool_anthropic() -> _AnthropicToolSpec:
"""
Returns the litellm_code_execution tool definition in Anthropic/messages API format.
@ -143,12 +197,12 @@ class CodeExecutionHandler:
async def execute_with_code_execution(
self,
model: str,
messages: list[dict],
tools: list[dict],
messages: list[dict[str, object]],
tools: list[_OpenAIToolSpec],
skill_files: dict[str, bytes],
skill_id: str | None = None,
**kwargs,
) -> dict[str, object]:
) -> _CodeExecutionOutcome:
"""
Execute an LLM call with automatic code execution handling.
@ -179,8 +233,8 @@ class CodeExecutionHandler:
)
current_messages: Final = list(messages)
generated_files: Final[list[dict[str, object]]] = [] # Files returned directly
execution_results: Final[list[dict[str, object]]] = []
generated_files: Final[list[_GeneratedFile]] = [] # Files returned directly
execution_results: Final[list[_ExecutionResult]] = []
executor: Final = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
response: Any = None # Initialize to avoid possibly unbound error
@ -196,9 +250,9 @@ class CodeExecutionHandler:
**kwargs,
)
choice: _CompletionChoice = response.choices[0]
choice: _ResponseChoice = response.choices[0]
assistant_message = choice.message
stop_reason: str = choice.finish_reason
stop_reason = choice.finish_reason
# Build assistant message for conversation history
assistant_msg_dict: dict[str, object] = {
@ -236,19 +290,19 @@ class CodeExecutionHandler:
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
# Execute code in sandbox
try:
args: _CodeExecutionArguments = json.loads(tool_call.function.arguments)
code: str = args.get("code", "")
args = _parse_code_execution_arguments(tool_call.function.arguments)
code = args.get("code", "")
verbose_logger.debug("CodeExecutionHandler: Executing code (%s chars)", len(code))
exec_result = executor.execute(
exec_result: _SandboxExecutionResult = executor.execute(
code=code,
skill_files=skill_files,
)
verbose_logger.debug("CodeExecutionHandler: Execution result: %s", exec_result)
sandbox_files: Sequence[_SandboxFile] = exec_result["files"]
sandbox_files: Sequence[_SandboxGeneratedFile] = exec_result["files"]
execution_results.append(
{
@ -326,7 +380,7 @@ class CodeExecutionHandler:
}
def has_code_execution_tool(tools: list[dict] | None) -> bool:
def has_code_execution_tool(tools: list[_OpenAIToolSpec] | None) -> bool:
"""Check if litellm_code_execution tool is in the tools list."""
if not tools:
return False
@ -337,7 +391,7 @@ def has_code_execution_tool(tools: list[dict] | None) -> bool:
return False
def add_code_execution_tool(tools: list[dict] | None) -> list[dict]:
def add_code_execution_tool(tools: list[_OpenAIToolSpec] | None) -> list[_OpenAIToolSpec]:
"""Add litellm_code_execution tool if not already present."""
tools = tools or []
if not has_code_execution_tool(tools):

30
litellm/llms/nvidia_riva/audio_transcription/audio_utils.py
View file

@ -16,7 +16,7 @@ import io
import os
import tempfile
from dataclasses import dataclass
from typing import Any, Final, cast
from typing import Final, Protocol, cast
from litellm.llms.nvidia_riva.audio_transcription.transformation import (
RIVA_TARGET_NUM_CHANNELS,
@ -24,10 +24,30 @@ from litellm.llms.nvidia_riva.audio_transcription.transformation import (
)
from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
# Keep this as Any: the module intentionally avoids importing numpy at module
# import time (optional dependency), and project-wide mypy config evaluates this
# file in contexts where conditional type aliases can degrade to "FloatArray?".
FloatArray = Any
class FloatArray(Protocol):
"""Structural view of the ``numpy.ndarray`` surface this module relies on."""
@property
def ndim(self) -> int: ...
@property
def shape(self) -> tuple[int, ...]: ...
@property
def size(self) -> int: ...
def mean(self, axis: int) -> "FloatArray": ...
def ravel(self) -> "FloatArray": ...
def astype(self, dtype: object) -> "FloatArray": ...
def tobytes(self) -> bytes: ...
def __getitem__(self, key: object) -> "FloatArray": ...
def __mul__(self, other: float) -> "FloatArray": ...
_INSTALL_HINT = "Install Riva STT extras to enable automatic audio resampling: `pip install 'litellm[stt-nvidia-riva]'`"

4
litellm/llms/oci/chat/cohere.py
View file

@ -9,7 +9,7 @@ response parsing, and streaming chunk parsing for models served with
import datetime
import json
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, Final
from typing import Final
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -76,7 +76,7 @@ def _content_text(content: str | Iterable[Mapping[str, object]] | None) -> str:
return str(content)
def _extract_text_content(content: Any) -> str:
def _extract_text_content(content: str | Iterable[Mapping[str, object]] | None) -> str:
"""Return the plain-text representation of a message content value."""
return _content_text(content)

41
litellm/llms/oci/common_utils.py
View file

@ -5,10 +5,11 @@ import os
import re
from dataclasses import dataclass
from email.utils import formatdate
from typing import Any, Final, Protocol
from typing import Final, Protocol
from urllib.parse import urlparse
import httpx
from pydantic import JsonValue
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -64,7 +65,7 @@ class OCISignerProtocol(Protocol):
See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html
"""
def do_request_sign(self, request: Any, *, enforce_content_headers: bool = False) -> None:
def do_request_sign(self, request: "OCIRequestWrapper", *, enforce_content_headers: bool = False) -> None:
pass
@ -113,7 +114,7 @@ def build_signature_string(method: str, path: str, headers: dict, signed_headers
return "\n".join(lines)
def load_private_key_from_str(key_str: str) -> Any:
def load_private_key_from_str(key_str: str) -> "rsa.RSAPrivateKey":
_require_cryptography()
key: Final = serialization.load_pem_private_key(
key_str.encode("utf-8"),
@ -124,7 +125,7 @@ def load_private_key_from_str(key_str: str) -> Any:
return key
def load_private_key_from_file(file_path: str) -> Any:
def load_private_key_from_file(file_path: str) -> "rsa.RSAPrivateKey":
"""Loads a private key from a file path."""
try:
with open(file_path, "r", encoding="utf-8") as f:
@ -421,16 +422,17 @@ OCI_JSON_TO_PYTHON_TYPES: Final[dict[str, str]] = {
}
def resolve_oci_schema_refs(schema: dict[str, Any]) -> dict[str, Any]:
def resolve_oci_schema_refs(schema: JsonValue) -> JsonValue:
"""Inline all ``$ref``/``$defs`` references — OCI does not support JSON Schema ``$ref``."""
defs: Final = schema.get("$defs", {})
resolving_stack: Final[set] = set()
raw_defs: Final = schema.get("$defs") if isinstance(schema, dict) else None
defs: Final[dict[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else {}
resolving_stack: Final[set[str]] = set()
def _resolve(obj: Any) -> Any:
def _resolve(obj: JsonValue) -> JsonValue:
if isinstance(obj, dict):
if "$ref" in obj:
ref: Final = obj["$ref"]
if ref.startswith("#/$defs/"):
ref: Final = obj.get("$ref")
if ref is not None:
if isinstance(ref, str) and ref.startswith("#/$defs/"):
key: Final = ref.split("/")[-1]
if key in resolving_stack:
return {"type": "object"} # break cycles
@ -451,7 +453,7 @@ def resolve_oci_schema_refs(schema: dict[str, Any]) -> dict[str, Any]:
return resolved
def resolve_oci_schema_anyof(obj: Any) -> Any:
def resolve_oci_schema_anyof(obj: JsonValue) -> JsonValue:
"""Resolve Pydantic v2 ``Optional[T]`` → ``anyOf`` patterns.
Pydantic v2 emits ``{"anyOf": [{"type": "T"}, {"type": "null"}]}`` for
@ -459,10 +461,13 @@ def resolve_oci_schema_anyof(obj: Any) -> Any:
first non-null branch and merge top-level metadata into it.
"""
if isinstance(obj, dict):
if "anyOf" in obj and "type" not in obj:
non_null: Final = [t for t in obj["anyOf"] if not (isinstance(t, dict) and t.get("type") == "null")]
raw_any_of: Final = obj.get("anyOf")
if raw_any_of is not None and "type" not in obj:
branches: Final = raw_any_of if isinstance(raw_any_of, list) else []
non_null: Final = [t for t in branches if not (isinstance(t, dict) and t.get("type") == "null")]
if non_null:
resolved: Final = {**obj, **non_null[0]}
first: Final = non_null[0]
resolved: Final[dict[str, JsonValue]] = {**obj, **first} if isinstance(first, dict) else {**obj}
resolved.pop("anyOf", None)
return resolve_oci_schema_anyof(resolved)
return {k: resolve_oci_schema_anyof(v) for k, v in obj.items()}
@ -471,7 +476,7 @@ def resolve_oci_schema_anyof(obj: Any) -> Any:
return obj
def sanitize_oci_schema(schema: Any) -> Any:
def sanitize_oci_schema(schema: JsonValue) -> JsonValue:
"""Recursively remove OCI-incompatible fields from a JSON schema.
Strips ``title`` keys, removes ``None``-valued ``default`` entries,
@ -483,7 +488,7 @@ def sanitize_oci_schema(schema: Any) -> Any:
if not isinstance(schema, dict):
return schema
sanitized: Final[dict[str, Any]] = {}
sanitized: Final[dict[str, JsonValue]] = {}
for key, value in schema.items():
if key == "title":
continue
@ -513,7 +518,7 @@ def sanitize_oci_schema(schema: Any) -> Any:
return sanitized
def enrich_cohere_param_description(description: str, param_schema: dict[str, Any]) -> str:
def enrich_cohere_param_description(description: str, param_schema: dict[str, JsonValue]) -> str:
"""Embed schema constraints into a Cohere parameter description.
``CohereParameterDefinition`` only has ``type``, ``description``, and

22
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -170,16 +170,20 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if model != "gpt-3.5-turbo-16k" and model != "gpt-4": # gpt-4 does not support 'response_format'
model_specific_params.append("response_format")
# Normalize model name for responses API (e.g., "responses/gpt-4.1" -> "gpt-4.1")
model_for_check: Final = model.split("responses/", 1)[1] if "responses/" in model else model
if (
model_for_check in litellm.open_ai_chat_completion_models
) or model_for_check in litellm.open_ai_text_completion_models:
if OpenAIGPTConfig.is_openai_catalog_model(model):
model_specific_params.append(
"user"
) # user is not a param supported by all openai-compatible endpoints - e.g. azure ai
return base_params + model_specific_params
@staticmethod
def is_openai_catalog_model(model: str) -> bool:
model_for_check: Final = model.split("responses/", 1)[1] if "responses/" in model else model
return (
model_for_check in litellm.open_ai_chat_completion_models
or model_for_check in litellm.open_ai_text_completion_models
)
def _map_openai_params(
self,
non_default_params: dict,
@ -755,6 +759,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
)
class OpenAIUnknownModelConfig(OpenAIGPTConfig):
"""A model the openai provider does not recognize is typically a LiteLLM proxy alias, so
forward reasoning_effort and let the server decide whether it is supported."""
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract
return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract
class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
def _map_reasoning_to_reasoning_content(self, choices: list) -> list:
"""

1
litellm/llms/openai/common_utils.py
View file

@ -268,6 +268,7 @@ class BaseOpenAILLM:
"max_retries",
"organization",
"api_base",
"workload_identity_config",
)
openai_client_fields: Final = (
BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type=client_type)

30
litellm/llms/openai/containers/transformation.py
View file

@ -155,10 +155,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerObject:
"""Transform the OpenAI container creation response."""
response_data: Final[OpenAIContainerPayload] = raw_response.json()
# Transform the response data
container_obj: Final = ContainerObject(**response_data)
container_obj: Final = ContainerObject.model_validate(raw_response.json())
# Add cost for container creation (OpenAI containers are code interpreter sessions)
# https://platform.openai.com/docs/pricing
@ -215,10 +212,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerListResponse:
"""Transform the OpenAI container list response."""
response_data: Final[OpenAIContainerListPayload] = raw_response.json()
# Transform the response data
container_list: Final = ContainerListResponse(**response_data)
container_list: Final = ContainerListResponse.model_validate(raw_response.json())
return container_list
@ -235,7 +229,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No additional data needed for GET request
data: Final[dict[str, object]] = {}
data: Final[dict[str, str]] = {}
return url, data
@ -245,9 +239,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerObject:
"""Transform the OpenAI container retrieve response."""
response_data: Final[OpenAIContainerPayload] = raw_response.json()
# Transform the response data
container_obj: Final = ContainerObject(**response_data)
container_obj: Final = ContainerObject.model_validate(raw_response.json())
return container_obj
@ -268,7 +260,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No data needed for DELETE request
data: Final[dict[str, object]] = {}
data: Final[dict[str, str]] = {}
return url, data
@ -278,10 +270,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> DeleteContainerResult:
"""Transform the OpenAI container delete response."""
response_data: Final[OpenAIContainerDeletedPayload] = raw_response.json()
# Transform the response data
delete_result: Final = DeleteContainerResult(**response_data)
delete_result: Final = DeleteContainerResult.model_validate(raw_response.json())
return delete_result
@ -326,10 +315,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
logging_obj: LiteLLMLoggingObj,
) -> ContainerFileListResponse:
"""Transform the OpenAI container file list response."""
response_data: Final[OpenAIContainerFileListPayload] = raw_response.json()
# Transform the response data
file_list: Final = ContainerFileListResponse(**response_data)
file_list: Final = ContainerFileListResponse.model_validate(raw_response.json())
return file_list
@ -352,7 +338,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
url: Final = join_container_api_base_path(api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content")
# No query parameters needed
params: Final[dict[str, object]] = {}
params: Final[dict[str, str]] = {}
return url, params

71
litellm/llms/openai/openai.py
View file

@ -43,6 +43,7 @@ from litellm.utils import (
from ...types.llms.openai import *
from ..base import BaseLLM
from .chat.gpt_5_transformation import OpenAIGPT5Config
from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig
from .chat.o_series_transformation import OpenAIOSeriesConfig
from .common_utils import (
BaseOpenAILLM,
@ -51,6 +52,7 @@ from .common_utils import (
drop_params_from_unprocessable_entity_error,
is_output_token_limit_error,
)
from .workload_identity import resolve_openai_workload_identity_config
openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
openAIGPT5Config: Final = OpenAIGPT5Config()
@ -188,7 +190,12 @@ class OpenAIConfig(BaseConfig):
elif litellm.openAIGPTAudioConfig.is_model_gpt_audio_model(model=model):
return litellm.openAIGPTAudioConfig.get_supported_openai_params(model=model)
else:
return litellm.openAIGPTConfig.get_supported_openai_params(model=model)
return self._gpt_config_for_model(model).get_supported_openai_params(model=model)
def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig:
if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model):
return OpenAIUnknownModelConfig()
return litellm.openAIGPTConfig
def _map_openai_params(self, non_default_params: dict, optional_params: dict, model: str) -> dict:
supported_openai_params: Final = self.get_supported_openai_params(model)
@ -230,7 +237,7 @@ class OpenAIConfig(BaseConfig):
drop_params=drop_params,
)
return litellm.openAIGPTConfig.map_openai_params(
return self._gpt_config_for_model(model).map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
@ -349,6 +356,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
client: OpenAI | AsyncOpenAI | None = None,
shared_session: Optional["ClientSession"] = None,
) -> OpenAI | AsyncOpenAI | None:
workload_identity_config: Final = resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base)
client_initialization_params: Final[dict] = locals()
if client is None:
if not isinstance(max_retries, int):
@ -364,28 +372,49 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
if cached_client:
if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI):
return cached_client
http_client: Final[httpx.Client | httpx.AsyncClient | None] = (
OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
if is_async
else OpenAIChatCompletion._get_sync_http_client()
)
if is_async:
_new_client: OpenAI | AsyncOpenAI = AsyncOpenAI(
api_key=api_key,
base_url=api_base,
http_client=http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
async_http_client: Final = OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
http_client: httpx.Client | httpx.AsyncClient | None = async_http_client
_new_client: OpenAI | AsyncOpenAI = (
AsyncOpenAI(
workload_identity=workload_identity_config.to_sdk_workload_identity(),
base_url=api_base,
http_client=async_http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
)
if workload_identity_config is not None
else AsyncOpenAI(
api_key=api_key,
base_url=api_base,
http_client=async_http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
)
)
else:
_new_client = OpenAI(
api_key=api_key,
base_url=api_base,
http_client=http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
sync_http_client: Final = OpenAIChatCompletion._get_sync_http_client()
http_client = sync_http_client
_new_client = (
OpenAI(
workload_identity=workload_identity_config.to_sdk_workload_identity(),
base_url=api_base,
http_client=sync_http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
)
if workload_identity_config is not None
else OpenAI(
api_key=api_key,
base_url=api_base,
http_client=sync_http_client,
timeout=timeout,
max_retries=max_retries,
organization=organization,
)
)
## SAVE CACHE KEY

106
litellm/llms/openai/responses/count_tokens/transformation.py
View file

@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic.
This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
"""
from typing import Any, Final
from collections.abc import Mapping, Sequence
from typing import Any, Final, Literal
from typing_extensions import ReadOnly, TypedDict
class ResponsesInputTextPart(TypedDict):
type: ReadOnly[Literal["input_text"]]
text: ReadOnly[str]
class ResponsesInputImagePart(TypedDict):
type: ReadOnly[Literal["input_image"]]
image_url: ReadOnly[str]
detail: ReadOnly[str]
class ResponsesInputFilePart(TypedDict):
type: ReadOnly[Literal["input_file"]]
filename: ReadOnly[str]
file_data: ReadOnly[str]
ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart
ResponsesContentRole = Literal["user", "assistant"]
def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
if not isinstance(url, str) or not url:
return None
detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None
part: Final[ResponsesInputImagePart] = {
"type": "input_image",
"image_url": url,
"detail": detail if isinstance(detail, str) and detail else "auto",
}
return part
def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None:
"""Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it."""
if not isinstance(file_value, Mapping):
return None
filename: Final = file_value.get("filename")
file_data: Final = file_value.get("file_data")
if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data:
return None
part: Final[ResponsesInputFilePart] = {
"type": "input_file",
"filename": filename,
"file_data": file_data,
}
return part
def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
if isinstance(block, str):
bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
return bare
if not isinstance(block, Mapping):
return None
match block.get("type"):
case "text":
text_value: Final = block.get("text")
text: Final[ResponsesInputTextPart] = {
"type": "input_text",
"text": text_value if isinstance(text_value, str) else "",
}
return text
case "image_url" if role == "user":
return _chat_image_block_to_responses_part(block.get("image_url"))
case "file" if role == "user":
return _chat_file_block_to_responses_part(block.get("file"))
case _:
return None
def chat_content_blocks_to_responses_content(
content: Sequence[object],
role: ResponsesContentRole,
) -> str | tuple[ResponsesInputPart, ...]:
"""Text-only content collapses to a joined string, which every role accepts and counts identically.
Only a user turn may carry an image or file part: the Responses API rejects any part but
output_text and refusal inside an assistant turn.
"""
parts: Final = tuple(
part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None
)
if any(part["type"] != "input_text" for part in parts):
return parts
return "\n".join(part["text"] for part in parts if part["type"] == "input_text")
class OpenAICountTokensConfig:
@ -120,18 +213,13 @@ class OpenAICountTokensConfig:
instructions_parts.append("\n".join(text_parts))
elif role == "user":
if isinstance(content, list):
# Extract text from content blocks for Responses API
text_parts = []
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text_parts.append(block.get("text", ""))
elif isinstance(block, str):
text_parts.append(block)
content = "\n".join(text_parts)
content = chat_content_blocks_to_responses_content(content, "user")
input_items.append({"role": "user", "content": content})
elif role == "assistant":
# Map tool_calls to Responses API function_call items
tool_calls = msg.get("tool_calls")
if isinstance(content, list):
content = chat_content_blocks_to_responses_content(content, "assistant")
if content:
input_items.append({"role": "assistant", "content": content})
if tool_calls:

9
litellm/llms/openai/responses/transformation.py
View file

@ -21,6 +21,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..common_utils import OpenAIError
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -392,6 +393,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
headers.setdefault("Content-Type", "application/json")
workload_identity_config: Final = (
resolve_openai_workload_identity_config(api_key=api_key, api_base=litellm_params.api_base)
if self.custom_llm_provider is LlmProviders.OPENAI
else None
)
if workload_identity_config is not None:
headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}"
return headers
headers["Authorization"] = f"Bearer {api_key}"
return headers

100
litellm/llms/openai/workload_identity.py Normal file
View file

@ -0,0 +1,100 @@
from __future__ import annotations
from dataclasses import dataclass
from functools import lru_cache
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
import litellm
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
from .common_utils import OpenAIError
if TYPE_CHECKING:
from collections.abc import Callable
from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
OPENAI_WIF_CLIENT_ID: Final = "litellm"
_OPENAI_API_HOST: Final = "api.openai.com"
_SDK_UPGRADE_MESSAGE: Final = (
"OpenAI workload identity federation requires openai>=2.32.0. "
"Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
"OPENAI_SERVICE_ACCOUNT_ID / OPENAI_IDENTITY_TOKEN_FILE."
)
@dataclass(frozen=True, slots=True)
class OpenAIWorkloadIdentityConfig:
identity_provider_id: str
service_account_id: str
token_file: str
def to_sdk_workload_identity(self) -> WorkloadIdentity:
k8s_token_provider: Final = _load_sdk_k8s_token_provider()
workload_identity: Final[WorkloadIdentity] = {
"client_id": OPENAI_WIF_CLIENT_ID,
"identity_provider_id": self.identity_provider_id,
"service_account_id": self.service_account_id,
"provider": k8s_token_provider(self.token_file),
}
return workload_identity
def resolve_openai_workload_identity_config(
api_key: str | None,
api_base: str | None,
) -> OpenAIWorkloadIdentityConfig | None:
static_api_key: Final = normalize_nonempty_secret_str(api_key) or normalize_nonempty_secret_str(
get_secret_str("OPENAI_API_KEY")
)
if static_api_key is not None:
return None
effective_api_base: Final = (
api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
)
if not _targets_openai_api(effective_api_base):
return None
identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID")
service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID")
token_file: Final = get_secret_str("OPENAI_IDENTITY_TOKEN_FILE")
if not identity_provider_id or not service_account_id or not token_file:
return None
return OpenAIWorkloadIdentityConfig(
identity_provider_id=identity_provider_id,
service_account_id=service_account_id,
token_file=token_file,
)
def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) -> str:
return _workload_identity_auth(config).get_token()
def _targets_openai_api(api_base: str | None) -> bool:
if api_base is None:
return True
parsed: Final = urlparse(api_base)
return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST
@lru_cache(maxsize=16)
def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())
def _load_sdk_workload_identity_auth() -> type[WorkloadIdentityAuth]:
try:
from openai.auth import WorkloadIdentityAuth as sdk_workload_identity_auth
except ImportError as e:
raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
return sdk_workload_identity_auth
def _load_sdk_k8s_token_provider() -> Callable[[str], SubjectTokenProvider]:
try:
from openai.auth import k8s_service_account_token_provider
except ImportError as e:
raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
return k8s_service_account_token_provider

31
litellm/llms/runwayml/text_to_speech/transformation.py
View file

@ -6,10 +6,11 @@ Maps OpenAI TTS spec to RunwayML Text-to-Speech API
import asyncio
import time
from collections.abc import Coroutine
from typing import TYPE_CHECKING, Any, Final, Union
from collections.abc import Coroutine, Sequence
from typing import TYPE_CHECKING, Any, Final, TypedDict, Union
import httpx
from typing_extensions import ReadOnly
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,14 @@ else:
HttpxBinaryResponseContent = Any
class _RunwayTtsTaskResponse(TypedDict, total=False):
id: ReadOnly[str]
status: ReadOnly[str]
output: ReadOnly[Sequence[object]]
failure: ReadOnly[str]
failureCode: ReadOnly[str]
class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
"""
Configuration for RunwayML Text-to-Speech
@ -64,7 +73,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
litellm_params_dict: dict,
logging_obj: "LiteLLMLoggingObj",
timeout: float | httpx.Timeout,
extra_headers: dict[str, Any] | None,
extra_headers: dict[str, object] | None,
base_llm_http_handler: Any,
aspeech: bool,
api_base: str | None,
@ -72,7 +81,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
**kwargs: Any,
) -> Union[
"HttpxBinaryResponseContent",
Coroutine[Any, Any, "HttpxBinaryResponseContent"],
Coroutine[object, object, "HttpxBinaryResponseContent"],
]:
"""
Dispatch method to handle RunwayML TTS requests
@ -242,7 +251,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
raise TimeoutError(f"RunwayML TTS task polling timed out after {timeout_secs} seconds")
@staticmethod
def _check_task_status(response_data: dict[str, Any]) -> str:
def _check_task_status(response_data: _RunwayTtsTaskResponse) -> str:
"""
Check RunwayML task status from response.
@ -314,7 +323,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
response = client.get(url=task_url, headers=headers)
response.raise_for_status()
response_data = response.json()
response_data: _RunwayTtsTaskResponse = response.json()
# Check task status
status = self._check_task_status(response_data=response_data)
@ -362,7 +371,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
response = await client.get(url=task_url, headers=headers)
response.raise_for_status()
response_data = response.json()
response_data: _RunwayTtsTaskResponse = response.json()
# Check task status
status = self._check_task_status(response_data=response_data)
@ -453,7 +462,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
response_data: Final = raw_response.json()
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing RunwayML TTS response: {e}",
@ -483,7 +492,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
)
# Get the completed task data
task_data: Final = polled_response.json()
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
verbose_logger.debug("RunwayML TTS polling complete, downloading audio")
@ -522,7 +531,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
response_data: Final = raw_response.json()
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing RunwayML TTS response: {e}",
@ -552,7 +561,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
)
# Get the completed task data
task_data: Final = polled_response.json()
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio")

2
litellm/llms/runwayml/videos/transformation.py
View file

@ -160,14 +160,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
**self._prompt_image_param(video_create_optional_params),
**self._ratio_param(video_create_optional_params),
**self._duration_param(video_create_optional_params),
# Pass through other parameters that aren't OpenAI-specific
**{key: value for key, value in video_create_optional_params.items() if key not in supported_openai_params},
}
@staticmethod
def _prompt_image_param(video_create_optional_params: VideoCreateOptionalRequestParams) -> Mapping[str, object]:
# Handle input_reference parameter - map to promptImage
# RunwayML supports URLs and data URIs directly
if "input_reference" in video_create_optional_params:
return {"promptImage": video_create_optional_params["input_reference"]}
return {}

45
litellm/llms/vertex_ai/common_utils.py
View file

@ -1,7 +1,7 @@
import re
from copy import deepcopy
from enum import Enum
from typing import Any, Final, Literal, get_type_hints
from typing import Any, Final, Literal, cast, get_type_hints
import httpx
@ -31,7 +31,7 @@ class VertexAIError(BaseLLMException):
super().__init__(message=message, status_code=status_code, headers=headers)
def redact_vertex_ai_metadata_from_logged_object(obj: Any) -> None:
def redact_vertex_ai_metadata_from_logged_object(obj: object) -> None:
if isinstance(obj, dict):
for field in VERTEX_AI_PROVIDER_METADATA_FIELDS:
if field in obj:
@ -651,7 +651,7 @@ def _build_json_schema(parameters: dict) -> dict:
return parameters
def _filter_anyof_fields(schema_dict: dict[str, Any]) -> dict[str, Any]:
def _filter_anyof_fields(schema_dict: dict[str, object]) -> dict[str, object]:
"""
When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164
Filter out other fields in the same dict.
@ -704,7 +704,7 @@ def process_items(schema, depth=0):
process_items(item, depth + 1)
def set_schema_property_ordering(schema: dict[str, Any], depth: int = 0) -> dict[str, Any]:
def set_schema_property_ordering(schema: dict[str, object], depth: int = 0) -> dict[str, object]:
"""
vertex ai and generativeai apis order output of fields alphabetically, unless you specify the order.
python dicts retain order, so we just use that. Note that this field only applies to structured outputs, and not tools.
@ -724,14 +724,16 @@ def set_schema_property_ordering(schema: dict[str, Any], depth: int = 0) -> dict
# retain propertyOrdering as an escape hatch if user already specifies it
if "propertyOrdering" not in schema:
schema["propertyOrdering"] = [k for k, v in schema["properties"].items()]
for k, v in schema["properties"].items():
set_schema_property_ordering(v, depth + 1)
if "items" in schema:
set_schema_property_ordering(schema["items"], depth + 1)
for v in schema["properties"].values():
if isinstance(v, dict):
set_schema_property_ordering(cast("dict[str, object]", v), depth + 1) # cast-ok: JSON Schema child
items: Final = schema.get("items")
if isinstance(items, dict):
set_schema_property_ordering(cast("dict[str, object]", items), depth + 1) # cast-ok: JSON Schema child
return schema
def filter_schema_fields(schema_dict: dict[str, Any], valid_fields: set[str], processed=None) -> dict[str, Any]:
def filter_schema_fields(schema_dict: dict[str, object], valid_fields: set[str], processed=None) -> dict[str, object]:
"""
Recursively filter a schema dictionary to keep only valid fields.
"""
@ -905,7 +907,7 @@ def _convert_schema_types(schema, depth=0):
"maxProperties",
}
any_of: Final[list[dict[str, Any]]] = []
any_of: Final[list[dict[str, object]]] = []
for t in type_val:
if not isinstance(t, str):
continue
@ -916,7 +918,7 @@ def _convert_schema_types(schema, depth=0):
# For object/array types, include type-specific fields
if t in ("object", "array"):
item_schema = {"type": t}
item_schema: dict[str, object] = {"type": t}
# Move type-specific fields into this anyOf item
for field in type_specific_fields:
if field in schema:
@ -1110,11 +1112,11 @@ class VertexAITokenCounter(BaseTokenCounter):
self,
model_to_use: str,
messages: list[dict[str, Any]] | None,
contents: list[dict[str, Any]] | None,
contents: list[dict[str, object]] | None,
deployment: dict[str, Any] | None = None,
request_model: str = "",
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
tools: list[dict[str, object]] | None = None,
system: object | None = None,
) -> TokenCountResponse | None:
import copy
@ -1131,25 +1133,26 @@ class VertexAITokenCounter(BaseTokenCounter):
partner_models_handler: Final = VertexAIPartnerModels()
# Extract vertex-specific params from litellm_params
vertex_project = count_tokens_params_request.get("vertex_project") or count_tokens_params_request.get(
partner_litellm_params: Final[dict[str, object]] = count_tokens_params_request
vertex_project = partner_litellm_params.get("vertex_project") or partner_litellm_params.get(
"vertex_ai_project"
)
vertex_location = count_tokens_params_request.get("vertex_location") or count_tokens_params_request.get(
vertex_location = partner_litellm_params.get("vertex_location") or partner_litellm_params.get(
"vertex_ai_location"
)
# Count tokens not available on global location: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
vertex_location = count_tokens_params_request.get("vertex_count_tokens_location") or vertex_location
vertex_location = partner_litellm_params.get("vertex_count_tokens_location") or vertex_location
vertex_credentials: Final = count_tokens_params_request.get(
"vertex_credentials"
) or count_tokens_params_request.get("vertex_ai_credentials")
vertex_credentials: Final = partner_litellm_params.get("vertex_credentials") or partner_litellm_params.get(
"vertex_ai_credentials"
)
result = await partner_models_handler.count_tokens(
model=model_to_use,
messages=messages or [],
litellm_params=count_tokens_params_request,
litellm_params=partner_litellm_params,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,

1
litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
View file

@ -182,7 +182,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
else None
)
# Generation config with proper structure for image editing
generation_config: Final[dict[str, object]] = {
key: value for key, value in (("response_modalities", ["IMAGE"]), ("image_config", image_config)) if value
}

9
litellm/llms/vertex_ai/text_to_speech/transformation.py
View file

@ -7,10 +7,14 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s
import base64
from collections.abc import Coroutine
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from litellm.litellm_core_utils.audio_utils.utils import (
speech_media_type_from_audio_bytes,
)
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
@ -457,12 +461,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
if not response_content:
raise ValueError("No audioContent in Vertex AI TTS response")
# Decode base64 to get binary content
binary_data: Final = base64.b64decode(response_content)
# Create an httpx.Response object with the binary data
media_type: Final = speech_media_type_from_audio_bytes(binary_data)
response: Final = httpx.Response(
status_code=200,
headers=None if media_type is None else MappingProxyType({"content-type": media_type}),
content=binary_data,
)

2
litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
View file

@ -203,7 +203,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
if value is not None
}
# Build the request body for Vertex AI RAG API
query_body: Final[Mapping[str, object]] = {
key: value
for key, value in (("text", query), ("rag_retrieval_config", rag_retrieval_config or None))
@ -294,7 +293,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
# Add metadata if provided
metadata: Final = vector_store_create_optional_params.get("metadata")
# Build the request body for Vertex AI RAG Corpus creation
request_body: Final[dict[str, object]] = {
key: value
for key, value in (

20
litellm/llms/vertex_ai/vertex_llm_base.py
View file

@ -27,6 +27,15 @@ from .common_utils import (
get_vertex_base_url,
)
def _graft_default_vertex_path(api_base: str, default_url: str) -> str:
parsed_api_base: Final = urlparse(api_base)
default_segments: Final = urlparse(default_url).path.lstrip("/").split("/")
graft_segments: Final = default_segments[1:] if default_segments[0] in ("v1", "v1beta1") else default_segments
grafted_path: Final = parsed_api_base.path.rstrip("/") + "/" + "/".join(graft_segments)
return parsed_api_base._replace(path=grafted_path).geturl()
GOOGLE_IMPORT_ERROR_MESSAGE: Final = (
"Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform"
)
@ -621,8 +630,9 @@ class VertexBase:
Handles custom api_base for:
1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
2. Vertex AI with standard proxies - constructs {api_base}:{endpoint};
if api_base has no path (bare host), grafts the default vertex URL path onto it
2. Vertex AI with standard proxies - grafts the default vertex URL path onto the
api_base when its path is empty or only an API version (/v1, /v1beta1);
otherwise constructs {api_base}:{endpoint}
3. Vertex AI with PSC endpoints - constructs full path structure
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
(only when use_psc_endpoint_format=True)
@ -669,10 +679,14 @@ class VertexBase:
)
elif urlparse(api_base).path in ("", "/"):
url = api_base.rstrip("/") + urlparse(url).path
elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path:
url = _graft_default_vertex_path(api_base=api_base, default_url=url)
else:
url = f"{api_base}:{endpoint}"
if stream is True:
url = url + "?alt=sse"
parsed_stream_url: Final = urlparse(url)
stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse"
url = parsed_stream_url._replace(query=stream_query).geturl()
return auth_header, url
def _get_token_and_url(

3
litellm/main.py
View file

@ -5507,6 +5507,9 @@ def completion(
tpm=kwargs.get("tpm"),
rpm=kwargs.get("rpm"),
use_xai_oauth=kwargs.get("use_xai_oauth", False),
gigachat_scope=kwargs.get("gigachat_scope"),
gigachat_auth_url=kwargs.get("gigachat_auth_url"),
gigachat_access_token=kwargs.get("gigachat_access_token"),
**{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs},
)
cast(LiteLLMLoggingObj, logging).update_environment_variables(

86
litellm/model_prices_and_context_window_backup.json
View file

@ -553,6 +553,26 @@
"supports_response_schema": true,
"supports_vision": true
},
"amazon.nova-sonic-v1:0": {
"input_cost_per_audio_token": 3.4e-06,
"input_cost_per_token": 6e-08,
"litellm_provider": "bedrock",
"mode": "realtime",
"output_cost_per_audio_token": 1.36e-05,
"output_cost_per_token": 2.4e-07,
"supports_audio_input": true,
"supports_audio_output": true
},
"amazon.nova-2-sonic-v1:0": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 3.3e-07,
"litellm_provider": "bedrock",
"mode": "realtime",
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_token": 2.75e-06,
"supports_audio_input": true,
"supports_audio_output": true
},
"amazon.rerank-v1:0": {
"input_cost_per_query": 0.001,
"input_cost_per_token": 0.0,
@ -19566,6 +19586,61 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"friendliai/zai-org/GLM-5.3-Flash": {
"litellm_provider": "friendliai",
"supports_reasoning": true,
"supports_function_calling": true,
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"max_output_tokens": 1048576,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 3e-08,
"supports_prompt_caching": true,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"mode": "chat",
"comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks",
"source": "https://api.friendli.ai/serverless/v1/models",
"supports_vision": true,
"supports_image_input": true,
"supports_video_input": true
},
"friendliai/zai-org/GLM-5.3": {
"litellm_provider": "friendliai",
"supports_reasoning": true,
"supports_function_calling": true,
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"max_output_tokens": 1048576,
"input_cost_per_token": 1.26e-06,
"output_cost_per_token": 3.96e-06,
"cache_read_input_token_cost": 2.34e-07,
"supports_prompt_caching": true,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"mode": "chat",
"comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery",
"source": "https://api.friendli.ai/serverless/v1/models",
"supports_vision": false,
"supports_image_input": false
},
"ft:babbage-002": {
"deprecation_date": "2026-10-23",
"input_cost_per_token": 1.6e-06,
@ -24350,7 +24425,7 @@
"supports_response_schema": true,
"supports_vision": true
},
"gigachat/GigaChat-2-Lite": {
"gigachat/GigaChat-2": {
"input_cost_per_token": 0.0,
"litellm_provider": "gigachat",
"max_input_tokens": 128000,
@ -24412,6 +24487,15 @@
"output_cost_per_token": 0.0,
"output_vector_size": 2560
},
"gigachat/GigaEmbeddings-3B-2025-09": {
"input_cost_per_token": 0.0,
"litellm_provider": "gigachat",
"max_input_tokens": 4096,
"max_tokens": 4096,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 2048
},
"gmi/anthropic/claude-opus-4.5": {
"input_cost_per_token": 5e-06,
"litellm_provider": "gmi",

2
litellm/models/model.py
View file

@ -29,6 +29,8 @@ class LiteLLM_ProxyModelTable(LiteLLMPydanticObjectBase):
@model_validator(mode="before")
@classmethod
def check_potential_json_str(cls, values):
if not isinstance(values, dict):
return values
if isinstance(values.get("litellm_params"), str):
try:
values["litellm_params"] = json.loads(values["litellm_params"])

372
litellm/passthrough/main.py
View file

@ -2,17 +2,22 @@
This module is used to pass through requests to the LLM APIs.
"""
from __future__ import annotations
import asyncio
import contextvars
from collections.abc import AsyncGenerator, Coroutine, Generator
from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Generator, Iterator
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from types import TracebackType
from typing import Any, Final, cast
import httpx
from httpx._types import CookieTypes, QueryParamTypes, RequestFiles
from httpx._types import CookieTypes, QueryParamTypes, RequestContent, RequestFiles
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.passthrough.utils import CommonUtils
@ -21,9 +26,222 @@ from litellm.utils import client
base_llm_http_handler = BaseLLMHTTPHandler()
from .utils import BasePassthroughUtils
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
async def _as_async_generator(iterable: AsyncIterator[bytes]) -> AsyncGenerator[bytes, Any]:
async for chunk in iterable:
yield chunk
def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, Any, Any]:
yield from iterable
class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]):
def __init__(
self,
response: Coroutine[Any, Any, httpx.Response],
litellm_logging_obj: LiteLLMLoggingObj,
provider_config: BasePassthroughConfig,
) -> None:
self._initialized = False
self._status_code: int = 0
self._headers = httpx.Headers()
self._response_coro = response
self._response: httpx.Response
self._iterator: AsyncGenerator[bytes, Any]
self._litellm_logging_obj = litellm_logging_obj
self._provider_config = provider_config
self._raw_bytes: list[bytes] = [] # mutable-ok: instance buffer for streaming chunks
self._flush_scheduled = False
self._background_tasks: set[asyncio.Task] = set() # mutable-ok: instance set for background task tracking
self._hidden_params: dict[str, object] = {} # mutable-ok: router attaches response headers here in place
@property
def status_code(self) -> int:
if not self._initialized:
raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing status_code")
return self._status_code
@status_code.setter
def status_code(self, value: int) -> None:
self._status_code = value
@property
def headers(self) -> httpx.Headers:
if not self._initialized:
raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing headers")
return self._headers
@headers.setter
def headers(self, value: httpx.Headers) -> None:
self._headers = value
def __await__(self) -> Iterator[Any]:
async def _init():
if not self._initialized:
self._response = await self._response_coro
self.headers = self._response.headers
self.status_code = self._response.status_code
self._initialized = True
try:
self._response.raise_for_status()
self._iterator = _as_async_generator(self._response.aiter_bytes())
except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic
try:
await self._response.aread()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
try:
await self._response.aclose()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
raise
return self
return _init().__await__()
def _start_flush(self) -> None:
if self._flush_scheduled or not self._raw_bytes:
return
self._flush_scheduled = True
try:
task: Final = asyncio.create_task(
self._litellm_logging_obj.async_flush_passthrough_collected_chunks(
raw_bytes=self._raw_bytes,
provider_config=self._provider_config,
)
)
# Compliant: Save a strong reference to prevent GC
self._background_tasks.add(task)
# Remove the task from the set when it finishes to avoid memory leaks
task.add_done_callback(self._background_tasks.discard)
except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging
verbose_logger.exception(
"Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s",
len(self._raw_bytes),
e,
)
def __aiter__(self) -> AsyncPassthroughStreamingResponse:
return self
def aiter_bytes(self) -> AsyncPassthroughStreamingResponse:
return self
async def __anext__(self) -> bytes:
if not self._initialized:
await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__
try:
chunk: Final = await anext(self._iterator)
self._raw_bytes.append(chunk)
except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic
self._start_flush()
try:
await self._response.aclose()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
raise
else:
return chunk
async def asend(self, value: bytes) -> bytes:
if not self._initialized:
await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__
return await self._iterator.asend(value)
async def athrow(
self,
typ: BaseException | type[BaseException],
val: BaseException | object = None,
tb: TracebackType | None = None,
) -> bytes:
if not self._initialized:
await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__
return await self._iterator.athrow(typ, val, tb) # pyright: ignore[reportCallIssue, reportArgumentType] # matches one of the athrow overloads
async def aclose(self) -> None:
self._start_flush()
try:
if self._initialized:
await self._iterator.aclose()
await self._response.aclose()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
class PassthroughStreamingResponse(Generator[Any, Any, Any]):
def __init__(
self,
response: httpx.Response,
litellm_logging_obj: LiteLLMLoggingObj,
provider_config: BasePassthroughConfig,
) -> None:
self._response = response
self.headers = response.headers
self.status_code = response.status_code
self._litellm_logging_obj = litellm_logging_obj
self._provider_config = provider_config
self._iterator: Generator[bytes, Any, Any] = _as_generator(response.iter_bytes())
self._raw_bytes: list[bytes] = [] # mutable-ok: instance buffer for streaming chunks
self._flush_scheduled = False
def _start_flush(self) -> None:
if self._flush_scheduled or not self._raw_bytes:
return
self._flush_scheduled = True
from litellm.utils import executor
try:
executor.submit(
self._litellm_logging_obj.flush_passthrough_collected_chunks,
raw_bytes=self._raw_bytes,
provider_config=self._provider_config,
)
except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging
verbose_logger.exception(
"Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s",
len(self._raw_bytes),
e,
)
def __iter__(self) -> PassthroughStreamingResponse:
return self
def __next__(self) -> bytes:
try:
chunk: Final = next(self._iterator)
self._raw_bytes.append(chunk)
except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic
self._start_flush()
try:
self._response.close()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
raise
else:
return chunk
def send(self, value: bytes) -> bytes:
return self._iterator.send(value)
def throw(
self,
typ: BaseException | type[BaseException],
val: BaseException | object = None,
tb: TracebackType | None = None,
) -> bytes:
return self._iterator.throw(typ, val, tb) # pyright: ignore[reportCallIssue, reportArgumentType] # matches one of the throw overloads
def close(self) -> None:
self._start_flush()
try:
self._response.close()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
@client
@ -37,10 +255,10 @@ async def allm_passthrough_route(
api_key: str | None = None,
request_query_params: dict | None = None,
request_headers: dict | None = None,
content: Any | None = None,
content: RequestContent | None = None,
data: dict | None = None,
files: RequestFiles | None = None,
json: Any | None = None,
json: object | None = None,
params: QueryParamTypes | None = None,
cookies: CookieTypes | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
@ -64,7 +282,7 @@ async def allm_passthrough_route(
from litellm.utils import ProviderConfigManager
provider_config = cast(
Optional["BasePassthroughConfig"], kwargs.get("provider_config")
BasePassthroughConfig | None, kwargs.get("provider_config")
) or ProviderConfigManager.get_provider_passthrough_config(
provider=LlmProviders(custom_llm_provider),
model=model,
@ -132,12 +350,12 @@ async def allm_passthrough_route(
if resolved_custom_llm_provider:
try:
provider_config = cast(
Optional["BasePassthroughConfig"], kwargs.get("provider_config")
BasePassthroughConfig | None, kwargs.get("provider_config")
) or ProviderConfigManager.get_provider_passthrough_config(
provider=LlmProviders(resolved_custom_llm_provider),
model=model,
)
except Exception:
except Exception: # noqa: BLE001 S110
# If we can't get provider config, pass None
pass
@ -162,10 +380,10 @@ def llm_passthrough_route(
api_key: str | None = None,
request_query_params: dict | None = None,
request_headers: dict | None = None,
content: Any | None = None,
content: RequestContent | None = None,
data: dict | None = None,
files: RequestFiles | None = None,
json: Any | None = None,
json: object | None = None,
params: QueryParamTypes | None = None,
cookies: CookieTypes | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
@ -190,7 +408,9 @@ def llm_passthrough_route(
_is_async: Final = bool(kwargs.get("allm_passthrough_route", False))
litellm_logging_obj: Final = cast("LiteLLMLoggingObj", kwargs.get("litellm_logging_obj"))
litellm_logging_obj: Final = cast(
LiteLLMLoggingObj, kwargs.get("litellm_logging_obj")
) # cast-ok: logging obj is constructed upstream; tests inject mocks
model, custom_llm_provider, api_key, api_base = get_llm_provider(
model=model,
@ -235,7 +455,7 @@ def llm_passthrough_route(
)
provider_config: Final = cast(
Optional["BasePassthroughConfig"], kwargs.get("provider_config")
BasePassthroughConfig | None, kwargs.get("provider_config")
) or ProviderConfigManager.get_provider_passthrough_config(
provider=LlmProviders(custom_llm_provider),
model=model,
@ -276,10 +496,13 @@ def llm_passthrough_route(
forward_headers=False,
)
_request_data: dict | None = (
data if isinstance(data, dict) else (json if isinstance(json, dict) else None)
) # rebind-ok: conditional
headers, signed_json_body = provider_config.sign_request(
headers=headers,
litellm_params=litellm_params_dict,
request_data=data if data else json,
request_data=_request_data,
api_base=str(updated_url),
model=model,
)
@ -301,9 +524,12 @@ def llm_passthrough_route(
)
## IS STREAMING REQUEST
_streaming_request_data: dict = (
data if isinstance(data, dict) else (json if isinstance(json, dict) else {})
) # rebind-ok: conditional
is_streaming_request: Final = provider_config.is_streaming_request(
endpoint=endpoint,
request_data=data or json or {},
request_data=_streaming_request_data,
)
# Update logging object with streaming status
@ -334,18 +560,26 @@ def llm_passthrough_route(
else:
# Sync path - client.client.send returns Response directly
response: httpx.Response = client.client.send(request=request, stream=is_streaming_request)
response.raise_for_status()
try:
response.raise_for_status()
except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic
try:
response.read()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
try:
response.close()
except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic
pass
raise
if (
hasattr(response, "iter_bytes") and is_streaming_request
): # yield the chunk, so we can store it in the logging object
return _sync_streaming(response, litellm_logging_obj, provider_config)
if hasattr(response, "iter_bytes") and is_streaming_request:
return PassthroughStreamingResponse(response, litellm_logging_obj, provider_config)
else:
# For non-streaming responses, yield the entire response
return response
except Exception as e:
if provider_config is None:
raise e
# provider_config is guaranteed non-None here due to the earlier guard
assert provider_config is not None
raise base_llm_http_handler._handle_error(
e=e,
provider_config=provider_config,
@ -356,8 +590,8 @@ async def _async_passthrough_request(
client: HTTPHandler | AsyncHTTPHandler,
request: httpx.Request,
is_streaming_request: bool,
litellm_logging_obj: "LiteLLMLoggingObj",
provider_config: "BasePassthroughConfig",
litellm_logging_obj: LiteLLMLoggingObj,
provider_config: BasePassthroughConfig,
) -> httpx.Response | AsyncGenerator[Any, Any]:
"""
Handle async passthrough requests.
@ -369,8 +603,7 @@ async def _async_passthrough_request(
# Check if it's a coroutine and await it
if asyncio.iscoroutine(response_result):
if is_streaming_request:
# Pass the coroutine to _async_streaming which will await it
return _async_streaming(
return await AsyncPassthroughStreamingResponse( # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__
response=response_result,
litellm_logging_obj=litellm_logging_obj,
provider_config=provider_config,
@ -383,84 +616,3 @@ async def _async_passthrough_request(
else:
# Fallback for sync-like behavior (shouldn't happen in async path)
raise Exception("Expected coroutine from async client")
def _sync_streaming(
response: httpx.Response,
litellm_logging_obj: "LiteLLMLoggingObj",
provider_config: "BasePassthroughConfig",
):
from litellm.utils import executor
raw_bytes: Final[list[bytes]] = []
flush_scheduled = False
try:
for chunk in response.iter_bytes():
raw_bytes.append(chunk)
yield chunk
finally:
if not flush_scheduled and raw_bytes:
flush_scheduled = True
try:
executor.submit(
litellm_logging_obj.flush_passthrough_collected_chunks,
raw_bytes=raw_bytes,
provider_config=provider_config,
)
except Exception as e:
verbose_logger.exception(
"Failed to schedule passthrough spend-tracking flush "
"in _sync_streaming; %d buffered chunks dropped: %s",
len(raw_bytes),
e,
)
async def _async_streaming(
response: Coroutine[Any, Any, httpx.Response],
litellm_logging_obj: "LiteLLMLoggingObj",
provider_config: "BasePassthroughConfig",
):
iter_response: Final = await response
try:
iter_response.raise_for_status()
except Exception:
try:
await iter_response.aclose()
except Exception:
pass
raise
raw_bytes: Final[list[bytes]] = []
flush_scheduled = False
try:
async for chunk in iter_response.aiter_bytes():
raw_bytes.append(chunk)
yield chunk
except Exception:
try:
await iter_response.aclose()
except Exception:
pass
raise
finally:
# GeneratorExit (raised on client disconnect) is not caught by
# `except Exception`; the finally block ensures partial usage
# still gets flushed for spend tracking. See LIT-2642.
if not flush_scheduled and raw_bytes:
flush_scheduled = True
try:
asyncio.create_task(
litellm_logging_obj.async_flush_passthrough_collected_chunks(
raw_bytes=raw_bytes,
provider_config=provider_config,
)
)
except Exception as e:
verbose_logger.exception(
"Failed to schedule passthrough spend-tracking flush "
"in _async_streaming; %d buffered chunks dropped: %s",
len(raw_bytes),
e,
)

14
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -13,7 +13,7 @@ import json
import os
import re
import time
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast
@ -1206,7 +1206,7 @@ def _deserialize_json_dict(data: str | _StringMap | None) -> dict[str, str] | No
return data
def _deserialize_json_list(data: Any) -> list[dict[str, Any]] | None:
def _deserialize_json_list(data: object) -> list[dict[str, Any]] | None:
"""Deserialize a JSON array stored in the DB (``env_vars`` and friends).
Returns ``None`` for empty / null / unparseable input. Accepts strings
@ -1219,7 +1219,7 @@ def _deserialize_json_list(data: Any) -> list[dict[str, Any]] | None:
return None
if isinstance(data, str):
try:
parsed: Final = json.loads(data)
parsed: Final[object] = json.loads(data)
except (json.JSONDecodeError, TypeError):
return None
data = parsed
@ -1914,7 +1914,7 @@ class MCPServerManager:
async def load_servers_from_config(
self,
mcp_servers_config: dict[str, Any],
mcp_servers_config: dict[str, MCPServerConfig],
mcp_aliases: dict[str, str] | None = None,
):
"""
@ -3068,7 +3068,7 @@ class MCPServerManager:
return {}
cache_key: Final = "toolset_perms:" + ",".join(sorted(toolset_ids))
cached: Final = await user_api_key_cache.async_get_cache(key=cache_key)
cached: Final[dict[str, list[str]] | None] = await user_api_key_cache.async_get_cache(key=cache_key)
if cached is not None:
return cached
@ -5154,7 +5154,7 @@ class MCPServerManager:
# Wrapped so the bridge runs inside the task: the caller only holds the task and
# gathers it later, so there is no other point that still sees a block here.
async def _run_during_call_hook() -> Mapping[str, Any] | None:
async def _run_during_call_hook() -> Mapping[str, object] | None:
try:
return await proxy_logging_obj.during_call_hook(
user_api_key_dict=user_api_key_auth,
@ -5656,7 +5656,7 @@ class MCPServerManager:
async def _gather_openapi_tool_tasks(
self,
tasks: list[Any],
tasks: Sequence[Awaitable[object]],
proxy_logging_obj: ProxyLogging | None,
) -> CallToolResult:
"""Await OpenAPI tool tasks and return the tool call result."""

18
litellm/proxy/_types.py
View file

@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum):
"/responses/{response_id}/cancel",
"/v1/responses/{response_id}/cancel",
"/openai/v1/responses/{response_id}/cancel",
"/responses/input_tokens",
"/v1/responses/input_tokens",
"/openai/v1/responses/input_tokens",
# vector stores
"/vector_stores",
"/v1/vector_stores",
@ -471,6 +474,7 @@ class LiteLLMRoutes(enum.Enum):
"/vllm",
"/mistral",
"/milvus",
"/gigachat",
"/watsonx",
]
@ -3549,6 +3553,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
)
class SpendLogsRouterMetadata(TypedDict):
"""
Router provenance stamped on spend logs for deployments flagged with
model_info.internal_router_model, correlating the requested model group
with the provider deployment that served the call
"""
requested_model: ReadOnly[str | None]
selected_model: ReadOnly[str | None]
selected_provider: ReadOnly[str | None]
router_correlation_id: ReadOnly[str | None]
class SpendLogsMetadata(TypedDict):
"""
Specific metadata k,v pairs logged to spendlogs for easier cost tracking
@ -3591,6 +3608,7 @@ class SpendLogsMetadata(TypedDict):
compression_savings: CompressionSavingsMetadata | None
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
litellm_gateway_injected_cache: ReadOnly[str | None]
router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model
class SpendLogsPayload(TypedDict):

103
litellm/proxy/auth/auth_exception_handler.py
View file

@ -2,13 +2,14 @@
Handles Authentication Errors
"""
import logging
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from fastapi import HTTPException, Request, status
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger
from litellm.constants import EMPTY_MAPPING
from litellm.integrations.otel.runtime import seed_request_identity
from litellm.litellm_core_utils.core_helpers import is_expected_client_error
@ -18,7 +19,11 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_utils import _get_request_ip_address
from litellm.proxy.auth.auth_utils import (
_get_request_ip_address,
is_invalid_virtual_key_error,
mark_invalid_virtual_key_error,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes
@ -36,6 +41,41 @@ else:
Span = Any
def _as_proxy_exception(e: Exception) -> ProxyException:
"""Convert an authentication failure into the ProxyException the client receives."""
if isinstance(e, litellm.BudgetExceededError):
return ProxyException(
message=e.message,
type=ProxyErrorTypes.budget_exceeded,
param=None,
code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS),
)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
)
if isinstance(e, ProxyException):
return e
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
return ProxyException(
message=(
"Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly."
),
type=ProxyErrorTypes.no_db_connection,
param="None",
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
return ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_401_UNAUTHORIZED,
)
def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]:
"""Auth gate rejections are raised before `add_litellm_data_to_request` records the
caller IP, so their failure logs would otherwise carry no IP nor key/user identity."""
@ -110,16 +150,21 @@ class UserAPIKeyAuthExceptionHandler:
request=request,
use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True,
)
log_fn: Final = (
verbose_proxy_logger.error
if is_expected_client_error(e) and not litellm.log_client_error_tracebacks
else verbose_proxy_logger.exception
)
log_fn(
# Log authentication failures before identity seeding and callbacks, so the log
# survives a raising callback pipeline. Classify and route malformed virtual-key
# rejections to WARNING on stdout (suppressible via LITELLM_LOG=ERROR).
log_extra: Final = {"requester_ip": requester_ip}
is_invalid_virtual_key: Final = is_invalid_virtual_key_error(e)
is_quiet_log: Final = is_invalid_virtual_key and not litellm.log_client_error_tracebacks
logger: Final = verbose_proxy_stdout_logger if is_quiet_log else verbose_proxy_logger
logger.log(
logging.WARNING if is_quiet_log else logging.ERROR,
"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s",
e,
requester_ip,
extra={"requester_ip": requester_ip},
exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None,
extra=log_extra,
)
# Log this exception to OTEL, Datadog etc. Reuse the identity resolved
@ -167,35 +212,13 @@ class UserAPIKeyAuthExceptionHandler:
if transformed_exception is not None:
e = transformed_exception
if isinstance(e, litellm.BudgetExceededError):
raise ProxyException(
message=e.message,
type=ProxyErrorTypes.budget_exceeded,
param=None,
code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS),
final_exception: Final = mark_invalid_virtual_key_error(_as_proxy_exception(e), is_invalid_virtual_key)
# If a quiet-logged malformed-key transform yields non-401, escalate to ERROR
if is_quiet_log and str(final_exception.code) != str(status.HTTP_401_UNAUTHORIZED):
verbose_proxy_logger.error(
"litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s",
final_exception,
requester_ip,
extra=log_extra,
)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
)
elif isinstance(e, ProxyException):
raise e
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise ProxyException(
message=(
"Service Unavailable, the authentication database is "
"temporarily unreachable. Please retry shortly."
),
type=ProxyErrorTypes.no_db_connection,
param="None",
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_401_UNAUTHORIZED,
)
raise final_exception

54
litellm/proxy/auth/auth_utils.py
View file

@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.constants import (
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
EMPTY_MAPPING,
INVALID_VIRTUAL_KEY_ERROR_MARKER,
MINIMUM_CUSTOM_KEY_LENGTH,
STANDARD_CUSTOMER_ID_HEADERS,
)
@ -34,6 +35,43 @@ from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS
from litellm.types.utils import CustomPricingLiteLLMParams
def is_invalid_virtual_key_error(exception: BaseException | None) -> bool:
"""True when an authentication error rejects a malformed virtual key.
Classifies only by the marker stamped where that 401 is raised. Message
content is never inspected: other 401s interpolate caller-supplied values
(vector store ids, organization ids) into their messages, so a phrase
match would let a request body demote an authorization failure to the
quiet log path.
"""
if not isinstance(exception, (HTTPException, ProxyException)):
return False
code: Final[object] = getattr(exception, "code", None)
status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
if str(status_code) != str(status.HTTP_401_UNAUTHORIZED):
return False
return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True
def mark_invalid_virtual_key_error(exception: ProxyException, is_invalid_virtual_key: bool) -> ProxyException:
"""Return an independently marked malformed-key exception after callback transformations."""
if not is_invalid_virtual_key or str(exception.code) != str(status.HTTP_401_UNAUTHORIZED):
return exception
marked_exception: Final = ProxyException(
message=exception.message,
type=exception.type,
param=exception.param,
code=exception.code,
headers=exception.headers.copy(),
openai_code=None if exception.openai_code is None else str(exception.openai_code),
provider_specific_fields=exception.provider_specific_fields,
)
setattr(marked_exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, True)
return marked_exception
def _get_request_ip_address(request: Request, use_x_forwarded_for: bool | None = False) -> str | None:
client_ip = None
if use_x_forwarded_for is True and "x-forwarded-for" in request.headers:
@ -956,7 +994,7 @@ def get_key_model_rpm_limit(
# 2. Check model_max_budget
if user_api_key_dict.model_max_budget:
model_rpm_limit: Final[dict[str, Any]] = {}
model_rpm_limit: Final[dict[str, int]] = {}
for model, budget in user_api_key_dict.model_max_budget.items():
if isinstance(budget, dict) and budget.get("rpm_limit") is not None:
model_rpm_limit[model] = budget["rpm_limit"]
@ -999,7 +1037,7 @@ def get_key_model_tpm_limit(
# 2. Check model_max_budget (iterate per-model like RPM does)
if user_api_key_dict.model_max_budget:
model_tpm_limit: Final[dict[str, Any]] = {}
model_tpm_limit: Final[dict[str, int]] = {}
for model, budget in user_api_key_dict.model_max_budget.items():
if isinstance(budget, dict) and budget.get("tpm_limit") is not None:
model_tpm_limit[model] = budget["tpm_limit"]
@ -1062,7 +1100,7 @@ def _validated_output_token_estimates_per_model(raw: object) -> Mapping[str, int
def _estimated_output_tokens_from_metadata(
metadata: Mapping[str, Any] | None,
metadata: Mapping[str, object] | None,
model_name: str | None,
) -> int | None:
"""Resolve the per-model, then global, estimate out of one metadata blob.
@ -1628,7 +1666,7 @@ def _dedupe_model_candidates(candidates: list[str]) -> list[str]:
return deduped
def _get_case_insensitive_mapping_value(mapping: Mapping[str, Any] | None, key: str) -> Any:
def _get_case_insensitive_mapping_value(mapping: Mapping[str, object] | None, key: str) -> object:
if not mapping:
return None
if key in mapping:
@ -1732,8 +1770,8 @@ def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | Non
def _extract_model_candidates_from_request(
request_data: dict,
route: str,
request_headers: Mapping[str, Any] | None = None,
request_query_params: Mapping[str, Any] | None = None,
request_headers: Mapping[str, object] | None = None,
request_query_params: Mapping[str, object] | None = None,
llm_router: Router | None = None,
) -> list[str]:
candidates: Final[list[str]] = []
@ -1825,8 +1863,8 @@ def request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool
def get_model_from_request(
request_data: dict,
route: str,
request_headers: Mapping[str, Any] | None = None,
request_query_params: Mapping[str, Any] | None = None,
request_headers: Mapping[str, object] | None = None,
request_query_params: Mapping[str, object] | None = None,
llm_router: Router | None = None,
request: Request | None = None,
) -> str | list[str] | None:

14
litellm/proxy/auth/user_api_key_auth.py
View file

@ -19,12 +19,15 @@ import fastapi
import orjson
from fastapi import HTTPException, Request, WebSocket, status
from fastapi.security.api_key import APIKeyHeader
from starlette.exceptions import WebSocketException
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.constants import (
GLOBAL_PROXY_SPEND_CACHE_KEY,
INVALID_VIRTUAL_KEY_ERROR_MARKER,
INVALID_VIRTUAL_KEY_ERROR_MESSAGE,
LITELLM_PROXY_BUDGET_NAME,
LITELLM_PROXY_MASTER_KEY_ALIAS,
)
@ -65,6 +68,7 @@ from litellm.proxy.auth.auth_utils import (
get_model_from_request,
get_request_route,
get_request_route_template,
is_invalid_virtual_key_error,
iter_request_fallback_targets,
normalize_request_route,
pre_db_read_auth_checks,
@ -539,6 +543,8 @@ async def user_api_key_auth_websocket(websocket: WebSocket):
try:
return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}")
except Exception as e:
if is_invalid_virtual_key_error(e):
raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)
verbose_proxy_logger.exception(e)
await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
raise HTTPException(status_code=403, detail=str(e))
@ -1867,13 +1873,17 @@ async def _user_api_key_auth_builder(
_masked_key: Final = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****"
if not api_key.startswith("sk-"):
_hint = _JWT_AUTH_DISABLED_HINT if not enable_jwt_auth and JWTHandler.is_jwt(token=api_key) else ""
raise HTTPException(
_malformed_key_error = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=(
f"LiteLLM Virtual Key expected. Received={_masked_key}, "
f"{INVALID_VIRTUAL_KEY_ERROR_MESSAGE}. Received={_masked_key}, "
f"expected to start with 'sk-'.{_hint}"
),
) # prevent token hashes from being used
# Stamp provenance here so log routing classifies this 401 by
# where it was raised, never by its message text.
setattr(_malformed_key_error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True)
raise _malformed_key_error
else:
verbose_logger.warning(
"litellm.proxy.proxy_server.user_api_key_auth(): Warning - Key is not a string. Got type={}".format(

49
litellm/proxy/common_request_processing.py
View file

@ -1495,6 +1495,35 @@ class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict):
self.data = data
@staticmethod
def _merge_passthrough_streaming_headers(
response_headers: httpx.Headers | dict | None,
custom_headers: dict,
) -> dict:
"""
Merge upstream passthrough headers with proxy/custom headers.
Proxy/custom headers win on key collisions.
"""
excluded_headers: Final = { # mutable-ok: set of header names to exclude from forwarding
"transfer-encoding",
"content-encoding",
"set-cookie",
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"upgrade",
}
merged_headers: Final = { # mutable-ok: dict comprehension for merged headers forwarded to httpx
key: value for key, value in dict(response_headers or {}).items() if key.lower() not in excluded_headers
}
merged_headers.update(custom_headers)
return merged_headers
@staticmethod
def get_custom_headers(
*,
@ -2389,6 +2418,16 @@ class ProxyBaseLLMRequestProcessing:
)
if route_type == "allm_passthrough_route":
upstream_response_headers: Final = getattr(response, "headers", None)
streaming_headers: Final = (
ProxyBaseLLMRequestProcessing._merge_passthrough_streaming_headers(
response_headers=upstream_response_headers,
custom_headers=custom_headers,
)
if upstream_response_headers is not None
else custom_headers
)
# Check if response is an async generator
if self._is_streaming_response(response):
if asyncio.iscoroutine(response):
@ -2418,11 +2457,11 @@ class ProxyBaseLLMRequestProcessing:
# For passthrough routes, stream directly without error parsing
# since we're dealing with raw binary data (e.g., AWS event streams)
return StreamingResponse(
content=generator,
status_code=status.HTTP_200_OK,
return _UpstreamClosingStreamingResponse(
content=generator, # pyright: ignore[reportArgumentType] # generator-configured StreamingResponse
status_code=getattr(response, "status_code", status.HTTP_200_OK),
media_type=self._passthrough_event_stream_media_type(),
headers=custom_headers,
headers=streaming_headers,
)
else:
_early = await self._handle_non_streaming_allm_passthrough_route(
@ -2437,7 +2476,7 @@ class ProxyBaseLLMRequestProcessing:
return StreamingResponse(
content=response.aiter_bytes(),
status_code=response.status_code,
headers=custom_headers,
headers=streaming_headers,
)
elif route_type == "anthropic_messages":
# Check if response is actually a streaming response (async generator)

Some files were not shown because too many files have changed in this diff Show more