scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

Merge remote-tracking branch 'origin/main' into litellm_techdebt_20261015

Browse source
This commit is contained in:
Devin AI 2026-11-03 07:46:18 +00:00
commit 4b7aaa3fe7
835 changed files with 59783 additions and 11384 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/scripts/run_integration.sh
View file

@ -128,6 +128,8 @@ fi
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
export UV_OFFLINE=true
if [ "$mode" = parity ]; then
export INTEGRATION_WORKERS=1
else

13
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -6,12 +6,13 @@
## Handwritten 1-3 sentence description
<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
plain sentences proves the author understands what it does, and human-written technical English is far easier
for a reviewer to parse than AI-written English
An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
Tags still present at review time mean the summary is AI-written and has not been rewritten yet
<!-- The author writes a plain summary in their own words before requesting review.
Wrap newly AI-written summary text in the make-me-human-readable tags below, including backticks.
The author reviews or rewrites the draft and removes the tags.
When updating an existing PR, preserve the author's wording and removed tags.
Changes elsewhere in the body do not make an unchanged summary an AI draft again.
Add tags again only around newly AI-written replacement summary text, never merely to match this template.
This preserves author review across later updates (2026-11-02, #49779).
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No bullets, tables, or code names here. Everything else goes inside the Details block below -->

4
.github/workflows/compat-matrix-image.yml vendored
View file

@ -4,7 +4,7 @@ on:
pull_request:
paths:
- tests/e2e/claude_code/cron_vm/**
- tests/e2e/claude_code/pr_gate_version_resolver.py
- tests/e2e/cli_harness/npm_version_resolver.py
- .github/workflows/compat-matrix-image.yml
workflow_dispatch:
@ -36,7 +36,7 @@ jobs:
whoami
gh --version
uv --version
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
/opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
/tmp/claude-cli/claude --version
'

30
.github/workflows/deploy-dev-on-main.yml vendored
View file

@ -1,30 +0,0 @@
name: Deploy main to gateway-dev
on:
push:
branches: [main]
permissions: {}
concurrency:
group: deploy-dev-dispatch
cancel-in-progress: true
jobs:
dispatch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
id: token
with:
app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
owner: BerriAI
repositories: litellm-ops
permission-actions: write
- name: Dispatch the litellm-ops dev deploy
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

2
.github/workflows/test-linting.yml vendored
View file

@ -208,7 +208,7 @@ jobs:
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

1
.github/workflows/test-unit.yml vendored
View file

@ -441,6 +441,7 @@ jobs:
tests/unit/ocr
tests/unit/passthrough
tests/unit/rag
tests/unit/cookbook
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search

85
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -1,85 +0,0 @@
name: "Weekly Load Anomaly Check"
on:
schedule:
- cron: "0 12 * * 6"
workflow_dispatch:
permissions:
contents: read
jobs:
weekly-load-anomaly:
if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-weekly-anomaly-check
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Run the weekly session anomaly test
env:
E2E_WEEKLY_ANOMALY: "1"
run: |
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

1
README.md
View file

@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | |

245
cookbook/gpt_realtime_translate.py Normal file
View file

@ -0,0 +1,245 @@
#!/usr/bin/env python3
import argparse
import asyncio
import base64
import json
import os
import sys
import wave
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlencode, urlsplit, urlunsplit
import websockets
from websockets.asyncio.client import ClientConnection
SAMPLE_RATE = 24_000
CHANNELS = 1
SAMPLE_WIDTH = 2
CHUNK_DURATION_SECONDS = 0.1
CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
RESPONSE_TIMEOUT_SECONDS = 30.0
AUDIO_EVENT_TYPES = frozenset(
{
"session.output_audio.delta",
"response.audio.delta",
"response.output_audio.delta",
}
)
TRANSCRIPT_EVENT_TYPES = frozenset(
{
"session.output_transcript.delta",
"response.text.delta",
"response.output_audio_transcript.delta",
}
)
@dataclass(frozen=True, slots=True)
class Settings:
input_wav: Path
output_wav: Path
base_url: str
model: str
target_language: str
trailing_silence_seconds: float
api_key: str
def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
sys.stdout.write(f"{message}{end}")
if flush:
sys.stdout.flush()
def write_stderr(message: str) -> None:
sys.stderr.write(f"{message}\n")
def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
parser = argparse.ArgumentParser(
description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
)
parser.add_argument("input_wav", type=Path)
parser.add_argument("--output", type=Path, default=Path("translated.wav"))
parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
parser.add_argument("--target-language", default="fr")
parser.add_argument("--trailing-silence", type=float, default=1.5)
parsed = parser.parse_args(argv)
api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
if not api_key:
return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
if parsed.trailing_silence < 0:
return "--trailing-silence must be zero or greater"
return Settings(
input_wav=parsed.input_wav,
output_wav=parsed.output,
base_url=parsed.base_url,
model=parsed.model,
target_language=parsed.target_language,
trailing_silence_seconds=parsed.trailing_silence,
api_key=api_key,
)
def translation_url(base_url: str, model: str) -> str | None:
parsed = urlsplit(base_url.rstrip("/"))
scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
if not scheme or not parsed.netloc:
return None
base_path = parsed.path.rstrip("/")
realtime_path = (
f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
)
return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
def read_pcm16_wav(path: Path) -> bytes | str:
try:
with wave.open(str(path), "rb") as source:
actual_format = (
source.getnchannels(),
source.getsampwidth(),
source.getframerate(),
source.getcomptype(),
)
expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
if actual_format != expected_format:
return (
f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
)
return source.readframes(source.getnframes())
except (OSError, EOFError, wave.Error) as exc:
return f"Unable to read {path}: {exc}"
def audio_chunks(audio: bytes) -> Iterator[bytes]:
return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
def audio_message(audio: bytes) -> str:
return json.dumps(
{
"type": "session.input_audio_buffer.append",
"audio": base64.b64encode(audio).decode("ascii"),
}
)
async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
await connection.send(
json.dumps(
{
"type": "session.update",
"session": {"audio": {"output": {"language": target_language}}},
}
)
)
while True:
raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type == "session.created":
write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
if event_type == "session.updated":
return None
if event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
async def send_audio(
connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
) -> None:
silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
try:
for chunk in audio_chunks(pcm + silence):
await connection.send(audio_message(chunk))
await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
await connection.send(json.dumps({"type": "session.close"}))
finally:
finished.set()
async def receive_translation(
connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
) -> str | None:
audio_received = asyncio.Event()
try:
with wave.open(str(output_path), "wb") as output:
output.setnchannels(CHANNELS)
output.setsampwidth(SAMPLE_WIDTH)
output.setframerate(SAMPLE_RATE)
write_stdout("Translation: ", end="", flush=True)
while True:
try:
raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
except TimeoutError:
if not sender_finished.is_set():
continue
return (
f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type in AUDIO_EVENT_TYPES:
output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
audio_received.set()
elif event_type in TRANSCRIPT_EVENT_TYPES:
write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
elif event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
elif event_type == "session.closed":
write_stdout()
return None if audio_received.is_set() else "The translation stream ended without translated audio"
except (OSError, wave.Error) as exc:
return f"Unable to write {output_path}: {exc}"
async def translate(settings: Settings, pcm: bytes) -> str | None:
url = translation_url(settings.base_url, settings.model)
if not url:
return f"Invalid --base-url: {settings.base_url}"
sender_finished = asyncio.Event()
try:
async with websockets.connect(
url,
additional_headers={"Authorization": f"Bearer {settings.api_key}"},
proxy=None,
open_timeout=20,
close_timeout=5,
) as connection:
configuration_error = await configure_session(connection, settings.target_language)
if configuration_error:
return configuration_error
async with asyncio.TaskGroup() as tasks:
receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
return receiver.result()
except Exception as exc:
return f"Translation failed: {type(exc).__name__}: {exc}"
def main(argv: Sequence[str] | None = None) -> int:
settings = parse_args(argv)
if isinstance(settings, str):
write_stderr(settings)
return 2
pcm = read_pcm16_wav(settings.input_wav)
if isinstance(pcm, str):
write_stderr(pcm)
return 2
error = asyncio.run(translate(settings, pcm))
if error:
write_stderr(error)
return 1
write_stdout(f"Translated audio: {settings.output_wav}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

184
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -3,9 +3,11 @@
import asyncio
import base64
import copy
import json
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta
from itertools import chain
from types import MappingProxyType
@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
import litellm
from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import (
@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
request_tags_from_metadata,
)
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
custom_llm_provider: ReadOnly[str | None]
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
@dataclass(frozen=True, slots=True)
class _ProviderUpload:
model_id: str
provider_file_id: str
upload_call_id: str | None = None
def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
file_hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
file_hidden_params.get("model_file_id_mapping") or {}
)
upload_call_id: Final = file_hidden_params.get("litellm_call_id")
return tuple(
_ProviderUpload(
model_id=model_id,
provider_file_id=provider_file_id,
upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
)
for model_id, provider_file_id in model_file_id_mapping.items()
)
def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
upload: Final = uploads[index]
other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
return all(other.model_id != upload.model_id for other in other_uploads)
def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
return None
return carried
async def _delete_provider_file(
llm_router: Router,
model_id: str,
provider_file_id: str,
request_data: Mapping[str, object],
) -> None:
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
delete_data: Final = {
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, delete_data),
)
def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
async def _delete_redundant_upload(
llm_router: Router,
upload: _ProviderUpload,
delete_request: Mapping[str, object],
parent_call_id: str | None,
) -> None:
try:
with internal_sub_call(parent_call_id):
await _delete_provider_file(
llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
)
except Exception as error:
verbose_logger.warning(
f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
async def _delete_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
delete_request: Mapping[str, object],
) -> None:
await asyncio.gather(
*(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
)
def _start_deleting_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
create_file_request: CreateFileRequest,
) -> None:
if not uploads:
return
delete_request: Final = MappingProxyType(
{k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
)
with post_response_phase():
spawn_logging_task(
asyncio.get_running_loop(),
_delete_redundant_uploads(llm_router, uploads, delete_request),
task_name="managed file repeat upload cleanup",
)
def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
return prisma_client.db
@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
target_model_names_list=target_model_names_list,
)
## STORE MODEL MAPPINGS IN DB
model_mappings: Dict[str, str] = {}
for file_object in responses:
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
model_mappings.update(model_file_id_mapping)
uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
per_target_model_ids: Final = frozenset(
model_id
for model_id in {upload.model_id for upload in uploads}
if _keeps_one_upload_per_target(llm_router, model_id)
)
kept_uploads: Final = tuple(
upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
)
model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
await self.store_unified_file_id(
file_id=response.id,
@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
_start_deleting_redundant_uploads(
llm_router,
tuple(
upload
for upload in uploads
if upload not in kept_uploads and upload.model_id not in per_target_model_ids
),
create_file_request,
)
# Emit Prometheus metrics for managed file creation
prom_logger = self._get_prometheus_logger()
@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
if not specific_model_file_id_mapping:
return
filtered_data: Final = {
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
credentials = llm_router.registry().credentials(model_id, form="with_provider")
delete_data = {
**filtered_data,
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=model_file_id,
**cast(_RouterFileCallKwargs, delete_data),
await _delete_provider_file(llm_router, model_id, model_file_id, data)
unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
delete_call_id: Final = data.get("litellm_call_id")
if unrecorded_upload is not None:
await _delete_redundant_upload(
llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
)
async def afile_content(

18
litellm-rust/crates/callbacks-legacy-python/python_contract.json
View file

@ -33,6 +33,17 @@
"api_key",
"additional_args"
],
"record_response": [
"logger",
"status",
"headers",
"body"
],
"record_api_window": [
"logger",
"start",
"end"
],
"defers_async_logging": [
"logger"
],
@ -91,6 +102,13 @@
"logger",
"head"
],
"stamp_stream_head": [
"head",
"logger",
"kwargs",
"start_time",
"end_time"
],
"stream_success": [
"logger",
"url_route",

117
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -57,6 +57,8 @@ pub struct LegacyLogging {
asynchronous: bool,
internal: bool,
cache_key: Option<String>,
/// When the provider request went out, until its window is recorded (Messages only).
api_started: Option<f64>,
}
fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
@ -87,6 +89,60 @@ impl LegacyLogging {
asynchronous,
internal: false,
cache_key: None,
api_started: None,
}
}
/// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
/// its own argument, so the logger never sees the caller's raw metadata, only the validated
/// request metadata merged in through `litellm_params`. Python updates the logger once, before
/// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
/// logged metadata is not rebuilt from what the call wrote into it since.
fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult<Bound<'py, PyDict>> {
let kwargs = self.call.kwargs().bind(py).copy()?;
if self.operation != LoggingOperation::Messages {
return Ok(kwargs);
}
let consumed: &[&str] = if refresh {
&["metadata", "litellm_metadata"]
} else {
&["metadata"]
};
for name in consumed {
if kwargs.contains(name)? {
kwargs.del_item(name)?;
}
}
Ok(kwargs)
}
/// Python's Messages handler hands the logger its request params, validated `metadata`
/// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
fn logged_request_params(
&self,
context: &RequestContext,
) -> Option<serde_json::Map<String, Value>> {
if self.operation != LoggingOperation::Messages {
return None;
}
let mut params = context
.optional_params
.as_object()
.cloned()
.unwrap_or_default();
params.insert(
"aanthropic_messages".to_owned(),
Value::Bool(self.asynchronous),
);
Some(params)
}
/// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
/// client is untimed.
fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
match self.api_started.take() {
Some(start) => self.logger()?.record_api_window(py, start, end),
None => Ok(()),
}
}
@ -111,6 +167,16 @@ impl LegacyLogging {
}
}
/// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
match self.operation {
LoggingOperation::Messages => Some(""),
LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
context.api_key.as_ref().map(|api_key| api_key.expose())
}
}
}
fn stream_billing(&self) -> Option<PassThroughStream> {
match self.operation {
LoggingOperation::Messages => Some(PassThroughStream {
@ -377,7 +443,13 @@ impl LegacyLogging {
context: &RequestContext,
) -> PyResult<HookStep<Self, Box<WireRequest>>> {
let logger = self.logger()?;
logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
logger.update_from_kwargs(
py,
&self.logging_kwargs(py, false)?,
&wire,
context,
self.logged_request_params(context).as_ref(),
)?;
let body = to_py(py, &wire.body)?
.into_bound(py)
.cast_into::<PyDict>()?;
@ -400,11 +472,14 @@ impl LegacyLogging {
self.logger()?.pre_call(
py,
self.input_description(),
context.api_key.as_ref().map(|api_key| api_key.expose()),
self.logged_api_key(context),
&body,
&headers,
&wire.url,
)?;
if self.operation == LoggingOperation::Messages && self.asynchronous {
self.api_started = Some(epoch_seconds());
}
let headers = headers
.iter()
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
@ -455,7 +530,7 @@ impl LegacyLogging {
py,
(
&logger,
self.call.kwargs(),
self.logging_kwargs(py, true)?,
&facts.provider.model,
logger.getattr("optional_params")?,
params,
@ -483,6 +558,16 @@ impl LegacyLogging {
py: Python<'_>,
raw: &RawResponse,
) -> PyResult<HookStep<Self, ()>> {
if self.operation == LoggingOperation::Messages {
// Python's Messages handler never calls `post_call` at receipt: it stores the raw
// response, and logging replays it through `AnthropicConfig.transform_response`.
self.close_api_window(py, epoch_seconds())?;
if let Some(head) = &raw.head {
self.logger()?
.record_response(py, head.status, &head.headers, &raw.body)?;
}
return Ok(HookStep::Ready(()));
}
let api_key = self
.request
.as_ref()
@ -522,6 +607,9 @@ impl LegacyLogging {
) -> PyResult<HookStep<Self, ()>> {
self.end = Some(datetime(py, timing.end_time)?);
self.error = Some(error.clone_ref(py).into_value(py));
if self.logger.is_some() {
self.close_api_window(py, timing.end_time)?;
}
if self.stream.is_some() {
return self.stream_failure(py);
}
@ -543,6 +631,20 @@ impl LegacyLogging {
head.bind(py).set_item("cache_key", key)?;
head.bind(py).set_item("cache_hit", true)?;
}
self.close_api_window(py, epoch_seconds())?;
if self.operation == LoggingOperation::Messages {
// Python's `@client` stamps the call metadata onto the stream it hands back.
Streaming::StampHead.call(
py,
(
head.bind(py),
self.logger()?.object(py),
self.call.kwargs(),
&self.start,
datetime(py, epoch_seconds())?,
),
)?;
}
Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
self.stream = Some(DeliveredStream {
chunks: PyList::empty(py).unbind(),
@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
let step = logging
.before_provider_request(py, Box::new(wire), context)
.unwrap();
let raw = RawResponse {
body: "raw response".into(),
};
let raw = RawResponse::new("raw response".into());
assert!(matches!(
logging
.on_event(
@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
py,
&locals,
c"
assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
assert logger.calls[0][1] is head
chunks = logger.calls[1][1]
assert logger.calls[1][1] is head
chunks = logger.calls[2][1]
assert len(chunks) == 2
assert chunks[0] is first
assert chunks[1] is last

51
litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs
View file

@ -5,6 +5,7 @@
use litellm_host::interceptors::{RequestContext, WireRequest};
use litellm_host_python::to_py;
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
use serde_json::Value;
use crate::logger::PythonLogger;
use crate::python::{Logging, Wrapper};
@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
request_params: Option<&serde_json::Map<String, Value>>,
) -> PyResult<()>;
/// `Logging.pre_call`.
@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
url: &str,
) -> PyResult<()>;
/// Records the provider's raw HTTP response on the logger, as Python's Messages handler
/// does with `httpx_response` and `response_headers`.
fn record_response(
&self,
py: Python<'_>,
status: u16,
headers: &[(String, String)],
body: &str,
) -> PyResult<()>;
/// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
/// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
/// `Logging.post_call`.
fn post_call(
&self,
@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
request_params: Option<&serde_json::Map<String, Value>>,
) -> PyResult<()> {
let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
let optional_params = redact(
py,
&to_py(py, &context.optional_params)?
@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
&secret_fields,
)?;
let params = PyDict::new(py);
params.set_item(
"litellm_call_id",
kwargs.bind(py).get_item("litellm_call_id")?,
)?;
params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
params.set_item("api_base", &wire.url)?;
for name in ["logger_fn", "litellm_request_debug"] {
if let Some(value) = kwargs.bind(py).get_item(name)? {
if let Some(value) = kwargs.get_item(name)? {
params.set_item(name, value)?;
}
}
for name in custom_pricing_fields(py)? {
if let Some(value) = kwargs.bind(py).get_item(&name)?
if let Some(value) = kwargs.get_item(&name)?
&& !value.is_none()
{
params.set_item(name, value)?;
}
}
for (name, value) in request_params.into_iter().flatten() {
params.set_item(name, to_py(py, value)?)?;
}
Logging::Update.call(
py,
(
@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn record_response(
&self,
py: Python<'_>,
status: u16,
headers: &[(String, String)],
body: &str,
) -> PyResult<()> {
Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
Ok(())
}
fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
Ok(())
}
fn post_call(
&self,
py: Python<'_>,

6
litellm-rust/crates/callbacks-legacy-python/src/python.rs
View file

@ -44,6 +44,10 @@ pub(crate) enum Logging {
PreCall,
#[strum(serialize = "post_call")]
PostCall,
#[strum(serialize = "record_response")]
RecordResponse,
#[strum(serialize = "record_api_window")]
RecordApiWindow,
#[strum(serialize = "defers_async_logging")]
DefersAsync,
#[strum(serialize = "defer_success")]
@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
pub(crate) enum Streaming {
#[strum(serialize = "stream_opened")]
Opened,
#[strum(serialize = "stamp_stream_head")]
StampHead,
#[strum(serialize = "stream_success")]
Success,
#[strum(serialize = "stream_failure")]

7
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -65,6 +65,12 @@ FAKES = {
'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
original_response, api_key, additional_args
),
'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
response_headers=dict(headers)
),
'record_api_window': lambda logger, start, end: logger.model_call_details.update(
llm_api_duration_ms=(end - start) * 1000
),
'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@ -85,6 +91,7 @@ FAKES = {
),
'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
'stream_opened': lambda logger, head: logger.record('stream_opened', head),
'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
'stream_success', list(chunks)
),

14
litellm-rust/crates/host-http/tests/serve.rs
View file

@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(Bytes::from(wire.url)))
},
@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
move |_, _, route_hooks, _observations| async move {
if event {
route_hooks
.after_provider_response(RawResponse {
body: "response".into(),
})
.after_provider_response(RawResponse::new("response".into()))
.await?;
} else {
route_hooks
@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(json!({"url": wire.url})))
},
@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
None,
|_, _, route_hooks, _observations| async move {
route_hooks
.after_provider_response(RawResponse { body: "raw".into() })
.after_provider_response(RawResponse::new("raw".into()))
.await?;
Err(TestError::Provider)
},

8
litellm-rust/crates/host-native/tests/driver.rs
View file

@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(wire.url))
},
@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
"input",
Some(sender),
|_, _, interceptors, observers| async move {
let raw = RawResponse {
body: "provider response".into(),
};
let raw = RawResponse::new("provider response".into());
observers.unwrap().emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
));

2
litellm-rust/crates/host-python/tests/support/mod.rs
View file

@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
.before_provider_request(wire(), context())
.await?;
host.interceptors
.after_provider_response(RawResponse { body: "raw".into() })
.after_provider_response(RawResponse::new("raw".into()))
.await?;
Ok(format!("{projected}|{signed}|{}", wire.url))
})

24
litellm-rust/crates/host/src/interceptors.rs
View file

@ -28,6 +28,28 @@ pub struct RequestContext {
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawResponse {
pub body: String,
/// The provider's HTTP status and headers, when the route forwards them to logging.
pub head: Option<ResponseHead>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ResponseHead {
pub status: u16,
pub headers: Vec<(String, String)>,
}
impl RawResponse {
pub fn new(body: String) -> Self {
Self { body, head: None }
}
#[must_use]
pub fn with_head(self, head: ResponseHead) -> Self {
Self {
head: Some(head),
..self
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -186,7 +208,7 @@ mod tests {
.await?;
Interceptors::after_provider_response(
&channel.interceptors,
RawResponse { body: "raw".into() },
RawResponse::new("raw".into()),
)
.await?;
Ok((sent, ()))

4
litellm-rust/crates/host/tests/observation.rs
View file

@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
#[rstest]
fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
let mut raw = RawResponse {
body: "provider response".into(),
};
let mut raw = RawResponse::new("provider response".into());
let event: CallEvent<(), (), &RawResponse> =
CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
let expected = raw.clone();

76
litellm-rust/crates/http/src/error.rs
View file

@ -58,8 +58,20 @@ pub(crate) enum Repr {
Network(#[source] reqwest::Error),
#[error("upstream request timed out")]
Timeout(#[source] reqwest::Error),
/// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
/// the response head had already arrived.
#[error("upstream request timed out")]
ReadTimeout {
#[source]
elapsed: Elapsed,
body: bool,
},
#[error("upstream request failed with status {status}: {body}")]
Status { status: u16, body: UpstreamBody },
Status {
status: u16,
body: UpstreamBody,
headers: Vec<(String, String)>,
},
#[error("invalid WebSocket URL")]
WebSocketUrl(#[source] url::ParseError),
@ -136,13 +148,59 @@ impl Repr {
impl Error {
pub fn upstream(status: u16, body: String) -> Self {
Self::upstream_with_headers(status, body, Vec::new())
}
/// An upstream failure that keeps the provider's response headers (`retry-after`,
/// `request-id`) for the caller.
pub fn upstream_with_headers(
status: u16,
body: String,
headers: Vec<(String, String)>,
) -> Self {
Repr::Status {
status,
body: UpstreamBody(body),
headers,
}
.into()
}
/// The provider's response headers on an upstream failure; empty for every other error.
pub fn upstream_headers(&self) -> &[(String, String)] {
match &*self.0 {
Repr::Status { headers, .. } => headers,
_ => &[],
}
}
/// A timeout reached before the connection was established, as opposed to while reading.
pub fn is_connect_timeout(&self) -> bool {
matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
}
/// A read the caller timed itself (httpx's per-read timeout) ran out.
pub fn read_timeout(elapsed: Elapsed) -> Self {
Repr::ReadTimeout {
elapsed,
body: false,
}
.into()
}
/// A read of the response body the caller timed itself ran out, after the head arrived.
pub fn body_read_timeout(elapsed: Elapsed) -> Self {
Repr::ReadTimeout {
elapsed,
body: true,
}
.into()
}
pub fn is_body_read_timeout(&self) -> bool {
matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
}
pub fn computed_header(name: impl Into<String>) -> Self {
Repr::ComputedHeader(name.into()).into()
}
@ -169,6 +227,7 @@ impl Error {
match &*self.0 {
Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@ -227,6 +286,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@ -259,6 +319,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -295,6 +356,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -333,6 +395,7 @@ impl Error {
| Repr::Signature(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -356,7 +419,10 @@ impl Error {
pub fn is_timeout(&self) -> bool {
match &*self.0 {
Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::HandshakeTimeout(_)
| Repr::DownloadTimeout(_) => true,
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -392,7 +458,7 @@ impl Error {
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match &*self.0 {
Repr::Status { status, body } => Some((*status, &body.0)),
Repr::Status { status, body, .. } => Some((*status, &body.0)),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -408,6 +474,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
| Repr::WebSocketSchemeRewrite(_)
@ -431,7 +498,7 @@ impl Error {
pub fn status(&self) -> Option<u16> {
match &*self.0 {
Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -482,6 +549,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)

2
litellm-rust/crates/inference-chat/src/handler.rs
View file

@ -108,7 +108,7 @@ pub(super) async fn execute(
if !status.is_success() {
return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
}
let raw = RawResponse { body: text.clone() };
let raw = RawResponse::new(text.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

2
litellm-rust/crates/inference-messages/Cargo.toml
View file

@ -24,6 +24,7 @@ litellm-tracing.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
tokio.workspace = true
tracing.workspace = true
[dev-dependencies]
@ -34,5 +35,4 @@ litellm-host-native.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }
litellm-inference-testing.workspace = true
rstest.workspace = true
tokio.workspace = true
wiremock.workspace = true

74
litellm-rust/crates/inference-messages/src/common_utils.rs
View file

@ -35,6 +35,14 @@ impl MessagesProvider {
pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
self.config
}
/// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
pub(crate) fn accepts_service_tier(self) -> bool {
matches!(
self.provider,
LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
)
}
}
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@ -119,3 +127,69 @@ mod tests {
);
}
}
/// Python's `_merge_messages_header_sources`: later sources win per header name, except
/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
/// sorted into one lowercase `anthropic-beta` entry.
pub fn merge_messages_header_sources(
sources: impl IntoIterator<Item = Map<String, Value>>,
) -> Map<String, Value> {
let mut merged = Map::new();
let mut betas = std::collections::BTreeSet::new();
for (name, value) in sources.into_iter().flatten() {
if !name.eq_ignore_ascii_case("anthropic-beta") {
merged.insert(name, value);
continue;
}
let values: Vec<&str> = match &value {
Value::String(text) => vec![text.as_str()],
Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
_ => Vec::new(),
};
betas.extend(
values
.iter()
.flat_map(|value| value.split(','))
.map(str::trim)
.filter(|beta| !beta.is_empty())
.map(str::to_owned),
);
}
if !betas.is_empty() {
merged.insert(
"anthropic-beta".to_owned(),
Value::String(betas.into_iter().collect::<Vec<_>>().join(",")),
);
}
merged
}
#[cfg(test)]
mod merge_tests {
use serde_json::{Map, Value, json};
use super::merge_messages_header_sources;
fn map(value: Value) -> Map<String, Value> {
value.as_object().cloned().unwrap()
}
#[test]
fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
let merged = merge_messages_header_sources([
map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
map(json!({"anthropic-beta": 5})),
]);
assert_eq!(
Value::Object(merged),
json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
);
}
#[test]
fn no_beta_values_leave_no_beta_header() {
let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
assert!(merged.is_empty());
}
}

191
litellm-rust/crates/inference-messages/src/handler.rs
View file

@ -10,9 +10,9 @@ use litellm_core_utils::{
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
};
use litellm_host::interceptors::{
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
ResponseHead, ResultSource, WireRequest,
};
use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
messages::{
@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
transformation::BaseMessagesConfig,
},
};
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use litellm_llms_types::formats::messages::{
MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::Value;
use super::{
Error, MessagesCall, MessagesCallResponse, MessagesRoute,
common_utils::{MessagesProvider, string_headers},
common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
constants::MESSAGES_TIMEOUT_SECS,
route::Messages,
types::invalid_request,
@ -126,14 +128,18 @@ async fn provider_call(
MessagesRequest { model, ..body },
shaping.settings.reasoning_auto_summary,
)?;
let transform_context =
MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let forwarded = string_headers(Some(merge_messages_header_sources([
extra_headers.unwrap_or_default(),
scoped,
])))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
@ -144,16 +150,13 @@ async fn provider_call(
)?;
let body = config.request_body(
config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?,
config.transform_anthropic_messages_request(trimmed, &transform_context)?,
&validated.headers,
shaping.settings.drop_params,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
with_python_default_headers(validated.headers, config.default_headers()),
&body,
),
auth: validated.auth,
@ -184,7 +187,7 @@ async fn provider_call(
.before_provider_request(
WireRequest {
url,
headers: authenticated.headers,
headers: joined_header_values(authenticated.headers),
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
},
request_context,
@ -213,6 +216,64 @@ async fn provider_call(
})
}
/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
/// case into one lowercase header whose values are joined with `", "`.
fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
let mut positions: std::collections::HashMap<String, usize> =
std::collections::HashMap::with_capacity(headers.len());
for (name, value) in headers {
match positions.entry(name.to_ascii_lowercase()) {
std::collections::hash_map::Entry::Occupied(position) => {
let present = &mut joined[*position.get()].1;
present.push_str(", ");
present.push_str(&value);
}
std::collections::hash_map::Entry::Vacant(position) => {
position.insert(joined.len());
joined.push((name, value));
}
}
}
joined
}
/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
fn with_python_default_headers(
headers: Vec<(String, String)>,
defaults: &[(&str, &str)],
) -> Vec<(String, String)> {
let missing: Vec<(String, String)> = defaults
.iter()
.filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
.map(|(name, value)| ((*name).to_string(), (*value).to_string()))
.collect();
headers.into_iter().chain(missing).collect()
}
/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
/// and sampling params before `additional_drop_params` and the environment check run.
fn requested_optional_params(
request: MessagesRequest,
provider: MessagesProvider,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
request, context,
)?;
if provider.accepts_service_tier() {
return Ok(request);
}
Ok(MessagesRequest {
params: MessagesOptionalParams {
service_tier: None,
..request.params
},
..request
})
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],
@ -245,6 +306,7 @@ async fn call_provider(
} = request;
let provider_name = provider.as_str();
log_request_body(provider_name, stream, &wire.body);
let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
let response = send(
http,
Authenticated {
@ -254,10 +316,11 @@ async fn call_provider(
&wire.url,
&wire.body,
timeout,
stream,
)
.await?;
if !response.status().is_success() {
return Err(provider_error(response).await);
return Err(provider_error(response, timeout).await);
}
let config = provider.config();
if stream {
@ -265,11 +328,24 @@ async fn call_provider(
response,
config.stream_decoder(),
provider_name,
timeout,
));
}
let head = ResponseHead {
status: response.status().as_u16(),
headers: litellm_http::request::header_pairs(response.headers()),
};
let status = head.status;
let text = response.text().await.map_err(litellm_http::Error::from)?;
log_response_body(&text);
context.response_received(&text).await?;
context
.raw_response_received(RawResponse::new(text.clone()).with_head(head))
.await?;
if serde_json::from_str::<serde::de::IgnoredAny>(&text).is_err() {
// Python raises `AnthropicError(status_code=<the response's>, message=<its text>)` for a
// body it cannot parse, and the public mapping classifies it by that status.
return Err(litellm_http::Error::upstream(status, text).into());
}
decode_response(config, &identity.model, &text)
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
}
@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
))
}
/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
async fn send(
http: &litellm_http::Client,
authenticated: Authenticated,
url: &str,
body: &Value,
timeout: Option<Duration>,
timeout: Duration,
stream: bool,
) -> Result<reqwest::Response, Error> {
let request = outbound_request(
authenticated,
url,
body,
Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
)?;
Ok(litellm_inference::outbound::send(request, http).await?)
let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
if !stream {
return Ok(litellm_inference::outbound::send(request, http).await?);
}
tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
.await
.map_err(litellm_http::Error::read_timeout)?
.map_err(Error::from)
}
async fn provider_error(response: reqwest::Response) -> Error {
/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
fn timed_chunks(
response: reqwest::Response,
timeout: Duration,
) -> impl futures_util::Stream<Item = Result<Bytes, litellm_http::Error>> + Send + 'static {
futures_util::stream::try_unfold(response, move |mut response| async move {
let chunk = tokio::time::timeout(timeout, response.chunk())
.await
.map_err(litellm_http::Error::body_read_timeout)?
.map_err(litellm_http::Error::from)?;
Ok(chunk.map(|chunk| (chunk, response)))
})
}
/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
/// its head arrives, so an error body that stalls would otherwise hold the call forever.
async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
let status = response.status().as_u16();
match response.text().await {
Ok(text) => {
let headers = litellm_http::request::header_pairs(response.headers());
match tokio::time::timeout(timeout, response.text()).await {
Ok(Ok(text)) => {
log_error_body(status, &text);
litellm_http::Error::upstream(status, text).into()
litellm_http::Error::upstream_with_headers(status, text, headers).into()
}
Err(error) => litellm_http::Error::from(error).into(),
Ok(Err(error)) => litellm_http::Error::from(error).into(),
Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
}
}
@ -329,22 +428,22 @@ fn streaming_response(
response: reqwest::Response,
decoder: Option<StreamDecoder>,
provider: &'static str,
timeout: Duration,
) -> MessagesCallResponse {
let headers = response
.headers()
.iter()
.filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
.collect();
let chunks = timed_chunks(response, timeout)
.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
let chunks = match decoder {
None => futures_util::stream::try_unfold(response, move |mut response| async move {
let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
Ok(chunk.map(|chunk| {
log_chunk(provider, "provider_response", &chunk);
(chunk, response)
}))
})
.boxed(),
Some(decode) => decoded_chunks(response, decode, provider),
None => chunks.map_err(Error::from).boxed(),
Some(decode) => decoded_chunks(
chunks.map_err(std::io::Error::other).boxed(),
decode,
provider,
),
};
MessagesCallResponse::Stream {
head: super::route::MessagesStreamHead { headers },
@ -353,15 +452,10 @@ fn streaming_response(
}
fn decoded_chunks(
response: reqwest::Response,
bytes: ByteStream,
decode: StreamDecoder,
provider: &'static str,
) -> BoxStream<'static, Result<Bytes, Error>> {
let bytes: ByteStream = response
.bytes_stream()
.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
.map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
.boxed();
futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
let Some(event) = events.try_next().await? else {
return Ok(None);
@ -964,9 +1058,12 @@ mod tests {
.send()
.await
.unwrap();
let MessagesCallResponse::Stream { mut chunks, .. } =
streaming_response(response, Some(streaming::event_stream), "test")
else {
let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
response,
Some(streaming::event_stream),
"test",
Duration::from_secs(5),
) else {
panic!("a streaming response returns chunks");
};

1
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -14,6 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use common_utils::merge_messages_header_sources;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,

14
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
/// Python adds a default unless that exact lowercase name is present, then folds the casings
/// into one header, so a capitalised caller header is sent beside the default.
async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
call: MessagesCall,
#[case] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(MessagesCall {
@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
.await;
let request = only_request(&upstream).await;
assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
assert_eq!(
request.header_values("anthropic-version"),
["2024-01-01, 2023-06-01"]
);
assert_eq!(
request.header_values("content-type"),
["application/json; charset=utf-8"]
["application/json; charset=utf-8, application/json"]
);
}

25
litellm-rust/crates/inference-messages/tests/messages/response.rs
View file

@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
}
#[rstest]
#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
#[case::not_a_message(json_response(json!({"unexpected": true})))]
#[tokio::test]
async fn an_unreadable_success_body_is_an_invalid_response(
call: MessagesCall,
#[case] response: ResponseTemplate,
) {
let upstream = upstream([response]).await;
async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
let error = run(MessagesCall {
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect_err("an unparseable body fails");
assert_eq!(error.upstream_response(), Some((200, "not json")));
}
#[rstest]
#[tokio::test]
async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
let error = run(MessagesCall {
api_key: Some("sk".into()),

43
litellm-rust/crates/inference-messages/tests/messages/stream.rs
View file

@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
.unwrap();
}
/// Answers with an error head whose promised body never arrives.
async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let connection = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0; 4096];
let _ = socket.read(&mut request).await;
socket
.write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
.await
.unwrap();
let _ = socket.read_to_end(&mut Vec::new()).await;
});
(base, connection)
}
#[rstest]
#[tokio::test]
async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
let (base, connection) = stalling_error_upstream().await;
let host = RecordingStreamHost::new(
MessagesCall {
timeout: Some(Duration::from_millis(300)),
..streaming(call, base)
},
usize::MAX,
);
let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
.await
.expect("the stalled error body gives up within the timeout")
.expect_err("a stalled error body fails the call");
assert!(error.dispatched(), "{error:?}");
assert_eq!(error.upstream_response(), None);
assert!(host.seen.into_inner().unwrap().is_empty());
tokio::time::timeout(Duration::from_secs(5), connection)
.await
.expect("timing out closes the upstream connection")
.unwrap();
}
#[rstest]
#[tokio::test]
async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {

4
litellm-rust/crates/inference-ocr/src/handler.rs
View file

@ -91,9 +91,7 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
}
fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
let raw = RawResponse {
body: String::from_utf8_lossy(body).into_owned(),
};
let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
if let Some(observers) = self.observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

2
litellm-rust/crates/inference-responses/src/handler.rs
View file

@ -114,7 +114,7 @@ pub(super) async fn execute(
});
}
let body = response.text().await.map_err(litellm_http::Error::from)?;
let raw = RawResponse { body: body.clone() };
let raw = RawResponse::new(body.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

8
litellm-rust/crates/inference/src/context.rs
View file

@ -42,9 +42,11 @@ impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
}
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
let raw = RawResponse {
body: body.to_owned(),
};
self.raw_response_received(RawResponse::new(body.to_owned()))
.await
}
pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

28
litellm-rust/crates/inference/src/error.rs
View file

@ -146,6 +146,34 @@ impl Error {
}
}
/// The provider's response headers on an upstream failure; empty for every other error.
pub fn upstream_headers(&self) -> &[(String, String)] {
match self {
Self::Llm(error) => error.upstream_headers(),
Self::Http(error) => error.upstream_headers(),
_ => &[],
}
}
/// A read of a streamed body timed out after the response head arrived.
pub fn body_read_timeout(&self) -> bool {
match self {
Self::Llm(error) => error.is_body_read_timeout(),
Self::Http(error) => error.is_body_read_timeout(),
_ => false,
}
}
/// `Some(connect)` when the call timed out, where `connect` says the connection was never
/// established; `None` for every other error.
pub fn timeout(&self) -> Option<bool> {
match self {
Self::Llm(error) => error.is_timeout(),
Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
_ => None,
}
}
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match self {
Self::Llm(error) => error.upstream_response(),

7
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct EnabledThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
/// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
#[serde(
default,
skip_serializing_if = "Option::is_none",
deserialize_with = "crate::recognized::present"
)]
pub budget_tokens: Option<Recognized<u64>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display: Option<Recognized<ThinkingDisplay>>,

36
litellm-rust/crates/llms-types/src/formats/messages/response.rs
View file

@ -10,14 +10,29 @@ pub struct MessagesResponse {
pub content: Vec<Value>,
pub stop_reason: Option<String>,
pub stop_sequence: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
/// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
#[serde(
default,
deserialize_with = "present_value",
skip_serializing_if = "Option::is_none"
)]
pub usage: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(
default,
deserialize_with = "present_value",
skip_serializing_if = "Option::is_none"
)]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
fn present_value<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Value>, D::Error> {
<Value as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
}
#[cfg(test)]
mod tests {
use super::*;
@ -73,4 +88,21 @@ mod tests {
assert_eq!(body.get("usage").cloned(), usage);
assert_eq!(body.get("container").cloned(), container);
}
#[test]
fn explicit_null_usage_and_container_survive_a_round_trip() {
let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
"content": [], "stop_reason": null, "stop_sequence": null,
"usage": null, "container": null});
let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
}
#[test]
fn absent_usage_and_container_stay_absent() {
let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
"content": [], "stop_reason": null, "stop_sequence": null});
let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
}
}

293
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -124,12 +124,86 @@ pub enum AnthropicBeta {
ToolSearchTool20251019,
#[strum(serialize = "inline-tools-2026-09-15")]
InlineTools20260915,
#[strum(serialize = "afk-mode-2026-01-31")]
AfkMode20260131,
#[strum(serialize = "agent-memory-2026-07-22")]
AgentMemory20260722,
#[strum(serialize = "analytics-usage-cost-2026-08-24")]
AnalyticsUsageCost20260824,
#[strum(serialize = "cache-diagnosis-2026-04-07")]
CacheDiagnosis20260407,
#[strum(serialize = "ce-plugins-2026-09-01")]
CePlugins20260901,
#[strum(serialize = "ce-user-management-2026-07-13")]
CeUserManagement20260713,
#[strum(serialize = "code-execution-2025-05-22")]
CodeExecution20250522,
#[strum(serialize = "computer-use-2024-10-22")]
ComputerUse20241022,
#[strum(serialize = "dev-full-thinking-2025-05-14")]
DevFullThinking20250514,
#[strum(serialize = "dreaming-2026-04-21")]
Dreaming20260421,
#[strum(serialize = "experimental-claude-tag-2026-10-21")]
ExperimentalClaudeTag20261021,
#[strum(serialize = "extended-cache-ttl-2025-04-11")]
ExtendedCacheTtl20250411,
#[strum(serialize = "fallback-credit-2026-06-01")]
FallbackCredit20260601,
#[strum(serialize = "fallback-credit-2026-07-01")]
FallbackCredit20260701,
#[strum(serialize = "managed-agents-2026-04-01")]
ManagedAgents20260401,
#[strum(serialize = "mcp-client-2026-09-15")]
McpClient20260915,
#[strum(serialize = "mcp-tunnels-2026-06-22")]
McpTunnels20260622,
#[strum(serialize = "message-batches-2024-09-24")]
MessageBatches20240924,
#[strum(serialize = "message-threads-2026-08-12")]
MessageThreads20260812,
#[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
MidConversationSystemClearAt20260821,
#[strum(serialize = "model-context-window-exceeded-2025-08-26")]
ModelContextWindowExceeded20250826,
#[strum(serialize = "output-300k-2026-03-24")]
Output300k20260324,
#[strum(serialize = "pdfs-2024-09-25")]
Pdfs20240925,
#[strum(serialize = "prompt-caching-2024-07-31")]
PromptCaching20240731,
#[strum(serialize = "redact-thinking-2026-02-12")]
RedactThinking20260212,
#[strum(serialize = "server-side-fallback-2026-06-01")]
ServerSideFallback20260601,
#[strum(serialize = "server-side-fallback-2026-07-01")]
ServerSideFallback20260701,
#[strum(serialize = "spend-limit-reads-2026-09-26")]
SpendLimitReads20260926,
#[strum(serialize = "structured-outputs-2025-12-15")]
StructuredOutputs20251215,
#[strum(serialize = "task-budgets-2026-03-13")]
TaskBudgets20260313,
#[strum(serialize = "telemetry-destinations-2026-08-11")]
TelemetryDestinations20260811,
#[strum(serialize = "thinking-token-count-2026-05-13")]
ThinkingTokenCount20260513,
#[strum(serialize = "token-counting-2024-11-01")]
TokenCounting20241101,
#[strum(serialize = "user-profiles-2026-03-24")]
UserProfiles20260324,
#[strum(serialize = "user-profiles-2026-08-18")]
UserProfiles20260818,
#[strum(serialize = "user-profiles-2026-09-04")]
UserProfiles20260904,
#[strum(serialize = "webhooks-2026-10-07")]
Webhooks20261007,
#[strum(default, transparent)]
Other(String),
}
impl AnthropicBeta {
pub const KNOWN: [Self; 40] = [
pub const KNOWN: [Self; 77] = [
Self::Oauth20250420,
Self::WebFetch20250910,
Self::WebSearch20250305,
@ -170,6 +244,43 @@ impl AnthropicBeta {
Self::ToolExamples20251029,
Self::ToolSearchTool20251019,
Self::InlineTools20260915,
Self::AfkMode20260131,
Self::AgentMemory20260722,
Self::AnalyticsUsageCost20260824,
Self::CacheDiagnosis20260407,
Self::CePlugins20260901,
Self::CeUserManagement20260713,
Self::CodeExecution20250522,
Self::ComputerUse20241022,
Self::DevFullThinking20250514,
Self::Dreaming20260421,
Self::ExperimentalClaudeTag20261021,
Self::ExtendedCacheTtl20250411,
Self::FallbackCredit20260601,
Self::FallbackCredit20260701,
Self::ManagedAgents20260401,
Self::McpClient20260915,
Self::McpTunnels20260622,
Self::MessageBatches20240924,
Self::MessageThreads20260812,
Self::MidConversationSystemClearAt20260821,
Self::ModelContextWindowExceeded20250826,
Self::Output300k20260324,
Self::Pdfs20240925,
Self::PromptCaching20240731,
Self::RedactThinking20260212,
Self::ServerSideFallback20260601,
Self::ServerSideFallback20260701,
Self::SpendLimitReads20260926,
Self::StructuredOutputs20251215,
Self::TaskBudgets20260313,
Self::TelemetryDestinations20260811,
Self::ThinkingTokenCount20260513,
Self::TokenCounting20241101,
Self::UserProfiles20260324,
Self::UserProfiles20260818,
Self::UserProfiles20260904,
Self::Webhooks20261007,
];
pub fn as_str(&self) -> &str {
@ -195,10 +306,9 @@ impl AnthropicBeta {
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 => {
Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -207,98 +317,99 @@ impl AnthropicBeta {
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260904 => &[
Self::AfkMode20260131
| Self::CePlugins20260901
| Self::CeUserManagement20260713
| Self::Compact20260904
| Self::Dreaming20260421
| Self::ExperimentalClaudeTag20261021
| Self::ExtendedCacheTtl20250411
| Self::FallbackCredit20260601
| Self::FallbackCredit20260701
| Self::InlineTools20260915
| Self::ManagedAgents20260401
| Self::McpTunnels20260622
| Self::MidConversationSystemClearAt20260821
| Self::ModelContextWindowExceeded20250826
| Self::Oauth20250420
| Self::Pdfs20240925
| Self::PromptCaching20240731
| Self::PromptCachingScope20260105
| Self::RedactThinking20260212
| Self::ServerSideFallback20260601
| Self::SpendLimitReads20260926
| Self::StructuredOutputs20251215
| Self::TaskBudgets20260313
| Self::TelemetryDestinations20260811
| Self::ThinkingTokenCount20260513
| Self::TokenCounting20241101
| Self::UserProfiles20260324
| Self::UserProfiles20260818
| Self::UserProfiles20260904 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Bash20241022
Self::AgentMemory20260722
| Self::Bash20241022
| Self::Bash20250124
| Self::DevFullThinking20250514
| Self::McpServers20251204
| Self::StructuredOutput20240301
| Self::TextEditor20241022
| Self::TextEditor20250124 => BetaProvider::VARIANTS,
Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::CodeExecution20250825
Self::AnalyticsUsageCost20260824
| Self::CacheDiagnosis20260407
| Self::CodeExecution20250522
| Self::CodeExecution20250825
| Self::ComputerUse20241022
| Self::FastMode20260201
| Self::FilesApi20250414
| Self::McpClient20250404
| Self::McpClient20251120
| Self::PromptCachingScope20260105
| Self::McpClient20260915
| Self::MessageBatches20240924
| Self::MessageThreads20260812
| Self::Output300k20260324
| Self::ServerSideFallback20260701
| Self::Skills20251002
| Self::WebFetch20250910 => &[
| Self::WebFetch20250910
| Self::Webhooks20261007 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ClaudeCode20250219
| Self::PerTurnControl20260701
| Self::ToolExamples20251029 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::Compact20260112
| Self::ContextManagement20250627
| Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::Effort20251124
| Self::FineGrainedToolStreaming20250514
| Self::ThinkingBindingControls20260801
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]
}
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::MidConversationToolChanges20260701 => &[
Self::InterleavedThinking20250514
| Self::Output128k20250219
| Self::TokenEfficientTools20250219
| Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
],
Self::PerTurnControl20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Databricks,
],
Self::InlineTools20260915 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
Self::Other(_) => BetaProvider::VARIANTS,
}
}
@ -451,7 +562,44 @@ mod tests {
AnthropicBeta::TokenEfficientTools20250219,
AnthropicBeta::ToolExamples20251029,
AnthropicBeta::ToolSearchTool20251019,
AnthropicBeta::InlineTools20260915
AnthropicBeta::InlineTools20260915,
AnthropicBeta::AfkMode20260131,
AnthropicBeta::AgentMemory20260722,
AnthropicBeta::AnalyticsUsageCost20260824,
AnthropicBeta::CacheDiagnosis20260407,
AnthropicBeta::CePlugins20260901,
AnthropicBeta::CeUserManagement20260713,
AnthropicBeta::CodeExecution20250522,
AnthropicBeta::ComputerUse20241022,
AnthropicBeta::DevFullThinking20250514,
AnthropicBeta::Dreaming20260421,
AnthropicBeta::ExperimentalClaudeTag20261021,
AnthropicBeta::ExtendedCacheTtl20250411,
AnthropicBeta::FallbackCredit20260601,
AnthropicBeta::FallbackCredit20260701,
AnthropicBeta::ManagedAgents20260401,
AnthropicBeta::McpClient20260915,
AnthropicBeta::McpTunnels20260622,
AnthropicBeta::MessageBatches20240924,
AnthropicBeta::MessageThreads20260812,
AnthropicBeta::MidConversationSystemClearAt20260821,
AnthropicBeta::ModelContextWindowExceeded20250826,
AnthropicBeta::Output300k20260324,
AnthropicBeta::Pdfs20240925,
AnthropicBeta::PromptCaching20240731,
AnthropicBeta::RedactThinking20260212,
AnthropicBeta::ServerSideFallback20260601,
AnthropicBeta::ServerSideFallback20260701,
AnthropicBeta::SpendLimitReads20260926,
AnthropicBeta::StructuredOutputs20251215,
AnthropicBeta::TaskBudgets20260313,
AnthropicBeta::TelemetryDestinations20260811,
AnthropicBeta::ThinkingTokenCount20260513,
AnthropicBeta::TokenCounting20241101,
AnthropicBeta::UserProfiles20260324,
AnthropicBeta::UserProfiles20260818,
AnthropicBeta::UserProfiles20260904,
AnthropicBeta::Webhooks20261007
)]
beta: AnthropicBeta,
) {
@ -551,7 +699,7 @@ mod tests {
BetaProvider::Anthropic,
Some(AnthropicBeta::Oauth20250420)
)]
#[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
#[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
#[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
fn on_resolves_known_spellings_held_as_other(
#[case] raw: &str,
@ -580,12 +728,11 @@ mod tests {
#[test]
fn unknown_values_are_kept_verbatim() {
let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
assert_eq!(
parsed,
AnthropicBeta::Other("claude-code-20250219".to_string())
let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
assert!(
matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
);
assert_eq!(parsed.to_string(), "claude-code-20250219");
assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
}
#[test]

10
litellm-rust/crates/llms-types/src/recognized.rs
View file

@ -7,6 +7,16 @@ pub enum Recognized<T> {
Unrecognized(Value),
}
/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay
/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`.
pub fn present<'de, D, T>(deserializer: D) -> Result<Option<Recognized<T>>, D::Error>
where
D: serde::Deserializer<'de>,
Recognized<T>: serde::Deserialize<'de>,
{
<Recognized<T> as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
}
impl<T> Recognized<T> {
pub fn known(&self) -> Option<&T> {
match self {

5
litellm-rust/crates/llms/src/anthropic/beta_headers.rs
View file

@ -96,7 +96,10 @@ mod tests {
}
#[rstest]
#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
#[case::azure_ai(
BetaProvider::AzureAi,
"advisor-tool-2026-03-01,example-beta-2099-01-01"
)]
#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
#[case::vertex_ai(BetaProvider::VertexAi, " , ")]
fn filter_removes_header_when_nothing_survives(

57
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -154,11 +154,42 @@ pub fn complete_anthropic_url(
) -> String {
let api_base = resolve_anthropic_api_base(api_base, env_lookup);
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH) {
ensure_api_route(&api_base, MESSAGES_PATH)
}
/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds.
fn url_path(url: &str) -> &str {
let before_fragment = url.split('#').next().unwrap_or_default();
let before_query = before_fragment.split('?').next().unwrap_or_default();
match before_query.split_once("://") {
Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]),
None => before_query,
}
}
/// Python's `_strip_trailing_slashes_keeping_the_scheme`.
fn strip_trailing_slashes(api_base: &str) -> String {
let stripped = api_base.trim_end_matches('/');
if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') {
return api_base.to_string();
}
format!("{api_base}{MESSAGES_PATH}")
match stripped.split_once('?') {
Some((before, query)) if url_path(stripped).ends_with('/') => {
format!("{}?{query}", before.trim_end_matches('/'))
}
_ => stripped.to_string(),
}
}
/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as
/// it is, query string included; anything else gets the route appended to the whole string.
fn ensure_api_route(api_base: &str, route: &str) -> String {
let base = strip_trailing_slashes(api_base);
let route = route.trim_start_matches('/');
if url_path(&base).ends_with(&format!("/{route}")) {
return base;
}
format!("{}/{route}", strip_trailing_slashes(&base))
}
pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {
@ -1922,4 +1953,24 @@ mod tests {
expected
);
}
#[rstest]
#[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")]
#[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")]
#[case::route_present(
"https://api.example.com/v1/messages/",
"https://api.example.com/v1/messages"
)]
#[case::route_and_query(
"https://api.example.com/v1/messages?beta=true",
"https://api.example.com/v1/messages?beta=true"
)]
#[case::query_without_route(
"https://api.example.com/?x=1",
"https://api.example.com?x=1/v1/messages"
)]
#[case::scheme_only("https://", "https:///v1/messages")]
fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) {
assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url);
}
}

233
litellm-rust/crates/llms/src/anthropic/messages/thinking.rs
View file

@ -2,8 +2,8 @@ use litellm_llms_types::{
formats::{
chat_completions::ReasoningEffort,
messages::{
EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig,
ThinkingDisplay,
AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
ThinkingConfig, ThinkingDisplay,
},
},
recognized::Recognized,
@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti
fn effort_for_budget(
budgets: &ThinkingBudgets,
budget_tokens: u64,
budget_tokens: i64,
capabilities: &MessagesModelCapabilities,
) -> EffortLevel {
if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh {
let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier);
if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh {
return EffortLevel::Xhigh;
}
if budget_tokens >= budgets.high {
if reaches(budgets.high) {
return EffortLevel::High;
}
if budget_tokens >= budgets.medium {
if reaches(budgets.medium) {
return EffortLevel::Medium;
}
EffortLevel::Low
@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) -
}
}
/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string
/// as a budget, and nothing else.
fn numeric_budget_tokens(budget: &Recognized<u64>) -> Option<i64> {
let value = match budget {
Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)),
Recognized::Unrecognized(value) => value,
};
match value {
Value::Number(number) => number
.as_i64()
.or_else(|| number.as_f64().map(|float| float.trunc() as i64)),
Value::String(text) => python_int(text),
Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None,
}
}
/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits.
fn python_int(text: &str) -> Option<i64> {
let trimmed = text.trim();
let digits = trimmed.trim_start_matches(['+', '-']);
let well_formed = trimmed.len() - digits.len() <= 1
&& !digits.is_empty()
&& !digits.starts_with('_')
&& !digits.ends_with('_')
&& !digits.contains("__")
&& digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_');
if !well_formed {
return None;
}
let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect();
cleaned.parse::<i64>().ok().or_else(|| {
Some(if cleaned.starts_with('-') {
i64::MIN
} else {
i64::MAX
})
})
}
fn python_type_name(value: &Value) -> &'static str {
match value {
Value::Null => "NoneType",
Value::Bool(_) => "bool",
Value::Number(number) if number.is_f64() => "float",
Value::Number(_) => "int",
Value::String(_) => "str",
Value::Array(_) => "list",
Value::Object(_) => "dict",
}
}
fn kept_display(
display: Option<&Recognized<ThinkingDisplay>>,
context: &ThinkingContext,
) -> Option<Recognized<ThinkingDisplay>> {
match display? {
Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => {
display.cloned()
}
Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => {
display.cloned()
}
Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None,
}
}
/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive
/// keys always, and any other key unless `drop_params` is set.
fn carried_thinking_keys(
extra: &serde_json::Map<String, Value>,
context: &ThinkingContext,
) -> serde_json::Map<String, Value> {
extra
.iter()
.filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params)
.map(|(key, value)| (key.clone(), value.clone()))
.collect()
}
/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`.
fn translate_legacy_thinking_for_adaptive_model(
request: MessagesRequest,
context: &ThinkingContext,
) -> MessagesRequest {
) -> Result<MessagesRequest, Error> {
let capabilities = &context.capabilities;
if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking {
return request;
return Ok(request);
}
let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else {
return request;
return Ok(request);
};
let budget = enabled
.budget_tokens
.as_ref()
.and_then(Recognized::known)
.copied()
.unwrap_or(0);
let level = effort_for_budget(&context.budgets, budget, capabilities);
MessagesRequest {
let budget = match &enabled.budget_tokens {
None => Some(0),
Some(budget) => numeric_budget_tokens(budget),
};
if budget.is_none() && !context.drop_params {
let actual = match &enabled.budget_tokens {
Some(Recognized::Unrecognized(value)) => python_type_name(value),
Some(Recognized::Known(_)) | None => "int",
};
return Err(Error::InvalidRequest(
crate::ErrorDetail::NonNumericThinkingBudget { actual },
));
}
let thinking = ThinkingConfig::Adaptive(AdaptiveThinking {
display: kept_display(enabled.display.as_ref(), context),
extra: carried_thinking_keys(&enabled.extra, context),
});
let output_config = match budget {
Some(budget) => with_default_effort(
request.params.output_config,
effort_for_budget(&context.budgets, budget, capabilities),
),
None => request.params.output_config,
};
Ok(MessagesRequest {
params: MessagesOptionalParams {
thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))),
output_config: with_default_effort(request.params.output_config, level),
thinking: Some(Recognized::Known(thinking)),
output_config,
..request.params
},
..request
}
})
}
fn translate_adaptive_effort_for_non_adaptive_model(
@ -337,7 +435,7 @@ pub fn translate_thinking(
) -> Result<MessagesRequest, Error> {
let request = translate_reasoning_effort(request, context)?;
let request = drop_disabled_thinking(request, context);
let request = translate_legacy_thinking_for_adaptive_model(request, context);
let request = translate_legacy_thinking_for_adaptive_model(request, context)?;
let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?;
Ok(drop_incompatible_temperature_for_thinking(request, context))
}
@ -363,6 +461,7 @@ mod tests {
ThinkingContext {
capabilities,
budgets: ThinkingBudgets::default(),
..ThinkingContext::default()
}
}
@ -1239,10 +1338,102 @@ mod tests {
let context = ThinkingContext {
capabilities,
budgets: overridden_budgets(overrides),
..ThinkingContext::default()
};
assert_eq!(
translate_thinking(request(input), &context),
Ok(request(expected))
);
}
use serde_json::json;
fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext {
ThinkingContext {
drop_params,
keeps_display_updates,
..context(opus_4_7())
}
}
#[rstest]
#[case::float_budget(json!(8192.0), "xhigh")]
#[case::float_below_a_tier(json!(1023.9), "low")]
#[case::string_budget(json!(" 16_000 "), "xhigh")]
#[case::negative_budget(json!(-5), "low")]
fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) {
let translated = translate_thinking(
request(
json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}),
),
&legacy_context(false, false),
);
assert_eq!(
translated,
Ok(request(json!({
"max_tokens": 64000,
"thinking": {"type": "adaptive"},
"output_config": {"effort": effort},
})))
);
}
#[rstest]
#[case::text(json!("lots"), "str")]
#[case::list(json!([1]), "list")]
#[case::null(Value::Null, "NoneType")]
#[case::boolean(json!(true), "bool")]
fn a_non_numeric_legacy_budget_is_rejected_unless_dropped(
#[case] budget: Value,
#[case] actual: &'static str,
) {
let fields =
json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}});
assert_eq!(
translate_thinking(request(fields.clone()), &legacy_context(false, false)),
Err(Error::InvalidRequest(
crate::ErrorDetail::NonNumericThinkingBudget { actual }
))
);
assert_eq!(
translate_thinking(request(fields), &legacy_context(true, false)),
Ok(request(
json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}})
))
);
}
#[rstest]
#[case::summarized("summarized", false, Some("summarized"))]
#[case::updates_kept("updates", true, Some("updates"))]
#[case::updates_dropped("updates", false, None)]
#[case::unknown("verbose", true, None)]
fn legacy_display_survives_only_where_python_keeps_it(
#[case] display: &str,
#[case] keeps_display_updates: bool,
#[case] kept: Option<&str>,
) {
let translated = translate_thinking(
request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})),
&legacy_context(false, keeps_display_updates),
)
.unwrap();
let thinking = serde_json::to_value(translated.params.thinking).unwrap();
assert_eq!(thinking.get("display").and_then(Value::as_str), kept);
}
#[rstest]
#[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))]
#[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))]
fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) {
let translated = translate_thinking(
request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})),
&legacy_context(drop_params, false),
)
.unwrap();
assert_eq!(
serde_json::to_value(translated.params.thinking).unwrap(),
expected
);
}
}

28
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
transform_messages_request(request, context)
transform_messages_request(request, &context.keeping_display_updates())
}
fn secret_names(&self) -> &'static [&'static str] {
@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
uses_thinking_block_binding(params)
.then_some(AnthropicBeta::ThinkingBindingControls20260801),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`.
fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool {
match params.thinking.as_ref() {
Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Unrecognized(Value::Object(thinking))) => {
thinking.contains_key("block_binding")
}
Some(Recognized::Unrecognized(_)) | None => false,
}
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error
})
}
fn drop_unsupported_params(
/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`.
pub fn drop_unsupported_params(
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
@ -838,6 +860,8 @@ mod tests {
low: 2000,
..ThinkingBudgets::default()
},
drop_params: true,
keeps_display_updates: false,
},
drop_params: true,
}

2
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
.capabilities
.supports_mid_conversation_system,
)),
context,
&context.keeping_display_updates(),
)
}

19
litellm-rust/crates/llms/src/base_llm/messages/context.rs
View file

@ -110,6 +110,11 @@ impl ThinkingBudgets {
pub struct ThinkingContext {
pub capabilities: MessagesModelCapabilities,
pub budgets: ThinkingBudgets,
/// `drop_params` as Python's thinking translation reads it.
pub drop_params: bool,
/// The provider accepts `display: "updates"` on thinking translated from the legacy shape
/// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`).
pub keeps_display_updates: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct MessagesTransformContext {
@ -130,8 +135,22 @@ impl MessagesTransformContext {
thinking: ThinkingContext {
capabilities,
budgets: ThinkingBudgets::from_lookup(env),
drop_params,
keeps_display_updates: false,
},
drop_params,
}
}
/// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`.
#[must_use]
pub fn keeping_display_updates(&self) -> Self {
Self {
thinking: ThinkingContext {
keeps_display_updates: true,
..self.thinking
},
..*self
}
}
}

22
litellm-rust/crates/llms/src/error.rs
View file

@ -92,6 +92,24 @@ impl Error {
}
}
pub fn upstream_headers(&self) -> &[(String, String)] {
match self {
Self::Http(error) => error.upstream_headers(),
_ => &[],
}
}
pub fn is_body_read_timeout(&self) -> bool {
matches!(self, Self::Http(error) if error.is_body_read_timeout())
}
pub fn is_timeout(&self) -> Option<bool> {
match self {
Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
_ => None,
}
}
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match self {
Self::Http(error) => error.upstream_response(),
@ -158,6 +176,10 @@ pub enum ErrorDetail {
value: String,
hint: String,
},
#[error(
"thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy"
)]
NonNumericThinkingBudget { actual: &'static str },
#[error(
"Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params"
)]

2
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = transform_messages_request(request, context)?;
let request = transform_messages_request(request, &context.keeping_display_updates())?;
let request = strip_cache_control_scope(normalize_system_role_messages(
request,
context

3
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -186,6 +186,9 @@ pub struct ModelInfoFields {
pub supports_image_size: Option<bool>,
pub supported_audio_formats: Option<Value>,
pub token_count_framing: Option<i64>,
pub token_count_tool_framing: Option<i64>,
pub token_count_responses_framing: Option<i64>,
pub token_count_responses_tool_framing: Option<i64>,
pub token_count_model: Option<Value>,
pub token_count_tokenizer: Option<Value>,
pub vertex_ai_audio_api: Option<Value>,

6
litellm-rust/crates/model-catalog/src/vocabulary.rs
View file

@ -93,6 +93,12 @@ pub enum Endpoint {
Realtime,
#[serde(rename = "/v1/realtime/transcription_sessions")]
RealtimeTranscriptionSessions,
#[serde(rename = "/v1/realtime/translations")]
RealtimeTranslations,
#[serde(rename = "/v1/realtime/translations/calls")]
RealtimeTranslationCalls,
#[serde(rename = "/v1/realtime/translations/client_secrets")]
RealtimeTranslationClientSecrets,
#[serde(rename = "/v1/videos")]
Videos,
#[serde(rename = "/v1/ocr")]

23
litellm-rust/crates/model-catalog/src/wire/attributes.rs
View file

@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
use crate::vocabulary::{
AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
};
use crate::wire::lax::Count;
use crate::wire::lax::{Count, NumericCount};
macro_rules! attributes {
($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@ -104,9 +104,14 @@ attributes! {
/// Cloud regions the model is available in ('global' or region ids).
supported_regions: Vec<String>,
thinking_always_on: bool,
token_count_framing: u64 as Count,
/// Whether the model thinks when the request carries no thinking param.
thinking_on_by_default: bool,
token_count_framing: u64 as NumericCount,
token_count_model: TokenCountModel,
token_count_responses_framing: u64 as NumericCount,
token_count_responses_tool_framing: u64 as NumericCount,
token_count_tokenizer: TokenCountTokenizer,
token_count_tool_framing: u64 as NumericCount,
/// Provider default tokens-per-minute limit.
tpm: u64 as Count,
use_openai_responses_path: bool,
@ -298,10 +303,24 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
#[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
#[case::framing("token_count_framing", json!(12), true)]
#[case::framing_written_as_text("token_count_framing", json!("12"), false)]
#[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
#[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
#[case::tool_framing("token_count_tool_framing", json!(214), true)]
#[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
#[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
#[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
#[case::responses_framing("token_count_responses_framing", json!(38), true)]
#[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
#[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
#[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
fn each_attribute_reads_its_own_type(
#[case] key: &str,
#[case] value: Value,

55
litellm-rust/crates/model-catalog/src/wire/lax.rs
View file

@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
pub(crate) struct Count;
pub(crate) struct NumericCount;
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
deserializer.deserialize_any(Self)
@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
}
}
impl<'de> DeserializeAs<'de, u64> for NumericCount {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
deserializer.deserialize_any(Self)
}
}
impl Visitor<'_> for NumericCount {
type Value = u64;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a whole number of zero or more")
}
fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
Ok(value)
}
fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
}
fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
integral_float(value)
.and_then(|whole| u64::try_from(whole).ok())
.ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
}
}
fn integer_string(value: &str) -> Option<i64> {
let integer = match value.split_once('.') {
Some((integer, fraction)) => {
@ -99,6 +129,12 @@ mod tests {
.map(DeserializeAsWrap::into_inner)
}
fn numeric_count(value: Value) -> Option<u64> {
DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
.ok()
.map(DeserializeAsWrap::into_inner)
}
use serde::Deserialize;
#[rstest]
@ -125,4 +161,23 @@ mod tests {
) {
assert_eq!(count(written), expected);
}
#[rstest]
#[case::integer(json!(214), Some(214))]
#[case::zero(json!(0), Some(0))]
#[case::whole_float(json!(214.0), Some(214))]
#[case::negative_zero(json!(-0.0), Some(0))]
#[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
#[case::negative(json!(-1), None)]
#[case::negative_whole_float(json!(-1.0), None)]
#[case::fraction(json!(1.5), None)]
#[case::string(json!("214"), None)]
#[case::boolean(json!(true), None)]
#[case::list(json!([214]), None)]
fn numeric_counts_accept_only_whole_json_numbers(
#[case] written: Value,
#[case] expected: Option<u64>,
) {
assert_eq!(numeric_count(written), expected);
}
}

59
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -23,6 +23,17 @@ pyo3::create_exception!(
);
const MESSAGES_REQUEST_ERROR: &str = "messages_request_error";
/// The rejection is a parameter the model does not support, which Python raises as
/// `UnsupportedParamsError`.
const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param";
/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or
/// `Body` for a streamed body read that timed out after the head arrived.
const MESSAGES_TIMEOUT: &str = "messages_timeout";
/// No credential reached the provider, which Python raises as `AuthenticationError`.
const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential";
/// Python refuses the request only once its logger knows the resolved model and provider (the
/// provider config's request transform), so the failure logs them.
const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection";
pub(crate) fn upstream_error(
py: Python<'_>,
@ -78,17 +89,57 @@ pub(crate) fn route_failure(
return Ok(raised);
}
if let Some((status, body)) = error.upstream_response() {
return public(upstream_error(py, status, body, &HeaderMap::new())?);
let headers = error
.upstream_headers()
.iter()
.filter_map(|(name, value)| {
Some((
reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?,
reqwest::header::HeaderValue::from_str(value).ok()?,
))
})
.collect::<HeaderMap>();
return public(upstream_error(py, status, body, &headers)?);
}
let native = match (marker, error.rejected_request()) {
(RequestMarker::Messages, Some(rejection)) => {
let native = PyValueError::new_err(Report::new(rejection).to_string());
native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?;
let value = native.value(py);
value.setattr(MESSAGES_REQUEST_ERROR, true)?;
if matches!(
rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
Some(
litellm_llms::ErrorDetail::UnsupportedParameter { .. }
| litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }
)
) {
value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?;
}
if matches!(
rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. })
) {
value.setattr(MESSAGES_LATE_REJECTION, true)?;
}
native
}
(RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => {
by_fault(error.is_request(), &error)
(RequestMarker::Messages, None) => {
let native = by_fault(error.is_request(), &error);
let value = native.value(py);
if let Some(connect) = error.timeout() {
let phase = match (connect, error.body_read_timeout()) {
(true, _) => "Connection",
(false, true) => "Body",
(false, false) => "Read",
};
value.setattr(MESSAGES_TIMEOUT, phase)?;
}
if error.is_missing_credential() {
value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
}
native
}
(RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
};
public(native)
}

21
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -34,11 +34,9 @@ fn merge_headers(
forwarded: Option<Map<String, Value>>,
extra_headers: Option<Map<String, Value>>,
) -> Option<Map<String, Value>> {
let merged: Map<String, Value> = forwarded
.into_iter()
.flatten()
.chain(extra_headers.into_iter().flatten())
.collect();
let merged = litellm_inference_messages::merge_messages_header_sources(
forwarded.into_iter().chain(extra_headers),
);
(!merged.is_empty()).then_some(merged)
}
@ -107,10 +105,19 @@ impl MessagesPythonHost {
.map(|value| python_timeout_seconds(py, value.unbind()))
.transpose()?
.flatten();
let custom_llm_provider = string("custom_llm_provider")?;
let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
let api_key = string("api_key")?;
let api_base = string("api_base")?;
// Python's handler resolves a bare model name (`claude-sonnet-4-5`) through
// `get_llm_provider`'s model lists, which the native prefix lookup does not carry.
let custom_llm_provider = match string("custom_llm_provider")? {
Some(provider) => Some(provider),
None => py
.import(ROUTE_HOST_MODULE)?
.getattr("inferred_provider")?
.call1((&model, api_base.as_deref()))?
.extract()?,
};
let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
let extra_headers = self.merged_headers(py, arguments)?;
let provider_specific_header = self.provider_specific_header(py, arguments)?;
let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;

13
litellm-rust/crates/python-bridge/src/routes/messages/mod.rs
View file

@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
)
}
/// Python's sync Messages handler raises for every native provider before it sends anything
/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies
/// that until both sides serve sync calls.
const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls";
#[pyfunction]
pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_messages(py, call, false)
#[pyo3(signature = (call))]
pub(crate) fn messages(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
drop(call);
Err(pyo3::exceptions::PyValueError::new_err(
SYNC_NOT_IMPLEMENTED,
))
}
#[pyfunction]

20
litellm/__init__.py
View file

@ -8,7 +8,7 @@ _apply_process_defaults()
import threading
import os
from collections.abc import Mapping, Sequence
from collections.abc import Sequence
from typing import (
Any,
Callable,
@ -1562,7 +1562,9 @@ from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
arealtime_calls,
arealtime_translation_calls as arealtime_translation_calls,
)
from .responses.main import _aresponses_websocket
@ -1754,6 +1756,9 @@ if TYPE_CHECKING:
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)
from .llms.parallel_ai.decisions.transformation import (
ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
)
from .llms.openrouter.decisions.transformation import (
OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
)
@ -1808,6 +1813,9 @@ if TYPE_CHECKING:
from .llms.vertex_ai.rerank.transformation import (
VertexAIRerankConfig as VertexAIRerankConfig,
)
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.fireworks_ai.rerank.transformation import (
FireworksAIRerankConfig as FireworksAIRerankConfig,
)
@ -1853,9 +1861,6 @@ if TYPE_CHECKING:
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig as VertexGeminiConfig,
@ -2014,6 +2019,9 @@ if TYPE_CHECKING:
from .llms.xai.responses.transformation import (
XAIResponsesAPIConfig as XAIResponsesAPIConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
)
@ -2041,9 +2049,6 @@ if TYPE_CHECKING:
from .llms.gemini.interactions.transformation import (
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
OpenAIOSeriesConfig as OpenAIO1Config,
@ -2374,6 +2379,7 @@ if TYPE_CHECKING:
supports_web_search: Callable[..., bool]
supports_tool_search: Callable[..., bool]
supports_url_context: Callable[..., bool]
supports_continuation: Callable[..., bool]
supports_response_schema: Callable[..., bool]
supports_parallel_function_calling: Callable[..., bool]
supports_vision: Callable[..., bool]

12
litellm/_internal_context.py
View file

@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
internal_parent_request_id.reset(parent_token)
@contextmanager
def billed_separately() -> Generator[None]:
"""A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
token: Final = is_internal_call.set(False)
parent_token: Final = internal_parent_request_id.set(None)
try:
yield
finally:
is_internal_call.reset(token)
internal_parent_request_id.reset(parent_token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

4
litellm/_lazy_imports_registry.py
View file

@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
"supports_web_search",
"supports_tool_search",
"supports_url_context",
"supports_continuation",
"supports_response_schema",
"supports_parallel_function_calling",
"supports_vision",
@ -162,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"ParallelAIDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
@ -488,6 +490,7 @@ _UTILS_IMPORT_MAP: Final = {
"supports_web_search": (".utils", "supports_web_search"),
"supports_tool_search": (".utils", "supports_tool_search"),
"supports_url_context": (".utils", "supports_url_context"),
"supports_continuation": (".utils", "supports_continuation"),
"supports_response_schema": (".utils", "supports_response_schema"),
"supports_parallel_function_calling": (
".utils",
@ -738,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),

175
litellm/anthropic_beta_headers_config.json
View file

@ -37,7 +37,47 @@
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
"cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": "code-execution-2025-05-22",
"computer-use-2024-10-22": "computer-use-2024-10-22",
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"message-batches-2024-09-24": "message-batches-2024-09-24",
"message-threads-2026-08-12": "message-threads-2026-08-12",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"output-300k-2026-03-24": "output-300k-2026-03-24",
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04",
"webhooks-2026-10-07": "webhooks-2026-10-07",
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"azure_ai": {
"advisor-tool-2026-03-01": null,
@ -45,23 +85,21 @@
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"compact-2026-09-04": "compact-2026-09-04",
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@ -71,9 +109,54 @@
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
"web-search-2025-03-05": "web-search-2025-03-05",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
"cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": "code-execution-2025-05-22",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2024-10-22": "computer-use-2024-10-22",
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"message-batches-2024-09-24": "message-batches-2024-09-24",
"message-threads-2026-08-12": "message-threads-2026-08-12",
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"oauth-2025-04-20": "oauth-2025-04-20",
"output-300k-2026-03-24": "output-300k-2026-03-24",
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04",
"webhooks-2026-10-07": "webhooks-2026-10-07",
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"bedrock_converse": {
"advisor-tool-2026-03-01": null,
@ -196,34 +279,92 @@
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": null,
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": null,
"cache-diagnosis-2026-04-07": null,
"ce-plugins-2026-09-01": null,
"ce-user-management-2026-07-13": null,
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": null,
"compact-2026-09-04": null,
"computer-use-2024-10-22": null,
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": null,
"experimental-claude-tag-2026-10-21": null,
"extended-cache-ttl-2025-04-11": null,
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"managed-agents-2026-04-01": null,
"mcp-client-2026-09-15": null,
"mcp-tunnels-2026-06-22": null,
"message-batches-2024-09-24": null,
"message-threads-2026-08-12": null,
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"oauth-2025-04-20": null,
"output-300k-2026-03-24": null,
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": null,
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": null,
"spend-limit-reads-2026-09-26": null,
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": null,
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"user-profiles-2026-03-24": null,
"user-profiles-2026-08-18": null,
"user-profiles-2026-09-04": null,
"webhooks-2026-10-07": null,
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": null,
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"vertex_ai_header_only": {
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"compact-2026-09-04": "compact-2026-09-04",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"oauth-2025-04-20": "oauth-2025-04-20",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04"
},
"databricks": {
"advisor-tool-2026-03-01": null,

50
litellm/anthropic_beta_headers_manager.py
View file

@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final, Literal, TypeAlias
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
UnlistedBetas: TypeAlias = Literal["drop", "forward"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
@ -279,12 +281,13 @@ def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
unlisted: UnlistedBetas = "drop",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
This function:
1. Only allows headers that are present in the provider's mapping keys
1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
2. Filters out headers with null values (unsupported)
3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
@ -292,6 +295,7 @@ def filter_and_transform_beta_headers(
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
Returns:
List of filtered and transformed beta headers for the provider
@ -306,6 +310,10 @@ def filter_and_transform_beta_headers(
for header in beta_headers:
header = header.strip()
if header not in provider_mapping and unlisted == "forward":
filtered_headers.add(header)
continue
# Check if header is in the mapping
if header not in provider_mapping:
verbose_logger.debug(
@ -387,9 +395,38 @@ def get_provider_beta_header(
return provider_mapping[anthropic_beta_header]
FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
"services.ai.azure.com",
"openai.azure.com",
"cognitiveservices.azure.com",
)
def is_foundry_api_base(api_base: str | None) -> bool:
"""Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
if not api_base:
return False
host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
"""
Pick the beta allowlist row for one deployment, not one provider string.
An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
"""
if provider == "anthropic" and is_foundry_api_base(api_base):
return "azure_ai"
return provider
def update_headers_with_filtered_beta(
headers: dict,
provider: str,
api_base: str | None = None,
unlisted: UnlistedBetas = "drop",
) -> dict:
"""
Update headers dict by filtering and transforming anthropic-beta header values.
@ -398,6 +435,8 @@ def update_headers_with_filtered_beta(
Args:
headers: Request headers dict (will be modified in place)
provider: Provider name
api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
unlisted: Whether a beta missing from the row is dropped or forwarded as sent
Returns:
Updated headers dict
@ -412,7 +451,8 @@ def update_headers_with_filtered_beta(
# Filter and transform based on provider
filtered_beta_values: Final = filter_and_transform_beta_headers(
beta_headers=beta_values,
provider=provider,
provider=resolve_beta_provider(provider, api_base),
unlisted=unlisted,
)
# Update or remove the header
@ -429,6 +469,7 @@ def update_request_with_filtered_beta(
headers: dict,
request_data: dict,
provider: str,
api_base: str | None = None,
) -> tuple[dict[str, str], dict[str, object]]:
"""
Update both headers and request body beta fields based on provider support.
@ -438,11 +479,12 @@ def update_request_with_filtered_beta(
headers: Request headers dict (will be modified in place)
request_data: Request body dict (will be modified in place)
provider: Provider name
api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
Returns:
Tuple of (updated headers, updated request_data)
"""
headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
existing_body_betas: Final = request_data.get("anthropic_beta")
if not existing_body_betas:
@ -450,7 +492,7 @@ def update_request_with_filtered_beta(
filtered_body_betas: Final = filter_and_transform_beta_headers(
beta_headers=existing_body_betas,
provider=provider,
provider=resolve_beta_provider(provider, api_base),
placement="body",
)

10
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
# Source: https://platform.claude.com/docs/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
400: "invalid_request_error",
401: "authentication_error",
@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
status_code: int,
message: str,
request_id: str | None = None,
error_type: AnthropicErrorType | None = None,
) -> AnthropicErrorResponse:
"""
Create an Anthropic-formatted error response dict.
@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
"request_id": "req_..."
}
"""
error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
response: Final[AnthropicErrorResponse] = {
"type": "error",
"error": {
"type": error_type,
"type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
"message": message,
},
}
@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
status_code: int,
raw_message: str,
request_id: str | None = None,
error_type: AnthropicErrorType | None = None,
) -> AnthropicErrorResponse:
"""
Transform an error message to Anthropic format.
@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
status_code: HTTP status code
raw_message: Raw error message (may be JSON string or plain text)
request_id: Optional request ID to include
error_type: Anthropic error type to use instead of the one the status maps to
Returns:
AnthropicErrorResponse dict
@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
status_code=status_code,
message=message,
request_id=request_id,
error_type=error_type,
)

6
litellm/anthropic_interface/exceptions/exceptions.py
View file

@ -6,15 +6,18 @@ from typing import Literal
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
# Known Anthropic error types
# Source: https://docs.anthropic.com/en/api/errors
# Source: https://platform.claude.com/docs/en/api/errors
AnthropicErrorType = Literal[
"invalid_request_error",
"authentication_error",
"billing_error",
"permission_error",
"not_found_error",
"conflict_error",
"request_too_large",
"rate_limit_error",
"api_error",
"timeout_error",
"overloaded_error",
]
@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
type: AnthropicErrorType
message: str
provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
litellm_error_type: NotRequired[ReadOnly[str]]
litellm_call_id: NotRequired[ReadOnly[str]]

21
litellm/caching/caching.py
View file

@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
raw_extra_body: Final = kwargs.get("extra_body")
if not isinstance(raw_extra_body, dict):
return ""
extra_body: Final = cast( # cast-ok: isinstance leaves the key and value types unknown
Mapping[str, object], raw_extra_body
)
return "".join(
f"extra_body.{param}: {extra_body[param]}"
for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
if extra_body.get(param)
)
@contextmanager
def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
"""The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@ -429,7 +443,10 @@ class Cache:
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
cache_key
+ _keyed_provider_params_in_extra_body(kwargs)
+ reasoning_summary_material
+ tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(

2
litellm/caching/dual_cache.py
View file

@ -673,7 +673,7 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
self.redis_cache.flush_cache()
def delete_cache(self, key):
def delete_cache(self, key: str) -> None:
"""
Delete a key from the cache
"""

69
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import (
ResponseCustomToolCallOutputParam,
ResponseCustomToolCallParam,
)
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
from openai.types.responses.tool_param import FunctionToolParam
@ -34,10 +35,16 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
requested_phase,
)
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
prompt_cache_breakpoint_with_readable_mode,
responses_reasoning_items_from_thinking_blocks,
whole_prompt_cache_breakpoint_or_none,
with_prompt_cache_breakpoint,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@ -97,6 +104,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
)
def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
if target_trims_breakpoints:
return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
if not isinstance(value, dict):
return value
@ -775,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _normalize_tool_choice_for_responses_api(
self, tool_choice: _ToolChoiceT
) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
) -> (
_ToolChoiceT
| ToolChoiceFunctionParam
| ToolChoiceCustomParam
| ToolChoiceAllowedParam
| Literal["auto", "none", "required"]
):
"""Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
if not isinstance(tool_choice, dict):
return tool_choice
choice_type: Final = tool_choice.get("type")
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
return choice_type
if choice_type == "allowed_tools":
parsed: Final = parse_chat_allowed_tools_tool_choice(
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice
)
)
flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None
if flat is not None:
return flat
if choice_type not in ("function", "custom"):
return tool_choice
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@ -844,6 +872,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
keep_prompt_cache_breakpoints: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
@ -852,6 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
messages,
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_openai_history=replay_openai_history,
@ -868,6 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
@ -921,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content,
role,
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
}
)
@ -940,6 +972,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content,
"user", # Use "user" role to get input_* types
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
)
else:
# Fallback: convert unexpected types to input_text
@ -968,7 +1001,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": "assistant",
"content": self._convert_content_to_responses_format(
content, "assistant", drop_params=drop_params
content,
"assistant",
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
**_assistant_phase_field(msg, replay_openai_history, drop_params),
}
@ -1009,7 +1045,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, cast(str, role), drop_params=drop_params
content,
cast(str, role),
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
**_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
}
@ -1135,6 +1174,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
@ -1640,6 +1680,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
| None,
role: str,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
) -> list[dict[str, object]]:
"""Convert chat completion content to responses API format"""
from litellm.types.llms.openai import ChatCompletionImageObject
@ -1666,7 +1707,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if original_type == "text":
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(item.get("text", ""), role),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: text -> %s", converted)
@ -1679,14 +1722,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
cast(ChatCompletionImageObject, item), role
),
),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: image_url -> %s", converted)
elif original_type == "input_audio":
converted = with_prompt_cache_breakpoint(
{"type": "input_audio", "input_audio": item.get("input_audio")},
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: input_audio -> %s", converted)
@ -1702,7 +1749,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
_input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: file -> %s", converted)
@ -1726,7 +1775,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Default to input_text for unknown types
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(str(item.get("text", item)), role),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted)

6
litellm/constants.py
View file

@ -699,6 +699,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
"DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400
@ -968,7 +971,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
OPENAI_TRANSCRIPTION_PARAMS: Final = [
"language",
"languages",
"keywords",
"response_format",
"stream",
"timestamp_granularities",
]

297
litellm/cost_calculator.py
View file

@ -6,6 +6,7 @@ import time
from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -22,9 +23,14 @@ from litellm.constants import (
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
REALTIME_METERED_USAGE_KEY,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
from litellm.litellm_core_utils.decisions_pricing import (
decisions_pricing_fields,
decisions_request_price,
is_decisions_call_type,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
@ -319,12 +325,41 @@ def _get_additional_costs(
return None
def _transcription_usage_has_token_details(
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second",
"output_cost_per_second",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
"input_cost_per_token",
"output_cost_per_token",
]
_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
)
_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_second",
"output_cost_per_second",
)
def _transcription_uses_token_pricing(
model: str,
custom_llm_provider: str | None,
usage_block: Usage | None,
) -> bool:
if usage_block is None:
return False
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
if (
_declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
):
return False
prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@ -338,13 +373,38 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
if pricing_model is None or custom_llm_provider is None:
return False
transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
transcription_metadata: Final = (
as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
)
return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
def _transcription_duration_for_cost(
completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
) -> float:
hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
measured_duration: Final = normalized_audio_duration_seconds(
hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
)
if measured_duration is not None and measured_duration > 0:
return measured_duration
provider_duration: Final = _get_transcription_usage_duration(completion_response)
if (
provider_duration is not None
and provider_duration > 0
and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
):
return provider_duration
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return 0.0
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -604,6 +664,9 @@ def cost_per_token(
decisions_overrides: Final = decisions_pricing_fields(
litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
)
request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
if request_price is not None:
return request_price, 0.0
if decisions_overrides:
return generic_cost_per_token(
model=model,
@ -693,7 +756,7 @@ def cost_per_token(
vertex_location=vertex_location,
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -1275,6 +1338,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
)
def _get_transcription_usage_duration(completion_response: object) -> float | None:
response_fields: Final = as_str_mapping(completion_response)
usage_object: Final = (
response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
)
usage_fields: Final = as_str_mapping(usage_object)
usage_type: Final = (
usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
)
if usage_type != "duration":
return None
seconds: Final = (
usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
)
return normalized_audio_duration_seconds(seconds)
def _extraction_usage(usage: ExtractionUsage) -> Usage:
return Usage(
prompt_tokens=usage.input_tokens,
@ -2000,20 +2080,8 @@ def completion_cost(
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
# the response attribute (for verbose_json responses that
# naturally include duration from the provider).
_hidden = getattr(completion_response, "_hidden_params", {}) or {}
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
audio_transcription_file_duration = _transcription_duration_for_cost(
completion_response, model, requested_model, custom_llm_provider
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -2123,6 +2191,7 @@ def completion_cost(
litellm_logging_obj=litellm_logging_obj,
custom_pricing_model=selected_model if custom_pricing else None,
base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
deployment_models=(base_model, requested_model),
)
elif call_type == _MCP_CALL_TYPE:
from litellm.proxy.mcp_server.cost_calculator import (
@ -3401,8 +3470,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
def create_logging_realtime_object(
usage: Usage, results: OpenAIRealtimeStreamList
) -> LiteLLMRealtimeStreamLoggingObject:
translated_input_seconds, _ = _translation_session_seconds(results)
return LiteLLMRealtimeStreamLoggingObject(
usage=usage,
usage=usage.model_copy(update={"seconds": translated_input_seconds})
if translated_input_seconds > 0
else usage,
results=results,
)
@ -3487,6 +3559,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
class _LiveResponsePayload(BaseModel):
@ -3709,6 +3782,7 @@ def handle_realtime_stream_cost_calculation(
litellm_logging_obj: LitellmLoggingObject | None = None,
custom_pricing_model: str | None = None,
base_pricing_model: str | None = None,
deployment_models: Sequence[str | None] = (),
) -> float:
"""
Handles the cost calculation for realtime stream responses.
@ -3753,6 +3827,13 @@ def handle_realtime_stream_cost_calculation(
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
else 0.0
)
translation_cost: Final = handle_realtime_translation_cost_calculation(
results=results,
custom_llm_provider=custom_llm_provider,
litellm_model_name=litellm_model_name,
deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
session_model=received_model,
)
live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
live_session_started: Final = any(result["type"] == "session.started" for result in results)
live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@ -3809,12 +3890,19 @@ def handle_realtime_stream_cost_calculation(
input_cost: Final = input_cost_per_token + live_response_input_cost
output_cost: Final = output_cost_per_token + live_response_output_cost
total_cost: Final = (
input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
input_cost
+ output_cost
+ transcription_cost
+ translation_cost
+ live_voice_cost
+ audio_duration_cost
+ text_message_cost
)
additional_costs: Final = tuple(
(cost_name, cost)
for cost_name, cost in (
("transcription_cost", transcription_cost),
("translation_cost", translation_cost),
("live_voice_cost", live_voice_cost),
("audio_duration_cost", audio_duration_cost),
("text_message_cost", text_message_cost),
@ -3835,6 +3923,55 @@ def handle_realtime_stream_cost_calculation(
return total_cost
def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
usage_events: Final = tuple(
usage
for result in results
if (event := as_str_mapping(result)) is not None
and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
and (usage := as_str_mapping(event.get("usage"))) is not None
)
input_seconds: Final = sum(
seconds
for usage in usage_events
if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
)
output_seconds: Final = sum(
output_duration
for usage in usage_events
if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
)
return float(input_seconds), float(output_seconds)
def _translation_per_second_rates(
models: Sequence[str | None], custom_llm_provider: str
) -> tuple[float | None, float | None]:
model_infos: Final = tuple(
_lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
)
return (
_first_declared_transcription_rate(("input_cost_per_second",), model_infos),
_first_declared_transcription_rate(("output_cost_per_second",), model_infos),
)
def handle_realtime_translation_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
litellm_model_name: str,
deployment_pricing_models: Sequence[str | None] = (),
session_model: str | None = None,
) -> float:
input_seconds, output_seconds = _translation_session_seconds(results)
if input_seconds <= 0 and output_seconds <= 0:
return 0.0
input_rate, output_rate = _translation_per_second_rates(
(*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
)
return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
def handle_realtime_transcription_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
@ -3850,57 +3987,77 @@ def handle_realtime_transcription_cost_calculation(
- {"type": "duration", "seconds": <float>} → priced via input_cost_per_second
- {"type": "tokens", "input_tokens": ...} → priced via input/audio token cost
"""
completed_events: Final = [
cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
]
if not completed_events:
configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
clips: Final = tuple(
(as_str_mapping(result.get("usage")) or {}, configured)
for result, configured in zip(results, configured_models, strict=True)
if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
)
if not clips:
return 0.0
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
model_infos: Final = MappingProxyType(
{model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
)
override_info: Final = (
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
)
return sum(
(_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
0.0,
)
total_cost = 0.0
for event in completed_events:
usage = event.get("usage") or {}
total_cost += _transcription_usage_cost(usage, model_info, override_info)
return total_cost
_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
{"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
)
def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
value: Final = None if mapping is None else mapping.get(key)
return value if isinstance(value, str) and value else None
def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
def _session_transcription_model(result: Mapping[str, object]) -> str | None:
session: Final = _session_of(result)
return None if session is None else _configured_transcription_model(session)
def _newer_model(current: str | None, newer: str | None) -> str | None:
return newer or current
def _get_transcription_model_name_from_results(
results: OpenAIRealtimeStreamList,
) -> str | None:
"""Resolve the ASR model from a transcription_session.* / session.* event."""
for result in results:
if result.get("type") in (
"transcription_session.created",
"transcription_session.updated",
"session.created",
"session.updated",
):
session = cast(dict, result).get("session", {}) or {}
transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
"transcription", {}
) or session.get("input_audio_transcription", {})
model = (transcription or {}).get("model") or session.get("model")
if model:
return model
return None
return next(
(model for result in results if (model := _str_field(_session_of(result), "model"))),
None,
)
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "output_cost_per_second":
return info.get("output_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "output_cost_per_audio_token":
return info.get("output_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
@ -3909,7 +4066,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3927,15 +4084,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
)
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)
def _first_declared_transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
) -> float | None:
rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
return next((rate for rate in rates if rate is not None), None)
def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
return _first_declared_transcription_rate(keys, infos) or 0.0
def _transcription_usage_cost(
usage: dict,
usage: Mapping[str, object],
model_info: ModelInfo | None,
override_info: ModelInfo | None = None,
) -> float:
@ -3944,13 +4105,13 @@ def _transcription_usage_cost(
usage_type: Final = usage.get("type")
if usage_type == "duration":
seconds: Final = usage.get("seconds") or 0.0
seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
if usage_type == "tokens":
input_token_details: Final = usage.get("input_token_details") or {}
audio_tokens: Final = input_token_details.get("audio_tokens") or 0
text_tokens: Final = input_token_details.get("text_tokens") or 0
output_tokens: Final = usage.get("output_tokens") or 0
input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
audio_cost: Final = float(audio_tokens) * _transcription_rate(
("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
)

10
litellm/decisions/call.py
View file

@ -13,6 +13,7 @@ from typing_extensions import assert_never
import litellm
from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -555,7 +556,7 @@ def prepare_call(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or call_api_base, dynamic_api_key or api_key
)
except ValueError as error:
except (ValueError, httpx.InvalidURL) as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
if resolved_api_base is None:
raise litellm.BadRequestError(
@ -613,6 +614,9 @@ def prepare_call(
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
"input_cost_per_query": (
kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
),
},
custom_llm_provider=provider,
)
@ -787,7 +791,9 @@ def _formatted_response(
case DecisionsRequestBody():
return ir_to_systemone_response(caller_response, caller_request)
case OpenAIDecisionRequestBody():
return ir_to_openai_response(caller_response, caller_request, call.requested_model)
return ir_to_openai_response(
caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
)
case _:
assert_never(call.request)

11
litellm/images/main.py
View file

@ -28,7 +28,6 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.custom_llm import CustomLLM
from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
#################### Initialize provider clients ####################
@ -612,10 +611,7 @@ def image_generation(
)
elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider
# Get the Custom Handler
custom_handler: CustomLLM | None = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
if custom_handler is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@ -919,10 +915,7 @@ def image_edit(
# Check for custom provider
if custom_llm_provider in litellm._custom_providers:
custom_handler: CustomLLM | None = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
if custom_handler is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)

30
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -17,7 +17,7 @@ from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Se
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
@ -73,7 +73,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@ -335,6 +335,30 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
return _named_model_breakpoint_support(model)
class _DeclaredBaseModels(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
base_model: object = None
model_info: object = None
def names(self) -> tuple[str, ...]:
model_info: Final = _validated_object_mapping(self.model_info) or {}
candidates: Final = (self.base_model, model_info.get("base_model"))
return tuple(name for name in candidates if isinstance(name, str) and name)
def deployment_prompt_cache_breakpoint_support(
model: str, custom_llm_provider: str, litellm_params: object
) -> bool | None:
"""``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
supports: Final = (
openai_prompt_cache_breakpoint_support(name, custom_llm_provider)
for name in dict.fromkeys((*declared.names(), model))
)
return next((support for support in supports if support is not None), None)
def _named_model_breakpoint_support(model: str) -> bool | None:
bare_model: Final = model.rsplit("/", 1)[-1].lower()
if _O_SERIES_PATTERN.match(bare_model) is not None:
@ -342,7 +366,7 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
if version_match is None:
return None
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION

50
litellm/integrations/azure_storage/azure_storage.py
View file

@ -1,7 +1,8 @@
import asyncio
import os
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from functools import cache
from typing import Final
@ -15,6 +16,7 @@ from litellm.constants import (
AZURE_STORAGE_MSFT_VERSION,
)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
from litellm.llms.custom_httpx.http_handler import (
@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
return f"{name.translate(_ADLS_SAFE_NAME)}.json"
@dataclass(frozen=True, slots=True)
class AzureStorageLogQueueItem:
payload: StandardLoggingPayload
file_name: str
def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
payload_id: Final = payload.get("id")
return AzureStorageLogQueueItem(
payload=payload,
file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
)
@cache
def _cached_credential_chain_token_provider() -> Callable[[], str]:
return get_azure_ad_token_provider(
@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
self.log_queue: list[StandardLoggingPayload] = []
self.log_queue: list[AzureStorageLogQueueItem] = []
super().__init__(**kwargs, flush_lock=self.flush_lock)
except Exception as e:
verbose_logger.exception(
@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
def azure_storage_blob_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log success events to Azure Blob Storage
@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log failure events to Azure Blob Storage
@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
len(self.log_queue),
)
for payload in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=payload)
for item in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
async def async_upload_payload_to_azure_blob_storage(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Blob Storage using a 3-step process:
1. Create file resource
@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
try:
if self.azure_storage_account_key:
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
else:
# Get a valid token instead of always requesting a new one
await self.set_valid_azure_ad_token()
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
payload_bytes: Final = json_payload.encode("utf-8")
filename: Final = adls_safe_file_name(payload.get("id"))
filename: Final = file_name or adls_safe_file_name(payload.get("id"))
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
# Execute the 3-step upload process
@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
return self._service_client
async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
async def upload_to_azure_data_lake_with_azure_account_key(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Data Lake using the Azure SDK
@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
try:
today: Final = datetime.now().strftime("%Y-%m-%d")
file_name: Final = adls_safe_file_name(payload.get("id"))
resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
await self._upload_bytes_with_account_key(
file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
except Exception as e:
verbose_logger.exception("Error occurred: %s", e)

7
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@ -134,7 +135,7 @@ def spend_log_row_from_payload(
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
trace_id, span_id = _find_traceparent(metadata, kwargs)
request_id = str(payload.get("id") or "")
response_id: Final = str(payload.get("id") or "")
redact = litellm.turn_off_message_logging is True
track_end_user: Final = not litellm.disable_end_user_cost_tracking
completion_start_ms = _to_ms(payload.get("completionStartTime"))
@ -157,8 +158,8 @@ def spend_log_row_from_payload(
)
)
return SpendLogRecord(
request_id=request_id,
response_id=strip_cache_hit_suffix(request_id),
request_id=logged_event_key(kwargs, response_id),
response_id=strip_cache_hit_suffix(response_id),
provider_request_id=(
get_provider_request_id(kwargs.get("response_headers"))
or get_provider_request_id(hidden_params.get("additional_headers"))

13
litellm/integrations/custom_guardrail.py
View file

@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
enrich_http_exception_with_guardrail_context(result, self)
raise result
def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
return (
isinstance(request_data.get("guardrails"), list)
and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
)
async def async_post_call_success_deployment_hook(
self,
request_data: dict,
@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
"""
Allow modifying / reviewing the response just after it's received from the deployment.
"""
# should run guardrail
litellm_guardrails: Final = request_data.get("guardrails")
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
return None
if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
if not self.runs_post_call_deployment_hook(request_data):
return None
target: Final = self._deployment_hook_target()

5
litellm/integrations/deepeval/deepeval.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
to_zod_compatible_iso,
validate_environment,
)
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
# This file includes the custom callbacks for LiteLLM Proxy
@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
)
else:
output = str(standard_logging_object.get("error_string", ""))
logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
return BaseApiSpan(
uuid=standard_logging_object.get("id", uuid.uuid4()),
uuid=span_id,
name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
type=SpanApiType.LLM,

18
litellm/integrations/dynamodb.py
View file

@ -9,6 +9,8 @@ from typing import Final, Protocol
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
class _DynamoTable(Protocol):
@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
class DyanmoDBLogger:
# Class variables or attributes
def __init__(self):
def __init__(self, dynamodb: _DynamoResource | None = None):
# Instance variables
import boto3
self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
"dynamodb", region_name=os.environ["AWS_REGION_NAME"]
)
if litellm.dynamodb_table_name is None:
raise ValueError(
"LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@ -45,23 +49,23 @@ class DyanmoDBLogger:
_async_log_event = async_log_event
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
try:
print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
# construct payload to send to DynamoDB
# follows the same params as langfuse.py
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
messages: Final = kwargs.get("messages")
optional_params: Final = kwargs.get("optional_params", {})
call_type: Final = kwargs.get("call_type", "litellm.completion")
usage: Final = response_obj["usage"]
id: Final = response_obj.get("id", str(uuid.uuid4()))
response_id: Final = response_obj.get("id", str(uuid.uuid4()))
# Build the initial payload
payload: Final[dict[str, object]] = {
"id": id,
"id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
"call_type": call_type,
"startTime": start_time,
"endTime": end_time,

37
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
else:
await self._send_individual_logs(items_to_process)
def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
) -> str:
"""
Get the object name to use for the current payload
"""
current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
gcs_log_id: Final = metadata.get("gcs_log_id")
safe_log_id: Final = sanitize_cloud_object_component(
gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
)
if safe_log_id:
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
object_name = self._generate_failure_object_name(
request_date_str=current_date,
)
else:
object_name = self._generate_success_object_name(
request_date_str=current_date,
response_id=response_obj.get("id", ""),
)
# used for testing
_litellm_params: Final = kwargs.get("litellm_params", None) or {}
_metadata: Final = _litellm_params.get("metadata", None) or {}
if "gcs_log_id" in _metadata:
safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
if safe_log_id:
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
return object_name
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
response_id=logged_event_key(kwargs, response_id),
)
async def get_request_response_payload(
self,

11
litellm/integrations/openmeter.py
View file

@ -3,12 +3,15 @@
import json
import os
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
if len(missing_keys) > 0:
raise Exception(f"Missing keys={missing_keys} in environment.")
def _common_logic(self, kwargs: dict, response_obj):
def _common_logic(self, kwargs: Mapping[str, object], response_obj):
call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
dt: Final = get_utc_datetime().isoformat()
cost: Final = kwargs.get("response_cost", None)
@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
# If no user provided directly, try to get it from token user_id
if user_param is None:
# Check if user_id is available from the API key metadata
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {})
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
if user_api_key_user_id is not None:
@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
return {
"specversion": "1.0",
"type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
"id": call_id,
"id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
"time": dt,
"subject": subject,
"source": "litellm-proxy",

12
litellm/integrations/prometheus.py
View file

@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
async def async_post_call_failure_hook(
self,
request_data: dict,
request_data: dict[str, object],
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,
@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
try:
_tags: Final = StandardLoggingPayloadSetup.get_request_tags(
litellm_params=request_data,
proxy_server_request=request_data.get("proxy_server_request", {}),
proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
)
_metadata: Final = _logged_fields(request_data.get("metadata"))
model_id: Final = (
_logged_fields(_metadata.get("model_info")).get("id")
or _logged_fields(request_data.get("model_info")).get("id")
or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
)
_metadata: Final = request_data.get("metadata", {}) or {}
model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider: Final = self._extract_api_provider_from_request_data(
request_data

26
litellm/integrations/websearch_interception/handler.py
View file

@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
from typing_extensions import Never, ReadOnly
import litellm
from litellm._internal_context import billed_separately
from litellm._logging import verbose_logger
from litellm.anthropic_interface import messages as anthropic_messages
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
search_kwargs: Final = MappingProxyType(
{**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
)
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
with billed_separately():
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
# Format using transformation function
search_result_text: Final = WebSearchTransformation.format_search_response(result)

111
litellm/litellm_core_utils/allowed_tools_tool_choice.py Normal file
View file

@ -0,0 +1,111 @@
from collections.abc import Mapping
from typing import Final, Literal
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm.exceptions import BadRequestError
ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
'{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
)
class ChatAllowedTools(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
mode: Literal["auto", "required"]
tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
class ChatAllowedToolsToolChoice(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
type: Literal["allowed_tools"]
allowed_tools: ChatAllowedTools
_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
_SELECTOR: Final = TypeAdapter(dict[str, object])
class _ToolChoiceType(BaseModel):
model_config = ConfigDict(frozen=True)
type: str
_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType)
def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
try:
return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools"
except ValidationError:
return False
def raise_if_allowed_tools_tool_choice_unsupported(
tool_choice: object,
custom_llm_provider: str,
model: str,
) -> None:
"""Providers outside the set would silently send the request with no tool restriction at all, so the call stops
here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
return
raise BadRequestError(
message=(
"tool_choice type 'allowed_tools' is only supported for "
f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
"drop_params does not drop it, since that would let the model call tools the request excluded. "
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
),
model=model,
llm_provider=custom_llm_provider,
)
def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
try:
return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
except ValidationError:
return None
def _has_name(selector: Mapping[str, object]) -> bool:
name: Final = selector.get("name")
return isinstance(name, str) and name != ""
def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
tool_type: Final = tool.get("type")
if tool_type not in ("function", "custom"):
return tool
if tool_type not in tool:
return tool if _has_name(tool) else None
try:
nested: Final = _SELECTOR.validate_python(tool[tool_type])
except ValidationError:
return None
outer: Final = {key: value for key, value in tool.items() if key != tool_type}
if not _has_name(nested) or outer.keys() & nested.keys():
return None
return {**outer, **nested}
def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None:
"""The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or
carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser."""
flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools)
if any(tool is None for tool in flattened):
return None
return ToolChoiceAllowedParam(
type="allowed_tools",
mode=choice.allowed_tools.mode,
tools=[dict(tool) for tool in flattened if tool is not None],
)

341
litellm/litellm_core_utils/audio_utils/transcription_streaming.py Normal file
View file

@ -0,0 +1,341 @@
from __future__ import annotations
import datetime
import json
import traceback
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
from typing import Final, Protocol
import anyio
from openai import AsyncStream, Stream
from openai.types.audio import (
TranscriptionStreamEvent,
TranscriptionTextDeltaEvent,
TranscriptionTextDoneEvent,
)
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
)
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
_STREAM_FLAG: Final = TypeAdapter(bool)
def transcription_stream_flag(value: object) -> bool | None:
try:
return _STREAM_FLAG.validate_python(value)
except ValidationError:
return None
def transcription_stream_unset(value: object) -> bool:
return value is None or value == ""
def transcription_stream_requested(value: object) -> bool:
return transcription_stream_flag(value) is True
TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
model: Final = call_details.get("model")
custom_llm_provider: Final = call_details.get("custom_llm_provider")
extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
def map_exception(exception: Exception) -> Exception:
try:
mapped: Final[object] = exception_type(
model=model if isinstance(model, str) else None,
original_exception=exception,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
extra_kwargs=extra_kwargs,
)
except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception
return mapped_exception
return mapped if isinstance(mapped, Exception) else exception
return map_exception
def _guarded_transcription_event_payloads(
response: TranscriptionResponse,
) -> tuple[dict[str, object], dict[str, object]]:
text: Final = response.text or ""
usage: Final = response.usage
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
return (
{"type": "transcript.text.delta", "delta": text},
{
"type": "transcript.text.done",
"text": text,
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
},
)
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
return (
*(
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
for event in _guarded_transcription_event_payloads(response)
),
TRANSCRIPTION_STREAM_DONE_FRAME,
)
class GuardedTranscriptionReplay:
def __init__(self, response: TranscriptionResponse) -> None:
self.response: Final = response
self._hidden_params: dict[str, object] = response.hidden_params
def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
return self._events()
async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
delta, done = _guarded_transcription_event_payloads(self.response)
yield TranscriptionTextDeltaEvent.model_construct(**delta)
yield TranscriptionTextDoneEvent.model_construct(**done)
class TranscriptionStreamLogging(Protocol):
def success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
async def async_success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def handle_sync_success_callbacks_for_async_calls(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
*,
for_async_request: bool = False,
) -> None: ...
async def async_failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
class _TranscriptionEventCollector:
def __init__(self, duration: float | None) -> None:
self.duration = duration
self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event
self.done_event: TranscriptionTextDoneEvent | None = None
def add(self, event: TranscriptionStreamEvent) -> None:
if isinstance(event, TranscriptionTextDeltaEvent):
self.text_deltas.append(event.delta)
elif isinstance(event, TranscriptionTextDoneEvent):
self.done_event = event
def response(self) -> TranscriptionResponse:
done_event: Final = self.done_event
response: Final = TranscriptionResponse(
text=done_event.text if done_event is not None else "".join(self.text_deltas),
usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
languages=getattr(done_event, "languages", None) if done_event is not None else None,
)
if self.duration is not None:
response.set_audio_transcription_duration(self.duration)
return response
def _billed_usage(response: TranscriptionResponse) -> Usage:
usage: Final[object] = getattr(response, "usage", None)
token_usage: Final = (
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
if isinstance(usage, TranscriptionUsageTokensObject)
else None
)
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
def _record_billable_partial_spend(
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
) -> None:
partial_response: Final = collector.response()
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
if partial_cost <= 0:
return
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
def __init__(
self,
stream: Stream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
def _logging_iterator(
self, source_iterator: Iterator[TranscriptionStreamEvent]
) -> Iterator[TranscriptionStreamEvent]:
try:
for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
raise mapped_exception from exception
finally:
self._finalize()
def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
self._logging_obj.success_handler(
self._collector.response(),
self._start_time,
datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract
)
def close(self) -> None:
try:
super().close()
finally:
self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
def __init__(
self,
stream: AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
async def _logging_iterator(
self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
) -> AsyncIterator[TranscriptionStreamEvent]:
try:
async for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(
mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
)
await self._logging_obj.async_failure_handler(
mapped_exception,
traceback.format_exc(),
self._start_time,
end_time,
)
raise mapped_exception from exception
finally:
await self._finalize()
async def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
response: Final = self._collector.response()
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.handle_sync_success_callbacks_for_async_calls(
result=response,
start_time=self._start_time,
end_time=end_time,
)
with anyio.CancelScope(shield=True):
await self._logging_obj.async_success_handler(
result=response,
start_time=self._start_time,
end_time=end_time,
)
async def close(self) -> None:
try:
await super().close()
finally:
await self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
def wrap_transcription_stream(
stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
if isinstance(stream, AsyncStream):
return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

10
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -21,6 +21,16 @@ from litellm.types.files import (
from litellm.types.utils import FileTypes
def normalized_audio_duration_seconds(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
try:
seconds: Final = float(value)
except OverflowError:
return None
return seconds if math.isfinite(seconds) and seconds >= 0 else None
@dataclass
class ProcessedAudioFile:
"""

14
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -8,6 +8,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm._internal_context import internal_sub_call
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@ -204,12 +205,13 @@ async def _execute_chat_completion_agentic_plan(
)
try:
response_followup = await litellm.acompletion(
model=full_model_name,
messages=patch.messages,
**optional_params_for_followup,
**kwargs_for_followup,
)
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None):
response_followup = await litellm.acompletion(
model=full_model_name,
messages=patch.messages,
**optional_params_for_followup,
**kwargs_for_followup,
)
if _post_hook_overridden(callback):
try:
response_followup = await callback.async_post_agentic_loop_response_hook(

6
litellm/litellm_core_utils/core_helpers.py
View file

@ -4,7 +4,7 @@ import copy
import logging
import math
import re
from collections.abc import Collection, Iterable, Iterator, Mapping
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@ -104,6 +104,10 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def as_object_list(value: object) -> Sequence[object] | None:
return value if isinstance(value, list) else None
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:

12
litellm/litellm_core_utils/decisions_pricing.py
View file

@ -12,6 +12,7 @@ from typing import Final
DECISIONS_PRICING_SUFFIX: Final = "_decisions"
DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
def is_decisions_call_type(call_type: str) -> bool:
@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
}
def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
return None
rates: Final = {**model_info, **overrides}
request_price: Final = rates.get("input_cost_per_query")
if not isinstance(request_price, (int, float)):
return None
bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
"""Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}

1
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -107,6 +107,7 @@ class ExceptionCheckers:
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
"prompt is too long",
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -147,6 +147,10 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
@ -155,7 +159,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
declared: Final = declared_provider(model, custom_llm_provider)
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None

27
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -26,7 +26,12 @@ from litellm.constants import (
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
NON_INFERENCE_CALL_TYPES,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@ -35,6 +40,8 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
return is_unbilled_non_inference_call(call_type, metadata, response)
def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
)
def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
call_type: Final = kwargs.get("call_type")
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = (
as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
)
if not event_id or not settles_background_response(
call_type if isinstance(call_type, str) else "", metadata, kwargs
):
return event_id
return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

68
litellm/litellm_core_utils/litellm_logging.py
View file

@ -22,6 +22,8 @@ from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
from openai import Stream as OpenAIStream
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -872,6 +874,7 @@ class Logging(LiteLLMLoggingBaseClass):
# The model the proxy shows the client on streamed chunks. The logged streamed response carries it
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -918,6 +921,7 @@ class Logging(LiteLLMLoggingBaseClass):
# post_call guardrails have run; the @client decorator then stores the
# enqueue closure here instead of firing it immediately.
self.defer_async_logging: bool = False
self.drops_stream_token_ids: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
@ -2128,6 +2132,9 @@ class Logging(LiteLLMLoggingBaseClass):
if resolved_cache_hit is True:
return 0.0
if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
return None
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):
@ -3237,6 +3244,23 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"id": response_id})
def _with_client_facing_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.client_facing_response_id
if response_id is None or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _client_facing_streaming_response(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
return self._with_client_facing_response_id(
self._with_continued_response_id(self._with_client_facing_stream_model(response))
)
def _success_handler_helper_fn(
self,
result: object = None,
@ -3606,6 +3630,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
start_time=start_time,
end_time=end_time,
@ -3620,7 +3645,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
) = None
if "complete_streaming_response" in self.model_call_details and not for_async_request:
return # break out of this.
@ -3636,19 +3666,19 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["complete_streaming_response"] = complete_streaming_response
self._surface_response_headers_from_result(complete_streaming_response)
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
@ -4175,6 +4205,7 @@ class Logging(LiteLLMLoggingBaseClass):
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
)
)
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result = self._success_handler_helper_fn(
start_time=start_time,
end_time=end_time,
@ -4194,9 +4225,7 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
complete_streaming_response: Final = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
@ -4218,7 +4247,7 @@ class Logging(LiteLLMLoggingBaseClass):
get_base_model_from_metadata(model_call_details=self.model_call_details)
# base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
@ -4239,15 +4268,15 @@ class Logging(LiteLLMLoggingBaseClass):
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
try:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release)
verbose_logger.exception(
@ -5117,10 +5146,17 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
) -> (
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
):
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
return result
elif isinstance(
result,

79
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost(
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_hit_audio_tokens: ReadOnly[int]
cached_text_tokens: ReadOnly[int]
cached_audio_tokens: ReadOnly[int]
cached_image_tokens: ReadOnly[int]
has_cached_tokens_details: ReadOnly[bool]
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails | None
text_tokens: int
@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=cached_audio_tokens,
cached_text_tokens=cached_text_tokens,
cached_audio_tokens=cached_audio_tokens,
cached_image_tokens=cached_image_tokens,
has_cached_tokens_details=cached_tokens_details is not None,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
@ -1234,15 +1243,11 @@ def _calculate_input_cost(
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = get_cost_per_unit(
model_info,
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
prompt_cost += _calculate_cache_read_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
cache_read_cost=cache_read_cost,
service_tier=service_tier,
)
### AUDIO COST
@ -1331,6 +1336,38 @@ def _calculate_input_cost(
return prompt_cost
def _calculate_cache_read_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
cache_read_cost: float,
service_tier: str | None,
) -> float:
cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"]
cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"]
cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"]
classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0)
total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost
float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost
)
if cached_audio_tokens:
cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier)
cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost)
total_cost += ( # rebind-ok: cached audio contributes to cache-read cost
float(cached_audio_tokens) * float(cached_audio_cost or 0.0)
)
if cached_image_tokens:
cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier)
cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost)
total_cost += ( # rebind-ok: cached images contribute to cache-read cost
float(cached_image_tokens) * float(cached_image_cost or 0.0)
)
return total_cost
def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
"""
Resolve the per-model regional-processing uplift multiplier for a given
@ -1502,6 +1539,10 @@ def generic_cost_per_token(
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cached_text_tokens=0,
cached_audio_tokens=0,
cached_image_tokens=0,
has_cached_tokens_details=False,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
@ -1680,6 +1721,7 @@ class BilledTokenRates:
cache_creation_input_token_cost: float
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
output_cost_per_reasoning_token: float
cache_read_input_image_token_cost: float | None = None
def scaled(self, multiplier: float) -> "BilledTokenRates":
if multiplier == 1.0:
@ -1694,6 +1736,11 @@ class BilledTokenRates:
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
),
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
cache_read_input_image_token_cost=(
self.cache_read_input_image_token_cost * multiplier
if self.cache_read_input_image_token_cost is not None
else None
),
)
@ -1802,6 +1849,11 @@ def _cost_map_billed_rates(
cache_creation_input_token_cost=base_costs.cache_creation_rate,
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
output_cost_per_reasoning_token=reasoning_rate,
cache_read_input_image_token_cost=_get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier),
None,
),
).scaled(multiplier)
@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown(
cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
usage
)
cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"]
image_cache_read_rate: Final = (
rates.cache_read_input_image_token_cost
if rates.cache_read_input_image_token_cost is not None
else rates.cache_read_input_token_cost
)
cache_creation_cost: Final = (
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
if custom_cost_per_token is not None
@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown(
return TokenTypeCostBreakdown(
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
cache_read_cost=(
float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost
+ float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
+ float(cached_image_tokens) * image_cache_read_rate
),
cache_creation_cost=cache_creation_cost,
rates=rates,

55
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -3,7 +3,8 @@ import json
import re
import time
import traceback
from collections.abc import Generator, Iterator, Mapping, Sequence
from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal, cast
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -11,7 +12,12 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.constants import (
RESPONSE_FORMAT_TOOL_NAME,
STREAM_CHUNK_CHOICE_FIELDS,
STREAM_FINISH_CHOICE_FIELDS,
VLLM_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_reasoning_content,
@ -170,6 +176,28 @@ def _split_assembled_content_for_replay(content: str | None) -> list[str]:
return _REPLAY_CONTENT_SLICE_RE.findall(content)
def _drop_choice_fields(choice: StreamingChoices, fields: Iterable[str]) -> None:
for field in fields:
if hasattr(choice, field):
delattr(choice, field)
def _vllm_choice_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
cached_fields: Final = _choice_provider_specific_fields(choice)
return MappingProxyType(
{
field: choice[field] if field in choice else cached_fields[field]
for field in VLLM_CHOICE_FIELDS
if field in choice or field in cached_fields
}
)
def _choice_provider_specific_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
fields: Final = choice.get("provider_specific_fields")
return _FIELD_MAP.validate_python(fields) if isinstance(fields, dict) else MappingProxyType({})
_REPLAY_REASONING_FIELDS: Final = frozenset({"reasoning_content", "thinking_blocks", "reasoning_items"})
_REPLAY_LEAD_FIELDS: Final = _REPLAY_REASONING_FIELDS | {"role", "provider_specific_fields"}
_REPLAY_TOOL_FIELDS: Final = frozenset({"tool_calls", "function_call"})
@ -220,8 +248,9 @@ def _replay_frame(
choice.finish_reason = finish_reason
if not keeps_choice_metadata:
choice.logprobs = None
if hasattr(choice, "enhancements"):
del choice.enhancements
_drop_choice_fields(choice, ("enhancements", *STREAM_CHUNK_CHOICE_FIELDS))
if finish_reason is None:
_drop_choice_fields(choice, STREAM_FINISH_CHOICE_FIELDS)
return frame
@ -325,7 +354,9 @@ async def convert_to_streaming_response_async(
logprobs = choice.get("logprobs", None)
choice = StreamingChoices(finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs)
choice = StreamingChoices(
finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs, **_vllm_choice_fields(choice)
)
choice_list.append(choice)
model_response_object.choices = choice_list
@ -388,6 +419,7 @@ def convert_to_streaming_response(
delta=delta,
logprobs=logprobs,
enhancements=enhancements,
**_vllm_choice_fields(choice),
)
choice_list.append(choice)
@ -872,6 +904,7 @@ def convert_to_model_response_object(
logprobs=logprobs,
enhancements=enhancements,
provider_specific_fields=provider_specific_fields,
**_vllm_choice_fields(choice),
)
if provider_sent_logprobs and logprobs is None:
choice.logprobs = None
@ -937,9 +970,11 @@ def convert_to_model_response_object(
model_response_object.data = response_object["data"]
if "usage" in response_object and response_object["usage"] is not None:
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
embedding_usage: Final = model_response_object.usage or Usage()
embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage = embedding_usage
if start_time is not None and end_time is not None:
model_response_object._response_ms = (
@ -982,6 +1017,10 @@ def convert_to_model_response_object(
if key in response_object:
setattr(model_response_object, key, response_object[key])
detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages"))
if detected_languages is not None:
model_response_object.languages = detected_languages
if "usage" in response_object and response_object["usage"] is not None:
tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None

8
litellm/litellm_core_utils/model_param_helper.py
View file

@ -1,6 +1,6 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import ClassVar, Final
from typing import ClassVar, Final, get_args
from openai.types.chat.completion_create_params import (
CompletionCreateParamsNonStreaming,
@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
from litellm.types.rerank import RerankRequest
@ -178,6 +179,7 @@ class ModelParamHelper:
This follows the OpenAI API Spec
"""
litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams))
try:
from openai.types.audio.transcription_create_params import (
TranscriptionCreateParamsNonStreaming,
@ -187,11 +189,11 @@ class ModelParamHelper:
non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys())
streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys())
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs)
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs)
return all_transcription_kwargs
except Exception as e:
verbose_logger.debug("Error getting transcription kwargs %s", str(e))
return set()
return litellm_transcription_kwargs
@staticmethod
def _get_litellm_supported_responses_api_kwargs() -> set[str]:

26
litellm/litellm_core_utils/prompt_templates/chat_template_tools.py
View file

@ -10,12 +10,18 @@ class ChatTemplateToolSection:
preamble_tokens: int
implicit_system_turn_tokens: int
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
counts_tool_choice: bool = False
renders_strict_only_when_true: bool = False
def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
return "".join(
"\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
_TOOL_SECTIONS: Final = (
(
re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
),
(
re.compile(
r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
),
ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
),
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
(
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
(
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
)

85
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -29,6 +29,7 @@ from litellm.types.llms.openai import (
ChatCompletionAudioObject,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionPromptCacheBreakpoint,
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
ChatCompletionRedactedThinkingBlock,
@ -2193,16 +2194,62 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
def _accepted_prompt_cache_breakpoint(
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
) -> _AcceptedBreakpointT | None:
try:
return accepted.validate_python(marker)
except ValidationError:
return None
def prompt_cache_breakpoint_within(
accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
) -> _AcceptedBreakpointT | None:
"""The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
and a marker whose mode the target cannot read is dropped whole."""
whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
if whole is not None:
return whole
fields: Final = _breakpoint_fields_or_none(marker)
without_ttl: Final = (
None
if fields is None
else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
)
verbose_logger.debug(
"Dropping malformed prompt_cache_breakpoint %r under drop_params"
if without_ttl is None
else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
marker,
)
return without_ttl
def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
try:
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
"""The client's marker as written, or None under drop_params when no target could read its mode, so the
target's own config decides which ttl and keys it keeps."""
return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
@ -2220,6 +2267,13 @@ _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
try:
return _BREAKPOINT_FIELDS.validate_python(value)
except ValidationError:
return None
def _without_prompt_cache_breakpoint(value: object) -> object:
if not isinstance(value, Mapping):
return value
@ -2242,6 +2296,27 @@ def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys
def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
marker: Final = fields.get("prompt_cache_breakpoint")
if marker is None:
return fields
unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
content: Final = message.get("content")
if not drop_params or not isinstance(content, list):
return message
parts: Final = [
_part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
]
return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed
LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
{"thinking_blocks", "reasoning_items", "provider_specific_fields"}
)

62
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
split_leading_system_run,
string_field,
)
from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
from litellm.types.files import get_file_extension_from_mime_type
@ -76,7 +77,7 @@ from .common_utils import (
parse_tool_call_arguments,
salvage_concatenated_tool_arguments,
)
from .image_handling import convert_url_to_base64
from .image_handling import convert_url_to_base64, is_pdf_media_type
def default_pt(messages):
@ -910,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
)
def _anthropic_https_url_block(
image_url: str, image_format: str | None
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url)
if image_format and is_pdf_media_type(image_format):
return AnthropicMessagesDocumentParam(type="document", source=source)
return AnthropicMessagesImageParam(type="image", source=source)
def create_anthropic_image_param(
image_url_input: str | dict,
image_url_input: str | Mapping[str, object],
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
@ -920,13 +930,11 @@ def create_anthropic_image_param(
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
# Extract URL and format from input
if isinstance(image_url_input, str):
image_url = image_url_input
else:
image_url = image_url_input.get("url", "")
if format is None:
format = image_url_input.get("format")
url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url")
format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format")
image_url: Final = url_value if isinstance(url_value, str) else ""
explicit_format: Final = format if format is not None else format_value
image_format: Final = explicit_format if isinstance(explicit_format, str) else None
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
@ -934,21 +942,14 @@ def create_anthropic_image_param(
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return _anthropic_content_element_factory(image_chunk)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format)
return anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSourceUrl(
type="url",
url=image_url,
),
)
return _anthropic_https_url_block(image_url, image_format)
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return _anthropic_content_element_factory(image_chunk)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format)
return anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1328,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
_get_thought_signature_from_tool = get_thought_signature_from_tool
def _get_dummy_thought_signature() -> str:
def get_dummy_thought_signature() -> str:
"""Generate a dummy thought signature for models that require it.
This is used when transferring conversation history from older models
@ -1415,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
# https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
is_first_function_call = len(_parts_list) == 0
if not thought_signature and is_first_function_call and needs_dummy_signature:
thought_signature = _get_dummy_thought_signature()
thought_signature = get_dummy_thought_signature()
if thought_signature:
part_dict["thoughtSignature"] = thought_signature
@ -1439,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
# If no signature found and model is gemini-3, use dummy signature
if not thought_signature and needs_dummy_signature:
thought_signature = _get_dummy_thought_signature()
thought_signature = get_dummy_thought_signature()
if thought_signature:
part_dict_function["thoughtSignature"] = thought_signature
@ -2254,16 +2255,16 @@ def add_cache_control_to_content(
return anthropic_content_element
def _anthropic_content_element_factory(
def anthropic_content_element_factory(
image_chunk: GenericImageParsingChunk,
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
if image_chunk["media_type"] == "application/pdf":
if is_pdf_media_type(image_chunk["media_type"]):
_anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
media_type="application/pdf",
data=image_chunk["data"],
),
)
@ -5916,8 +5917,13 @@ def function_call_prompt(
llm_provider: str,
drop_params: bool,
) -> list[dict[str, object]]:
prompt_functions: Final = tools_without_hosted_mcp(
functions, model=model, route=llm_provider, drop_params=drop_params
)
if not prompt_functions:
return messages
header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions)
function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions)
if not any(_is_system_message(message) for message in messages):
return [*messages, {"role": "system", "content": function_prompt}]
return [

263
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -5,28 +5,33 @@ Helper functions to handle images passed in messages
import asyncio
import base64
import time
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from itertools import chain
from functools import partial
from itertools import accumulate, chain, takewhile, tee
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
from urllib.parse import urlparse
import httpx
from httpx import Response
from httpx import USE_CLIENT_DEFAULT, Response
import litellm
from litellm import verbose_logger
from litellm.caching.caching import InMemoryCache
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.types.llms.openai import AllMessageValues
MAX_IMGS_IN_MEMORY: Final = 10
MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
_Fetched = TypeVar("_Fetched")
_FETCH_ATTEMPTS: Final = 3
in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
pdf_probe_verdicts: Final = InMemoryCache()
def _header_value(response: Response, name: str) -> str | None:
@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool:
return media.part_type == "image_url"
_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"})
_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"})
_PDF_SIGNATURE: Final = b"%PDF-"
_PDF_SIGNATURE_WINDOW: Final = 1024
_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"})
_PDF_PROBE_STATUSES: Final = frozenset({200, 206})
_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError)
_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError)
_PDF_PROBE_VERDICT_TTL: Final = 600
_PDF_PROBE_FAILURE_TTL: Final = 60
_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"})
def _bare_media_type(media_type: str) -> str:
return media_type.split(";", 1)[0].strip().lower()
def is_pdf_media_type(media_type: str) -> bool:
return _bare_media_type(media_type) in _PDF_MEDIA_TYPES
def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool:
lowered: Final = url.lower()
return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf"))
def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool:
image_format: Final = media.fields.get("format")
return inline_remote_image_urls(media) and is_unformatted_pdf_link(
media.url, image_format if isinstance(image_format, str) else None
)
def _serves_pdf(response: Response) -> bool:
if response.status_code not in _PDF_PROBE_STATUSES:
return False
served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "")
if served_type and served_type not in _GENERIC_MEDIA_TYPES:
return served_type in _PDF_MEDIA_TYPES
return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW]
def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]:
reading, counting = tee(chunks)
read_so_far: Final = chain((0,), accumulate(map(len, counting)))
still_reading: Final = takewhile(
lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far
)
return (chunk for _, chunk in zip(still_reading, reading))
def _with_body_prefix(response: Response, prefix: bytes) -> Response:
return Response(
response.status_code,
headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS],
content=prefix,
request=response.request,
)
@dataclass(frozen=True, slots=True)
class _BodyPrefixReader:
client: httpx.Client
deadline: float
def get(
self,
url: str,
*,
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
) -> Response:
with self.client.stream(
"GET",
url,
headers=headers,
follow_redirects=follow_redirects,
timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
) as response:
prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline))
return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW])
@dataclass(frozen=True, slots=True)
class _AsyncBodyPrefixReader:
client: httpx.AsyncClient
async def get(
self,
url: str,
*,
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> Response:
async with self.client.stream(
"GET",
url,
headers=headers,
follow_redirects=follow_redirects,
timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
) as response:
return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b""))
def _cached_pdf_verdict(url: str) -> bool | None:
cached: Final = pdf_probe_verdicts.get_cache(url)
return cached if isinstance(cached, bool) else None
def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool:
pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl)
return verdict
def _remember_probe_response(url: str, response: Response) -> bool:
ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL
return _remember_pdf_verdict(url, _serves_pdf(response), ttl)
def _failed_pdf_probe(url: str, error: Exception) -> bool:
verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error)
return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL)
def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
return False
cached: Final = _cached_pdf_verdict(url)
if cached is not None:
return cached
try:
response: Final = safe_get(
_BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout),
url,
headers=dict(_PDF_PROBE_HEADERS),
timeout=timeout,
)
except _PDF_PROBE_FAILURES as e:
return _failed_pdf_probe(url, e)
return _remember_probe_response(url, response)
async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
return False
cached: Final = _cached_pdf_verdict(url)
if cached is not None:
return cached
try:
response: Final = await asyncio.wait_for(
async_safe_get(
_AsyncBodyPrefixReader(litellm.module_level_aclient.client),
url,
headers=dict(_PDF_PROBE_HEADERS),
timeout=timeout,
max_response_bytes=_PDF_SIGNATURE_WINDOW,
),
timeout,
)
except _ASYNC_PDF_PROBE_FAILURES as e:
return _failed_pdf_probe(url, e)
return _remember_probe_response(url, response)
def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
if fields.get("type") != "image_url":
return None
@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call
return _inline(remote, data_url) if data_url is not None else part
def _inline_message(
message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
) -> AllMessageValues:
def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues:
parts: Final = _content_parts(message)
if not parts:
return message
inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts]
inlined_message: Final = {**message, "content": inlined_parts}
return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined
rewritten: Final = {**message, "content": [rewrite(part) for part in parts]}
return rewritten # pyright: ignore[reportReturnType] # the same message with some of its parts rewritten
def _inline_message(
message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
) -> AllMessageValues:
return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline))
def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object:
remote: Final = _parse_remote_part(part)
match remote:
case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)):
image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS
return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}}
case _:
return part
def _with_pdf_formats(
messages: Iterable[AllMessageValues],
probed: Iterable[tuple[str, bool]],
should_probe: Callable[[RemoteMedia], bool],
) -> tuple[AllMessageValues, ...]:
pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf)
if not pdf_links:
return tuple(messages)
rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe)
return tuple(_with_parts(message, rewrite) for message in messages)
async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float
return await async_convert_url_to_base64(url, timeout=timeout)
async def async_convert_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> tuple[str, ...]:
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool:
async with in_flight:
return await async_serves_pdf(url)
async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]:
try:
return tuple(await asyncio.gather(*fetches))
except BaseException:
@ -322,6 +520,15 @@ async def async_convert_urls_to_base64(
raise
async def async_convert_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> tuple[str, ...]:
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
return await _gather_or_cancel_all(
tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
)
def _remote_urls_to_inline(
messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
) -> tuple[str, ...]:
@ -351,3 +558,27 @@ async def async_inline_remote_media(
data_urls: Final = await async_convert_urls_to_base64(remote_urls)
inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
return [_inline_message(message, inlined, should_inline) for message in messages]
def format_pdf_links(
messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
) -> tuple[AllMessageValues, ...]:
links: Final = _remote_urls_to_inline(messages, should_probe)
if not links:
return tuple(messages)
with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes:
verdicts: Final = tuple(probes.map(serves_pdf, links))
return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)
async def async_format_pdf_links(
messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
) -> tuple[AllMessageValues, ...]:
links: Final = _remote_urls_to_inline(messages, should_probe)
if not links:
return tuple(messages)
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
verdicts: Final = await _gather_or_cancel_all(
tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links)
)
return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)

718
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,4 +1,6 @@
import asyncio
import base64
import functools
import json
import time
import traceback
@ -8,7 +10,7 @@ from enum import Enum, auto
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly
import litellm
@ -22,6 +24,8 @@ from litellm.constants import (
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import (
@ -35,6 +39,8 @@ from litellm.types.llms.openai import (
OpenAIRealtimeResponseDelta,
OpenAIRealtimeStreamResponseBaseObject,
OpenAIRealtimeStreamSessionEvents,
OpenAIRealtimeTranslationClosedEvent,
OpenAIRealtimeTranslationDurationUsage,
)
from litellm.types.realtime import ALL_DELTA_TYPES
@ -42,9 +48,11 @@ from .litellm_logging import Logging as LiteLLMLogging
from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
if TYPE_CHECKING:
from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
from websockets.asyncio.client import ClientConnection
from websockets.exceptions import ConnectionClosed
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
CLIENT_CONNECTION_CLASS = ClientConnection
@ -96,6 +104,7 @@ class _ClientEventFrame(TypedDict, total=False):
type: ReadOnly[str]
item: ReadOnly[_ClientEventItem]
session: ReadOnly[Mapping[str, object]]
response: ReadOnly[Mapping[str, object]]
class _ResponseDoneBody(TypedDict, total=False):
@ -212,6 +221,87 @@ def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, ob
return _json_object(container.get(key))
_BETA_AUDIO_FORMAT_BY_GA_TYPE: Final[Mapping[str, str]] = MappingProxyType(
{"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"}
)
_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"output_modalities": "modalities", "max_output_tokens": "max_response_output_tokens"}
)
_GA_TO_BETA_AUDIO_INPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{
"format": "input_audio_format",
"transcription": "input_audio_transcription",
"noise_reduction": "input_audio_noise_reduction",
"turn_detection": "turn_detection",
}
)
_GA_TO_BETA_AUDIO_OUTPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"format": "output_audio_format", "voice": "voice", "speed": "speed"}
)
_GA_ONLY_SESSION_FIELDS: Final = frozenset(("type", "audio", *_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS))
def _beta_session_value(ga_field: str, value: object) -> object:
if ga_field == "output_modalities":
return ["audio", "text"] if value == ["audio"] else value
if ga_field != "format":
return value
ga_format: Final = _json_object(value)
format_type: Final = None if ga_format is None else ga_format.get("type")
return _BETA_AUDIO_FORMAT_BY_GA_TYPE.get(format_type, value) if isinstance(format_type, str) else value
def _beta_session_fields(ga_fields: Mapping[str, object], beta_names: Mapping[str, str]) -> Mapping[str, object]:
return {
beta_names[name]: _beta_session_value(name, value) for name, value in ga_fields.items() if name in beta_names
}
def ga_session_to_beta(session: Mapping[str, object]) -> Mapping[str, object]:
audio: Final = _mapping_field(session, "audio") or _EMPTY_SESSION
return {
**{key: value for key, value in session.items() if key not in _GA_ONLY_SESSION_FIELDS},
**_beta_session_fields(session, _GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS),
**_beta_session_fields(_mapping_field(audio, "input") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_INPUT_FIELDS),
**_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS),
}
_GA_ONLY_RESPONSE_FIELDS: Final = frozenset(("audio", "output_modalities"))
_GA_TO_BETA_RESPONSE_FIELDS: Final[Mapping[str, str]] = MappingProxyType({"output_modalities": "modalities"})
def ga_response_to_beta(response: Mapping[str, object]) -> Mapping[str, object]:
audio: Final = _mapping_field(response, "audio") or _EMPTY_SESSION
return {
**{key: value for key, value in response.items() if key not in _GA_ONLY_RESPONSE_FIELDS},
**_beta_session_fields(response, _GA_TO_BETA_RESPONSE_FIELDS),
**_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS),
}
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
_GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE: Final[Mapping[str, str]] = MappingProxyType(
{"text": "output_text", "audio": "output_audio"}
)
def _ga_assistant_content_block(block: object) -> object:
content_block: Final = _json_object(block)
block_type: Final = None if content_block is None else content_block.get("type")
if content_block is None or not isinstance(block_type, str):
return block
ga_type: Final = _GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE.get(block_type)
return block if ga_type is None else {**content_block, "type": ga_type}
def beta_item_to_ga(item: Mapping[str, object]) -> Mapping[str, object]:
content: Final = item.get("content")
if item.get("role") != "assistant" or not isinstance(content, list):
return item
return {**item, "content": [_ga_assistant_content_block(block) for block in _JSON_ARRAY.validate_python(content)]}
def _guardrail_event_id() -> str:
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
@ -270,6 +360,26 @@ def _gate_session_update(
return {**client_session, "turn_detection": gate(None)}
@functools.cache
def _translation_audio_format_adapter() -> "TypeAdapter[str | RealtimeAudioFormats | None]":
from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
return TypeAdapter(str | RealtimeAudioFormats | None)
def _translation_audio_bytes_per_second(audio_format: "str | RealtimeAudioFormats | None") -> float | None:
from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU
match audio_format:
case "pcm16" | AudioPCM():
return 48000.0
case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA():
return 8000.0
case _:
pass
return None
class RealtimeEventNormalizer(Protocol):
def should_drop(self, event: object) -> bool: ...
def normalize(self, event: dict) -> dict: ...
@ -294,6 +404,79 @@ DefaultLoggedRealTimeEventTypes: Final = [
]
TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE: Final = "Translation sessions cannot enforce configured realtime guardrails"
def _guardrails_run_for_event_hooks(
request_data: Mapping[str, object],
event_hooks: Sequence["GuardrailEventHooks"],
callbacks: Sequence[object],
) -> bool:
from litellm.integrations.custom_guardrail import CustomGuardrail
return any(
isinstance(cb, CustomGuardrail)
and any(cb.should_run_guardrail(data={**request_data, "stream": True}, event_type=et) for et in event_hooks)
for cb in callbacks
)
def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[str, object]) -> bool:
from litellm.integrations.custom_guardrail import CustomGuardrail
return (
isinstance(callback, CustomGuardrail)
and callback.event_hook is None
and callback.get_disable_global_guardrail(dict(request_data)) is True
)
def _realtime_runner_guardrail(
callback: object,
check_data: Mapping[str, object],
event_hooks: Sequence["GuardrailEventHooks"],
) -> "CustomGuardrail | None":
from litellm.integrations.custom_guardrail import CustomGuardrail
if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
return None
runs: Final = any(
callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
)
return callback if runs else None
def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool:
from litellm.types.guardrails import GuardrailEventHooks
callbacks: Final = tuple(
callback for callback in litellm.callbacks if not _opted_out_of_always_on_guardrail(callback, request_data)
)
if _guardrails_run_for_event_hooks(request_data, (GuardrailEventHooks.post_call,), callbacks):
return True
if any(
isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket
for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata"))
):
from litellm.proxy.utils import pipeline_managed_guardrail_names
if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names(
request_data, "post_call"
):
return True
return realtime_guardrails_apply(request_data, callbacks)
def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Sequence[object]) -> bool:
from litellm.types.guardrails import GuardrailEventHooks
check_data: Final = {**request_data, "stream": True}
runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call)
return any(
_realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
)
class RealTimeStreaming:
def __init__(
self,
@ -303,12 +486,15 @@ class RealTimeStreaming:
provider_config: BaseRealtimeConfig | None = None,
model: str = "",
user_api_key_dict: object | None = None,
request_data: dict | None = None,
request_data: Mapping[str, object] | None = None,
backend_uses_beta_protocol: bool | None = None,
force_transcription_model: str | None = None,
event_normalizer: RealtimeEventNormalizer | None = None,
usage_meter: RealtimeUsageMeter | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
translation_session: bool = False,
bridges_beta_client_to_ga: bool = False,
drop_params: bool = False,
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
clock: Callable[[], float] = time.monotonic,
@ -326,14 +512,24 @@ class RealTimeStreaming:
self._logging_worker = logging_worker
self.messages: list[OpenAIRealtimeEvents] = []
self._backend_sent_frames: bool = False
self._backend_sent_non_error_frames: bool = False
self.input_message: dict = {}
self.input_messages: list[dict[str, str]] = []
self.session_tools: list[dict] = []
self.tool_calls: list[dict] = []
self._is_translation_session = translation_session
self._bridges_beta_client_to_ga: Final = bridges_beta_client_to_ga
self._remaps_all_beta_fields_to_ga: Final = bridges_beta_client_to_ga or translation_session
self._drop_params: Final = drop_params
self._translation_input_seconds = 0.0
self._translation_input_bytes_per_second = 48000.0
self._translation_output_seconds = 0.0
self._translation_output_bytes_per_second = 48000.0
self._translation_usage_finalized = False
# Detect whether the client is explicitly opting into the beta protocol.
self._client_wants_beta = self.detect_beta_header(websocket)
self._backend_uses_beta_protocol = (
self._backend_uses_beta_protocol = not translation_session and (
self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol
)
@ -352,7 +548,7 @@ class RealTimeStreaming:
self.current_delta_type: ALL_DELTA_TYPES | None = None
self.session_configuration_request: str | None = None
self.user_api_key_dict = user_api_key_dict
self.request_data: dict = request_data or {}
self.request_data: Mapping[str, object] = request_data or {}
# Violation counter for end_session_after_n_fails support
self._violation_count: int = 0
# When a text message is blocked, hold the guardrail reason so the next
@ -384,6 +580,7 @@ class RealTimeStreaming:
# their input_audio_transcription.completed usage drives duration-based cost.
self._force_transcription_model = force_transcription_model
self._is_transcription_session: bool = force_transcription_model is not None
self._bound_nested_transcription_model: str | None = None
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
self._event_normalizer = event_normalizer
self._gate_turn_detection: Final[_TurnDetectionGate] = (
@ -407,6 +604,7 @@ class RealTimeStreaming:
_MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB
_SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
_RESPONSE_EVENT_TYPES: Final = frozenset(("response.created", "response.done"))
_CLIENT_AUDIO_BUFFER_TYPES = frozenset(
[
"input_audio_buffer.append",
@ -429,8 +627,8 @@ class RealTimeStreaming:
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
"g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
"g711_ulaw": {"type": "audio/pcmu"},
"g711_alaw": {"type": "audio/pcma"},
}
# GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
_GA_TO_BETA_EVENT_TYPES: dict[str, str] = {
@ -449,12 +647,12 @@ class RealTimeStreaming:
def _should_store_message(
self,
message_obj: dict | OpenAIRealtimeEvents,
message_obj: Mapping[str, object],
) -> bool:
_msg_type: Final = message_obj["type"] if "type" in message_obj else None
if self.logged_real_time_event_types == "*":
return True
if _msg_type and _msg_type in self.logged_real_time_event_types:
if isinstance(_msg_type, str) and _msg_type in self.logged_real_time_event_types:
return True
return False
@ -555,7 +753,7 @@ class RealTimeStreaming:
input_messages, so it is not duplicated into the response log here.
"""
try:
usage: Final = event_obj.get("usage")
usage: Final = as_str_mapping(event_obj.get("usage"))
if usage is None:
return
# If this event type is already captured by store_message (e.g. the user
@ -623,6 +821,7 @@ class RealTimeStreaming:
async def log_messages(self):
"""Log messages in list"""
self._finalize_translation_usage()
if self.logging_obj:
if self._usage_meter is not None:
self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot()
@ -641,6 +840,142 @@ class RealTimeStreaming:
)
self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None:
if not self._is_translation_session:
return
if event_obj.get("type") == "session.closed":
usage: Final = as_str_mapping(event_obj.get("usage"))
output_seconds: Final = (
normalized_audio_duration_seconds(usage.get("output_seconds")) if usage is not None else None
)
reported_input_seconds: Final = (
normalized_audio_duration_seconds(usage.get("input_seconds")) if usage is not None else None
)
input_seconds: Final = (
reported_input_seconds
if reported_input_seconds is not None
else self._translation_input_seconds or None
)
synthetic_output_seconds: Final = self._translation_output_seconds or None
resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds
if input_seconds is not None or resolved_output_seconds is not None:
if self._should_store_message(event_obj):
supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage(
type="duration",
input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0,
output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0,
)
if (
supplemental_usage.get("input_seconds", 0.0) > 0
or supplemental_usage.get("output_seconds", 0.0) > 0
):
self.messages.append(
OpenAIRealtimeTranslationClosedEvent(
type="session.closed",
usage=supplemental_usage,
)
)
else:
normalized_usage: Final = (
OpenAIRealtimeTranslationDurationUsage(
type="duration",
input_seconds=input_seconds,
output_seconds=float(resolved_output_seconds or 0.0),
)
if input_seconds is not None
else OpenAIRealtimeTranslationDurationUsage(
type="duration", output_seconds=float(resolved_output_seconds or 0.0)
)
)
self.messages.append(
OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage)
)
self._translation_usage_finalized = True
return
self._capture_translation_audio_formats(event_obj)
if event_obj.get("type") not in (
"session.output_audio.delta",
"response.output_audio.delta",
"response.audio.delta",
):
return
delta: Final = event_obj.get("delta")
if not isinstance(delta, str):
return
try:
decoded: Final = base64.b64decode(delta, validate=True)
except (ValueError, TypeError):
return
self._translation_output_seconds += len(decoded) / self._translation_output_bytes_per_second
def _capture_translation_input_audio(self, message: str) -> None:
if not self._is_translation_session:
return
try:
event: Final = _decode_json_object(message)
except (json.JSONDecodeError, TypeError):
return
if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str):
return
try:
decoded: Final = base64.b64decode(audio, validate=True)
except (ValueError, TypeError):
return
self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second
def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None:
if event_obj.get("type") not in ("session.created", "session.updated"):
return
session: Final = as_str_mapping(event_obj.get("session"))
if session is None:
return
audio: Final = as_str_mapping(session.get("audio"))
if audio is None:
return
for direction, raw_configuration in (("input", audio.get("input")), ("output", audio.get("output"))):
if (configuration := as_str_mapping(raw_configuration)) is None:
continue
try:
if (
bytes_per_second := _translation_audio_bytes_per_second(
_translation_audio_format_adapter().validate_python(configuration.get("format"))
)
) is None:
continue
except ValidationError:
continue
if direction == "input":
self._translation_input_bytes_per_second = bytes_per_second
else:
self._translation_output_bytes_per_second = bytes_per_second
def _finalize_translation_usage(self) -> None:
if self._translation_usage_finalized:
return
for event in self.messages:
if event.get("type") != "session.closed":
continue
if (event_usage := as_str_mapping(event.get("usage"))) is not None and normalized_audio_duration_seconds(
event_usage.get("output_seconds")
) is not None:
self._translation_usage_finalized = True
return
if self._translation_output_seconds == 0 and self._translation_input_seconds == 0:
return
synthetic_usage: Final = (
OpenAIRealtimeTranslationDurationUsage(
type="duration",
input_seconds=self._translation_input_seconds,
output_seconds=self._translation_output_seconds,
)
if self._translation_input_seconds > 0
else OpenAIRealtimeTranslationDurationUsage(
type="duration", output_seconds=self._translation_output_seconds
)
)
self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage))
self._translation_usage_finalized = True
async def _send_to_backend(self, message: str) -> bool:
"""Send a message to the backend WebSocket.
@ -653,10 +988,25 @@ class RealTimeStreaming:
backend, False if the provider transformation produced no output and
the message was effectively dropped.
"""
message = self._enforce_transcription_session_model(message)
from litellm.proxy._types import ProxyException
try:
authorized_message: Final = await self._apply_nested_transcription_model_policy(message)
except ProxyException as denied:
denial_message: Final = redact_internal_details_from_client_message(denied.message)
try:
await self.websocket.send_text(realtime_error_event(denial_message, error_type="permission_error"))
finally:
await self.websocket.close(
code=1008,
reason=websocket_close_reason(
denial_message, fallback="Nested transcription model is not authorized"
),
)
return False
if self.provider_config:
transformed: Final = self.provider_config.transform_realtime_request(
message, self.model, self.session_configuration_request
authorized_message, self.model, self.session_configuration_request
)
sent = False
for msg in transformed:
@ -691,14 +1041,94 @@ class RealTimeStreaming:
if is_content_message:
self._content_sent_after_setup = True
sent = True
if sent:
self._capture_translation_input_audio(authorized_message)
return sent
await self.backend_ws.send(message)
await self.backend_ws.send(authorized_message)
if self._usage_meter is not None:
sent_event: Final = self._parse_backend_event(message)
sent_event: Final = self._parse_backend_event(authorized_message)
if sent_event is not None:
self._usage_meter.observe_client_event(sent_event)
self._capture_translation_input_audio(authorized_message)
return True
async def _apply_nested_transcription_model_policy(self, message: str) -> str:
if self._force_transcription_model is not None:
return self._enforce_transcription_session_model(message)
if self._is_translation_session:
return await self._enforce_translation_nested_transcription_model(message)
return message
def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None:
try:
message_obj: Final = _decode_json_object(message)
except (json.JSONDecodeError, TypeError):
return None
if message_obj.get("type") not in (
"session.update",
"transcription_session.update",
):
return None
return message_obj
def _nested_transcription_models_from_session(
self,
session: Mapping[str, object],
) -> tuple[str, ...]:
audio: Final = as_str_mapping(session.get("audio"))
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
nested_transcription: Final = audio_input.get("transcription") if audio_input is not None else None
nested_model: Final = self._transcription_model_value(nested_transcription)
flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription"))
return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None))
def _transcription_model_value(self, transcription_config: object) -> str | None:
if (fields := as_str_mapping(transcription_config)) is None:
return None
model: Final = fields.get("model")
if isinstance(model, str) and model:
return model
return None
def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str:
message_obj: Final = self._session_update_message_obj(message)
if message_obj is None:
return message
session: Final = as_str_mapping(message_obj.get("session"))
if session is None:
return message
transcription: Final = as_str_mapping(session.get("input_audio_transcription"))
rewrite_flat: Final = transcription is not None and transcription.get("model") != authorized_model
audio: Final = as_str_mapping(session.get("audio"))
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
nested_transcription: Final = (
as_str_mapping(audio_input.get("transcription")) if audio_input is not None else None
)
rewrite_nested: Final = (
nested_transcription is not None and nested_transcription.get("model") != authorized_model
)
if not rewrite_flat and not rewrite_nested:
return message
updated_session: Final = {
**session,
**(
{"input_audio_transcription": {**transcription, "model": authorized_model}}
if transcription is not None and rewrite_flat
else {}
),
**(
{
"audio": {
**audio,
"input": {**audio_input, "transcription": {**nested_transcription, "model": authorized_model}},
}
}
if audio is not None and audio_input is not None and nested_transcription is not None and rewrite_nested
else {}
),
}
return json.dumps({**message_obj, "session": updated_session})
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
@ -877,56 +1307,52 @@ class RealTimeStreaming:
if self._force_transcription_model is None:
return message
try:
message_obj: Final = _decode_json_object(message)
except (json.JSONDecodeError, TypeError):
message_obj: Final = self._session_update_message_obj(message)
if message_obj is None:
return message
if message_obj.get("type") not in (
"session.update",
"transcription_session.update",
):
return message
session: Final = message_obj.get("session")
if not isinstance(session, dict):
return message
if session.get("type") == "transcription":
session: Final = as_str_mapping(message_obj.get("session"))
if session is not None and session.get("type") == "transcription":
self._is_transcription_session = True
return self._rewrite_session_update_transcription_model(message, self._force_transcription_model)
authorized_model: Final = self._force_transcription_model
changed = False
async def _enforce_translation_nested_transcription_model(self, message: str) -> str:
message_obj: Final = self._session_update_message_obj(message)
session: Final = as_str_mapping(message_obj.get("session")) if message_obj is not None else None
bound_model: Final = self._bound_nested_transcription_model
requested_models: Final = (
tuple(model for model in self._nested_transcription_models_from_session(session) if model != bound_model)
if session is not None
else ()
)
if not requested_models:
return (
message
if bound_model is None
else self._rewrite_session_update_transcription_model(message, bound_model)
)
transcription: Final = session.get("input_audio_transcription")
if isinstance(transcription, dict) and transcription.get("model") != authorized_model:
session["input_audio_transcription"] = {
**transcription,
"model": authorized_model,
}
changed = True
valid_token: Final = self.user_api_key_dict
audio: Final = session.get("audio")
if isinstance(audio, dict):
audio_input: Final = audio.get("input")
if isinstance(audio_input, dict):
nested_transcription: Final = audio_input.get("transcription")
if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model:
session["audio"] = {
**audio,
"input": {
**audio_input,
"transcription": {
**nested_transcription,
"model": authorized_model,
},
},
}
changed = True
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
from litellm.proxy.proxy_server import llm_model_list, llm_router
from litellm.proxy.realtime_endpoints.endpoints import nested_transcription_upstream_model
if not changed:
if not isinstance(valid_token, UserAPIKeyAuth):
return message
return json.dumps(message_obj)
for requested_model in requested_models:
await can_key_call_resolved_model(
model=requested_model,
valid_token=valid_token,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
upstream_model: Final = nested_transcription_upstream_model(
requested_models[0], valid_token.team_id, llm_router
)
self._bound_nested_transcription_model = upstream_model
return self._rewrite_session_update_transcription_model(message, upstream_model)
def _uses_deferred_backend_setup(self) -> bool:
"""True when setup is deferred until the client's first session.update."""
@ -1047,7 +1473,19 @@ class RealTimeStreaming:
return event
def _event_to_client_json(self, event: dict) -> str:
return json.dumps(self._normalize_event_for_ga_client(event))
return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event)))
def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]:
if not self._bridges_beta_client_to_ga:
return event
event_type: Final = event.get("type")
if event_type in self._SESSION_EVENT_TYPES:
session: Final = _mapping_field(event, "session")
return event if session is None else {**event, "session": ga_session_to_beta(session)}
if event_type in self._RESPONSE_EVENT_TYPES:
response: Final = _mapping_field(event, "response")
return event if response is None else {**event, "response": ga_response_to_beta(response)}
return event
async def _send_event_to_client(self, event: object, event_str: str) -> bool:
if self._should_drop_event_from_client(event):
@ -1060,7 +1498,7 @@ class RealTimeStreaming:
translated: Final = self._translate_event_to_beta(event)
if translated is None:
return False
await self.websocket.send_text(json.dumps(translated))
await self.websocket.send_text(json.dumps(self._session_event_for_client(translated)))
return True
except Exception as e:
verbose_logger.warning(
@ -1183,32 +1621,7 @@ class RealTimeStreaming:
self,
event_hooks: Sequence["GuardrailEventHooks"],
) -> bool:
"""Return True if any callback would run for one of ``event_hooks``."""
from litellm.integrations.custom_guardrail import CustomGuardrail
return any(
isinstance(cb, CustomGuardrail)
and any(
cb.should_run_guardrail(
data={**self.request_data, "stream": True},
event_type=et,
)
for et in event_hooks
)
for cb in litellm.callbacks
)
def _has_realtime_guardrails(self) -> bool:
"""Return True if any callback is registered for realtime guardrail event types."""
from litellm.types.guardrails import GuardrailEventHooks
return self._has_realtime_guardrails_for_event_hooks(
[
GuardrailEventHooks.realtime_input_transcription,
GuardrailEventHooks.pre_call,
GuardrailEventHooks.post_call,
]
)
return _guardrails_run_for_event_hooks(self.request_data, event_hooks, litellm.callbacks)
def _has_audio_transcription_guardrails(self) -> bool:
"""Return True when a guardrail is configured for the audio/VAD transcript path.
@ -1222,8 +1635,12 @@ class RealTimeStreaming:
return self._has_realtime_guardrails_for_event_hooks([GuardrailEventHooks.realtime_input_transcription])
def _should_disable_vad_auto_response(self) -> bool:
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
"""Transcription and translation sessions have no assistant turn to gate and reject realtime session updates."""
return (
not self._is_transcription_session
and not self._is_translation_session
and self._has_audio_transcription_guardrails()
)
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
return (
@ -1264,7 +1681,6 @@ class RealTimeStreaming:
transcript completion uses ``realtime_input_transcription`` only;
typed user messages and tool outputs use ``pre_call``.
"""
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
if event_hooks is None:
@ -1273,14 +1689,11 @@ class RealTimeStreaming:
_check_data: Final = {**self.request_data, "transcript": transcript, "stream": True}
_already_run: Final[set] = set()
for callback in litellm.callbacks:
if not isinstance(callback, CustomGuardrail):
continue
if callback.use_native_lifecycle_hooks:
continue
if id(callback) in _already_run:
continue
if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types):
runner_guardrails: Final = (
_realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks
)
for callback in runner_guardrails:
if callback is None or id(callback) in _already_run:
continue
_already_run.add(id(callback))
try:
@ -1369,7 +1782,10 @@ class RealTimeStreaming:
async def _handle_provider_config_message(self, raw_response: str) -> None:
"""Process a backend message when a provider_config is set (transformed path)."""
returned_object: Final = self.provider_config.transform_realtime_response(
provider_config: Final = self.provider_config
if provider_config is None:
raise RuntimeError("Provider response handling requires a provider configuration")
returned_object: Final = provider_config.transform_realtime_response(
raw_response,
self.model,
self.logging_obj,
@ -1398,6 +1814,7 @@ class RealTimeStreaming:
continue
if isinstance(event, dict):
self._detect_transcription_session_from_backend(event)
self._capture_translation_output_audio(event)
is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created"
if is_session_created_event:
if self._uses_deferred_backend_setup() and not self._backend_setup_complete:
@ -1514,6 +1931,7 @@ class RealTimeStreaming:
continue
if self.provider_config:
self._backend_sent_non_error_frames = True
try:
await self._handle_provider_config_message(raw_response)
except Exception as e:
@ -1521,6 +1939,8 @@ class RealTimeStreaming:
continue
else:
event = self._parse_backend_event(raw_response)
if event is None or event.get("type") != "error":
self._backend_sent_non_error_frames = True
if event is None:
await self.websocket.send_text(raw_response)
continue
@ -1535,6 +1955,7 @@ class RealTimeStreaming:
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
continue
self._capture_translation_output_audio(event)
if await self._handle_raw_backend_message(event, raw_response):
continue
@ -1548,7 +1969,7 @@ class RealTimeStreaming:
translated = self._translate_event_to_beta(event)
if translated is None:
continue
await self.websocket.send_text(json.dumps(translated))
await self.websocket.send_text(json.dumps(self._session_event_for_client(translated)))
async def backend_to_client_send_messages(self) -> BackendClose:
import websockets
@ -1573,8 +1994,8 @@ class RealTimeStreaming:
def _backend_refused_session(self, close: BackendClose) -> bool:
return (
close.code != 1000
and not self._backend_sent_frames
not self._backend_sent_non_error_frames
and (close.code != 1000 or self._backend_sent_frames)
and not self._holds_session_updates_until_first_content()
)
@ -1617,7 +2038,7 @@ class RealTimeStreaming:
_detect_beta_header = detect_beta_header
@staticmethod
def _remap_beta_session_to_ga(session: dict) -> dict:
def _remap_beta_session_to_ga(session: dict, all_beta_fields: bool = False, drop_temperature: bool = False) -> dict:
"""
Convert a beta-style session.update payload to the GA nested schema.
@ -1630,6 +2051,11 @@ class RealTimeStreaming:
session.output_audio_format → session.audio.output.format (with type/rate)
session.turn_detection → session.audio.input.turn_detection
session.input_audio_transcription → session.audio.input.transcription
Only with all_beta_fields:
session.input_audio_noise_reduction → session.audio.input.noise_reduction
session.speed → session.audio.output.speed
session.max_response_output_tokens → session.max_output_tokens
session.temperature (dropped when drop_temperature)
─────────────────────────────────────────────────────────────────────
Fields not in the mapping (instructions, tools, etc.) are passed through.
GA clients that already use the nested shape are unaffected.
@ -1681,6 +2107,19 @@ class RealTimeStreaming:
if "input_audio_transcription" in session:
inp["transcription"] = session.pop("input_audio_transcription")
if all_beta_fields and "input_audio_noise_reduction" in session:
inp["noise_reduction"] = session.pop("input_audio_noise_reduction")
if all_beta_fields and "speed" in session:
out["speed"] = session.pop("speed")
if all_beta_fields and "max_response_output_tokens" in session:
max_tokens: Final = session.pop("max_response_output_tokens")
session.setdefault("max_output_tokens", max_tokens)
if all_beta_fields and drop_temperature:
session.pop("temperature", None)
if inp:
audio["input"] = inp
if out:
@ -1699,6 +2138,53 @@ class RealTimeStreaming:
return session
@staticmethod
def _remap_beta_response_to_ga(response: Mapping[str, object], drop_temperature: bool) -> dict[str, object]:
moved: Final = frozenset(
("modalities", "voice", "output_audio_format", "max_response_output_tokens")
+ (("temperature",) if drop_temperature else ())
)
raw_format: Final = response.get("output_audio_format")
output_audio: Final = {
**({} if "voice" not in response else {"voice": response["voice"]}),
**(
{}
if "output_audio_format" not in response
else {
"format": RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw_format, raw_format)
if isinstance(raw_format, str)
else raw_format
}
),
}
modalities: Final = response.get("modalities")
modality_names: Final = (
frozenset(m.lower() for m in modalities if isinstance(m, str))
if isinstance(modalities, list)
else frozenset()
)
output_modalities: Final = (
["audio"] if "audio" in modality_names else ["text"] if "text" in modality_names else None
)
existing_audio: Final = as_str_mapping(response.get("audio")) or {}
existing_output: Final = as_str_mapping(existing_audio.get("output")) or {}
return {
**{key: value for key, value in response.items() if key not in moved},
**(
{}
if output_modalities is None or "output_modalities" in response
else {"output_modalities": output_modalities}
),
**(
{}
if "max_response_output_tokens" not in response or "max_output_tokens" in response
else {"max_output_tokens": response["max_response_output_tokens"]}
),
**(
{} if not output_audio else {"audio": {**existing_audio, "output": {**existing_output, **output_audio}}}
),
}
@staticmethod
def _translate_event_to_beta(event: dict) -> dict | None:
"""Translate a single GA event dict to its beta equivalent.
@ -1932,10 +2418,34 @@ class RealTimeStreaming:
if msg_type == "session.update" and not self._backend_uses_beta_protocol:
session = client_event.get("session", {})
if isinstance(session, dict):
session = self._remap_beta_session_to_ga(session)
session = self._remap_beta_session_to_ga(
session,
all_beta_fields=self._remaps_all_beta_fields_to_ga,
drop_temperature=self._drop_params,
)
if self._is_translation_session:
session.pop("type", None)
msg_obj["session"] = session
message = json.dumps(msg_obj)
if (
msg_type == "response.create"
and self._remaps_all_beta_fields_to_ga
and (beta_response := as_str_mapping(client_event.get("response"))) is not None
):
msg_obj["response"] = self._remap_beta_response_to_ga(
beta_response, drop_temperature=self._drop_params
)
message = json.dumps(msg_obj)
if (
msg_type == "conversation.item.create"
and self._bridges_beta_client_to_ga
and (beta_item := as_str_mapping(client_event.get("item"))) is not None
):
msg_obj["item"] = beta_item_to_ga(beta_item)
message = json.dumps(msg_obj)
if msg_type == "session.update" and self._event_normalizer:
session = client_event.get("session")
if isinstance(session, dict):
@ -2019,6 +2529,12 @@ class RealTimeStreaming:
return ClientLoopExit.CLIENT_DISCONNECTED
async def bidirectional_forward(self) -> None:
if self._is_translation_session and translation_guardrails_apply(self.request_data):
await self.websocket.send_text(
realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error")
)
await self.websocket.close(code=1008, reason=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE)
return
forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages())
client_task: Final = asyncio.create_task(self.client_ack_messages())
try:

32
litellm/litellm_core_utils/redact_messages.py
View file

@ -171,6 +171,8 @@ def _redact_streaming_response(streaming_response):
_redact_bedrock_guardrail_trace(streaming_response)
elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
return perform_redaction({}, streaming_response)
elif isinstance(streaming_response, litellm.TranscriptionResponse):
streaming_response.text = REDACTED_BY_LITELLM
elif hasattr(streaming_response, "output"):
_redact_responses_api_output(streaming_response.output)
if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None:
@ -196,6 +198,9 @@ def _redact_function_call(function_call) -> None:
def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, (litellm.Choices, litellm.utils.StreamingChoices, litellm.utils.TextChoices)):
drop_token_ids(choice)
_redact_provider_specific_fields(choice)
if isinstance(choice, litellm.Choices):
if choice.message.content is not None:
choice.message.content = REDACTED_BY_LITELLM
@ -216,6 +221,9 @@ def _redact_choice_content(choice):
_redact_provider_specific_fields(choice.delta)
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
_redact_function_call(getattr(choice.delta, "function_call", None))
elif isinstance(choice, litellm.utils.TextChoices):
_redact_field_when_present(choice, "text")
_redact_field_when_present(choice, "reasoning_content")
def _as_mapping(value: object) -> Mapping[str, object] | None:
@ -227,14 +235,27 @@ def _redact_provider_specific_fields(message: object) -> None:
setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields)) # noqa: B010 # typed as object
def _drop_provider_specific_token_ids(holder: object) -> None:
fields: Final = _as_mapping(getattr(holder, "provider_specific_fields", None))
if fields is not None and fields.get("token_ids") is not None:
setattr(holder, "provider_specific_fields", {**fields, "token_ids": None}) # noqa: B010 # typed as object
def drop_token_ids(choice: object) -> None:
if getattr(choice, "token_ids", None) is not None:
setattr(choice, "token_ids", None) # noqa: B010 # pydantic extra, not a declared field
for holder in (choice, getattr(choice, "delta", None), getattr(choice, "message", None)):
_drop_provider_specific_token_ids(holder)
def _redact_provider_specific_fields_dict(choice: Mapping[str, object]) -> None:
for message in (choice.get("message"), choice.get("delta")):
for holder in (choice, choice.get("message"), choice.get("delta")):
if (
(readable := _as_mapping(message)) is not None
(readable := _as_mapping(holder)) is not None
and (fields := _as_mapping(readable.get("provider_specific_fields"))) is not None
and isinstance(message, dict)
and isinstance(holder, dict)
):
message["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
holder["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[str, object]:
@ -251,6 +272,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
"compaction_blocks",
"compaction_start",
"compaction_delta",
"token_ids",
)
if key in fields
},
@ -422,6 +444,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
_redact_provider_specific_fields_dict(choice)
if choice.get("token_ids") is not None:
choice["token_ids"] = None
if "message" in choice and isinstance(choice["message"], dict):
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str

8
litellm/litellm_core_utils/stream_usage_chunks.py
View file

@ -1,6 +1,7 @@
from collections.abc import Iterable
from typing import Final
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.types.utils import ModelResponseStream, StreamingChoices
@ -9,12 +10,19 @@ def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
return False
if getattr(choice, "logprobs", None) is not None:
return False
if not _all_values_none(_vllm_choice_values(choice)):
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _vllm_choice_values(choice: StreamingChoices) -> Iterable[object]:
choice_extras: Final = choice.model_extra or {}
return (choice_extras.get(field) for field in VLLM_CHOICE_FIELDS)
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)

46
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.litellm_core_utils.core_helpers import (
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
UPSTREAM_USAGE_COST,
@ -31,6 +32,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
)
from litellm.litellm_core_utils.redact_messages import drop_token_ids
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
from litellm.types.llms.openai import (
AllMessageValues,
@ -434,6 +436,8 @@ class ChunkProcessor:
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
),
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
provider_specific_fields=None,
**_streamed_vllm_choice_fields(chain.from_iterable(chunk.get("choices") or () for chunk in chunks)),
)
@staticmethod
@ -504,6 +508,7 @@ class ChunkProcessor:
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
**_streamed_vllm_choice_fields(_first_choices(chunks)),
}
]
),
@ -1371,6 +1376,7 @@ def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoi
index=index,
logprobs=_merged_logprobs(choices),
finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
**_streamed_vllm_choice_fields(choices),
)
@ -1504,6 +1510,31 @@ def _field_of(value: object, name: str) -> object:
return getattr(value, name, None)
def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
def _choice_extras(choice: object) -> Mapping[str, object]:
if isinstance(choice, BaseModel):
return choice.model_extra or {}
return object_mapping(choice) or {}
def _streamed_vllm_choice_fields(choices: Iterable[object]) -> Mapping[str, object]:
extras: Final = tuple(map(_choice_extras, choices))
token_id_runs: Final = tuple(object_items(fields.get("token_ids")) for fields in extras)
stop_reasons: Final = tuple(reason for fields in extras if (reason := fields.get("stop_reason")) is not None)
has_token_ids: Final = any(fields.get("token_ids") is not None for fields in extras)
streamed_values: Final[Mapping[str, object]] = MappingProxyType(
{
"token_ids": list(chain.from_iterable(token_id_runs)) if has_token_ids else None,
"stop_reason": stop_reasons[-1] if stop_reasons else None,
}
)
present_fields: Final = frozenset(chain.from_iterable(fields.keys() for fields in extras))
return MappingProxyType({field: streamed_values[field] for field in VLLM_CHOICE_FIELDS if field in present_fields})
def _delta_reasoning_items(delta: object) -> Sequence[object]:
items: Final = _field_of(delta, "reasoning_items")
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
@ -1776,6 +1807,19 @@ def stream_chunk_builder(
end_time: object = None,
logging_obj: Optional["Logging"] = None,
count_prompt_tokens: Callable[[], int] | None = None,
) -> ModelResponse | TextCompletionResponse | None:
response: Final = _assembled_stream_response(chunks, messages, logging_obj, count_prompt_tokens)
if response is not None and getattr(logging_obj, "drops_stream_token_ids", False) is True:
for choice in response.choices:
drop_token_ids(choice)
return response
def _assembled_stream_response(
chunks: Sequence[object],
messages: Sequence[AllMessageValues | Message] | None,
logging_obj: Optional["Logging"],
count_prompt_tokens: Callable[[], int] | None,
) -> ModelResponse | TextCompletionResponse | None:
try:
if chunks is None:
@ -1788,7 +1832,7 @@ def stream_chunk_builder(
if not chunks:
return None
processor: Final = ChunkProcessor(chunks, messages)
processor: Final = ChunkProcessor(list(chunks), messages)
chunks = processor.chunks
### BASE-CASE ###

196
litellm/litellm_core_utils/streaming_handler.py
View file

@ -23,6 +23,7 @@ from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
@ -37,10 +38,11 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_param,
stored_attribute,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
@ -137,6 +139,70 @@ def _prepend_upstream_choice_extras(
choice.__pydantic_fields_set__.update(upstream_extras)
_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
match getattr(upstream_chunk, "choices", None):
case [BaseModel() as first_choice, *_]:
return first_choice
case _:
return None
def _upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
first_choice: Final = _upstream_first_choice(upstream_chunk)
if first_choice is None:
return _NO_UPSTREAM_CHOICE_FIELDS
upstream_extras: Final[Mapping[str, object]] = first_choice.model_extra or {}
return MappingProxyType({field: upstream_extras[field] for field in fields if field in upstream_extras})
def _upstream_choice_index(upstream_chunk: object) -> int:
upstream_index: Final[object] = getattr(_upstream_first_choice(upstream_chunk), "index", 0)
return upstream_index if isinstance(upstream_index, int) else 0
def _response_choice_index(response_obj: Mapping[str, object]) -> int:
choice_index: Final = response_obj.get("choice_index")
if isinstance(choice_index, int):
return choice_index
return _upstream_choice_index(response_obj.get("original_chunk"))
def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
first_choice: Final = _upstream_first_choice(upstream_chunk)
upstream_extras: Final = first_choice.model_extra if first_choice is not None else None
if not upstream_extras:
return _NO_UPSTREAM_CHOICE_FIELDS
return MappingProxyType(
{field: upstream_extras[field] for field in fields if upstream_extras.get(field) is not None}
)
def _apply_choice_fields(choice: StreamingChoices, fields: Mapping[str, object]) -> None:
for field, value in fields.items():
setattr(choice, field, value)
def _choice_fields_chunk(
model_response: ModelResponseStream, upstream_index: int, fields: Mapping[str, object]
) -> ModelResponseStream:
model_response.choices[0].delta = Delta(content="") # rebind-ok: chunks are built in place here
model_response.choices[0].index = upstream_index # rebind-ok: chunks are built in place here
_apply_choice_fields(model_response.choices[0], fields)
return model_response
def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
if chunk is None or not litellm.post_call_rules:
return chunk
for choice in chunk.choices:
drop_token_ids(choice)
return chunk
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
@ -378,6 +444,8 @@ class CustomStreamWrapper:
self._provider_response_model: str | None = None
self._upstream_provider: str | None = None
self.received_finish_reason: str | None = None
self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
self.special_tokens = [
@ -823,6 +891,7 @@ class CustomStreamWrapper:
"logprobs": logprobs,
"original_chunk": str_line,
"usage": usage,
"chunk_choice_fields": _non_null_upstream_choice_fields(str_line, STREAM_CHUNK_CHOICE_FIELDS),
}
except Exception as e:
raise e
@ -847,7 +916,7 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def handle_openai_text_completion_chunk(self, chunk):
def handle_openai_text_completion_chunk(self, chunk: object):
try:
text = ""
is_finished = False
@ -860,11 +929,15 @@ class CustomStreamWrapper:
is_finished = True
finish_reason = choices[0].finish_reason
usage = getattr(chunk, "usage", None)
choice_index: Final = _upstream_choice_index(chunk)
return {
"text": text,
"is_finished": is_finished,
"finish_reason": finish_reason,
"usage": usage,
"chunk_choice_fields": _non_null_upstream_choice_fields(chunk, STREAM_CHUNK_CHOICE_FIELDS),
"choice_index": choice_index,
"sibling_finish_reason": finish_reason if choice_index != 0 else None,
}
except Exception as e:
@ -1183,6 +1256,13 @@ class CustomStreamWrapper:
)
is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
_NO_UPSTREAM_CHOICE_FIELDS
if litellm.post_call_rules
else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
)
chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
if is_chunk_non_empty: # cannot set content of an OpenAI Object to be an empty string
self.raise_on_model_repetition()
@ -1270,16 +1350,40 @@ class CustomStreamWrapper:
if response_obj.get("provider_specific_fields") is not None:
completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
model_response.choices[0].delta = Delta(**completion_obj)
_index: Final[int | None] = completion_obj.get("index")
_index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index"))
if _index is not None:
model_response.choices[0].index = _index
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
if sibling_finish_reason is not None:
self._finish_sibling_choice(model_response.choices[0], sibling_finish_reason)
elif response_obj.get("is_finished"):
self._apply_finish_choice_fields(model_response.choices[0])
self._optional_combine_thinking_block_in_choices(model_response=model_response)
return model_response
else:
return
elif sibling_finish_reason is not None:
sibling_chunk: Final = _choice_fields_chunk(
model_response, _response_choice_index(response_obj), chunk_choice_fields
)
self._finish_sibling_choice(sibling_chunk.choices[0], sibling_finish_reason)
return sibling_chunk
elif self.received_finish_reason is not None:
chunk_upstream_index: Final = _response_choice_index(response_obj)
if chunk_choice_fields and self._streams_past_the_pending_finish(
model_response, response_obj, chunk_upstream_index
):
self.carried_chunk_choice_fields = MappingProxyType(
{**self.carried_chunk_choice_fields, chunk_upstream_index: chunk_choice_fields}
)
else:
another_choice_fields_chunk: Final = self._another_choice_fields_chunk(
model_response, response_obj, chunk_choice_fields
)
if another_choice_fields_chunk is not None:
return another_choice_fields_chunk
if self.sent_last_chunk is True:
# Bedrock returns the guardrail trace in the last chunk - we want to return this here
if self.custom_llm_provider == "bedrock" and "trace" in model_response:
@ -1317,12 +1421,20 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
map_finish_reason(finish_reason=self.received_finish_reason)
) # ensure consistent output to openai
self._apply_finish_choice_fields(model_response.choices[0])
if (
response_obj.get("is_finished")
and _response_choice_index(response_obj) == model_response.choices[0].index
):
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
self.sent_last_chunk = True
return model_response
elif self._has_special_delta_content(model_response):
return self._handle_special_delta_content(model_response)
elif chunk_choice_fields:
return _choice_fields_chunk(model_response, _response_choice_index(response_obj), chunk_choice_fields)
else:
if hasattr(model_response, "usage"):
self.chunks.append(model_response)
@ -1545,6 +1657,8 @@ class CustomStreamWrapper:
completion_obj["content"] = response_obj["text"]
print_verbose(f"completion obj content: {completion_obj['content']}")
if response_obj["is_finished"]:
self._record_finish_choice_fields(chunk)
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
self.received_finish_reason = response_obj["finish_reason"]
if response_obj["usage"] is not None:
_text_completion_usage: Final[Usage] = response_obj["usage"]
@ -1608,6 +1722,7 @@ class CustomStreamWrapper:
self.system_fingerprint = cached_chunk.system_fingerprint
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(cached_chunk)
else: # openai / azure chat model
self.uses_openai_chat_stream = True
if isinstance(chunk, ChatCompletionChunk):
@ -1637,6 +1752,7 @@ class CustomStreamWrapper:
f"{self.custom_llm_provider} raised a streaming error - finish_reason: error, no content string given. Received Chunk={response_obj}"
)
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(chunk)
if response_obj.get("original_chunk", None) is not None:
if hasattr(response_obj["original_chunk"], "id"):
model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
@ -1704,7 +1820,7 @@ class CustomStreamWrapper:
completion_obj=completion_obj,
)
if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
return dispatch_result.value
return _without_token_ids_under_post_call_rules(dispatch_result.value)
response_obj = dispatch_result.response_obj
model_response.model = self._served_openrouter_model() or self.model
@ -1794,7 +1910,7 @@ class CustomStreamWrapper:
response_obj=response_obj,
plain_delta_created=plain_delta_created,
)
return result
return _without_token_ids_under_post_call_rules(result)
except StopIteration:
raise StopIteration
@ -1987,6 +2103,75 @@ class CustomStreamWrapper:
return "tool_calls"
return finish_reason
def _record_finish_choice_fields(self, upstream_chunk: object) -> None:
self.received_finish_choice_fields = MappingProxyType(
{
**self.received_finish_choice_fields,
_upstream_choice_index(upstream_chunk): _upstream_choice_fields(
upstream_chunk, STREAM_FINISH_CHOICE_FIELDS
),
}
)
def _apply_finish_choice_fields(self, choice: StreamingChoices) -> None:
_apply_choice_fields(choice, self.received_finish_choice_fields.get(choice.index, _NO_UPSTREAM_CHOICE_FIELDS))
def _finish_sibling_choice(self, choice: StreamingChoices, finish_reason: str) -> None:
choice.finish_reason = map_finish_reason(finish_reason) # rebind-ok: chunks are built in place here
self._apply_finish_choice_fields(choice)
def _streams_past_the_pending_finish(
self, model_response: ModelResponseStream, response_obj: Mapping[str, object], upstream_index: int
) -> bool:
finishing_index: Final = model_response.choices[0].index
return (
not self.sent_last_chunk
and not response_obj.get("is_finished")
and upstream_index != finishing_index
and finishing_index in self.received_finish_choice_fields
)
def _take_carried_chunk_choice_fields(
self, response_obj: Mapping[str, object], chunk_choice_fields: Mapping[str, object]
) -> Mapping[str, object]:
if not self.carried_chunk_choice_fields:
return chunk_choice_fields
upstream_index: Final = _response_choice_index(response_obj)
carried: Final = self.carried_chunk_choice_fields.get(upstream_index)
if carried is None:
return chunk_choice_fields
self.carried_chunk_choice_fields = MappingProxyType(
{index: fields for index, fields in self.carried_chunk_choice_fields.items() if index != upstream_index}
)
return MappingProxyType(
{
field: [*object_items(carried.get(field)), *object_items(chunk_choice_fields.get(field))]
for field in STREAM_CHUNK_CHOICE_FIELDS
if field in carried or field in chunk_choice_fields
}
)
def _another_choice_fields_chunk(
self,
model_response: ModelResponseStream,
response_obj: Mapping[str, object],
chunk_choice_fields: Mapping[str, object],
) -> ModelResponseStream | None:
upstream_index: Final = _response_choice_index(response_obj)
is_finished: Final = bool(response_obj.get("is_finished"))
if is_finished and upstream_index == model_response.choices[0].index and not self.sent_last_chunk:
return None
finish_fields: Final = (
self.received_finish_choice_fields.get(upstream_index, _NO_UPSTREAM_CHOICE_FIELDS)
if is_finished
else _NO_UPSTREAM_CHOICE_FIELDS
)
if not chunk_choice_fields and all(value is None for value in finish_fields.values()):
return None
return _choice_fields_chunk(
model_response, upstream_index, MappingProxyType({**finish_fields, **chunk_choice_fields})
)
def finish_reason_handler(self):
model_response: Final = self.model_response_creator(
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@ -2005,6 +2190,7 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
_finish_reason if _finish_reason is not None else "stop"
)
self._apply_finish_choice_fields(model_response.choices[0])
close_tag: Final = self._close_open_think_block()
if close_tag is not None:
model_response.choices[0].delta.content = close_tag

25
litellm/litellm_core_utils/token_counter.py
View file

@ -990,7 +990,11 @@ def _count_extra(
tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
"""
reply_primer_tokens: Final = 3
tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
tool_choice_tokens: Final = (
_count_tool_choice(count_function, tool_choice)
if tool_section is None or tool_section.counts_tool_choice
else 0
)
return (
reply_primer_tokens
+ _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
@ -1029,14 +1033,29 @@ def _count_tool_choice(
return 0
def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name"))
def sends_function_tools(tools: Sequence[object] | None) -> bool:
return tools is not None and any(_named_function_definitions(tools))
def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in function.items() if key != "strict" or value}
def _count_chat_template_tools(
count_function: TokenCounterFunction,
tools: Sequence[object],
includes_system_message: bool,
tool_section: ChatTemplateToolSection,
) -> int:
functions: Final = tuple(
function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
named_functions: Final = tuple(_named_function_definitions(tools))
functions: Final = (
tuple(map(_without_false_strict, named_functions))
if tool_section.renders_strict_only_when_true
else named_functions
)
if not functions:
return 0

151
litellm/litellm_core_utils/tool_fields.py
View file

@ -1,13 +1,16 @@
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from copy import copy
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from typing import Annotated, Final, TypeAlias, TypeVar
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
@ -162,3 +165,145 @@ def resolve_deployment_tool_fields(
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))
_ToolT: Final = TypeVar("_ToolT")
HOSTED_MCP_DROP_WARNING: Final = (
"Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a "
"hosted MCP tool."
)
def _hosted_mcp_server_label(tool: object) -> str | None:
match tool:
case {"type": "mcp", "server_label": str() as server_label}:
return server_label
case {"type": "mcp"}:
return "unlabeled"
case _:
return None
def _drop_params_hint(dropped: str) -> str:
return (
f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
)
def _one_log_line(text: str) -> str:
return text.replace("\r", "").replace("\n", "")
def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError:
return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400)
def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]:
server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None)
if not server_labels:
return tools
named_servers: Final = ", ".join(f"`{label}`" for label in server_labels)
if not drops_unsupported_params(drop_params):
raise _unrunnable_mcp_error(
f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
"run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
'`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("hosted MCP tools"),
model=model,
route=route,
)
verbose_logger.warning(
HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
)
return [tool for tool in tools if _hosted_mcp_server_label(tool) is None]
MCP_SERVERS_DROP_WARNING: Final = (
"Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM "
"cannot pass mcp_servers to this route."
)
ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset(
{"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"}
)
def _mcp_server_name(server: object) -> str:
match server:
case {"name": str() as name}:
return name
case _:
return "unnamed"
def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]:
if not mcp_servers:
return ()
if not isinstance(mcp_servers, list):
return ("unnamed",)
return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers)))
def _is_mcp_toolset(tool: object) -> bool:
match tool:
case {"type": "mcp_toolset"}:
return True
case _:
return False
def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool:
additional_drop_params: Final = request_kwargs.get("additional_drop_params")
return drops_unsupported_params(request_kwargs.get("drop_params")) or (
isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params
)
def tools_without_mcp_servers(
mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool
) -> Sequence[_ToolT]:
server_names: Final = _mcp_server_names(mcp_servers)
if not server_names:
return tools
named_servers: Final = ", ".join(f"`{name}`" for name in server_names)
if not drops_unsupported_params(drop_params):
raise _unrunnable_mcp_error(
f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not "
'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` '
'tool with `server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("`mcp_servers`"),
model=model,
route=route,
)
verbose_logger.warning(
MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
)
return [tool for tool in tools if not _is_mcp_toolset(tool)]
_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"})
_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object])
def tool_params_without_hosted_mcp(
params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None
) -> _ParamsT:
tools: Final = params.get("tools")
if not isinstance(tools, list) or not tools:
return params
sent_tools: Final = _TOOLS.validate_python(tools)
if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or (
runs_hosted_mcp is not None and runs_hosted_mcp()
):
return params
kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params)
if len(kept_tools) == len(sent_tools):
return params
kept_params: Final = copy(params)
if kept_tools:
kept_params["tools"] = list(kept_tools)
return kept_params
dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS
for key in dropped_keys:
_ = kept_params.pop(key, None)
return kept_params

Some files were not shown because too many files have changed in this diff Show more