scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 175

Merge remote-tracking branch 'upstream/main' into litellm_router_failover_non_retryable

Browse source 
Upstream started re-raising failed agentic follow-up calls in the retry loop
at the same spot that re-raises guardrail interventions here, so the loop now
re-raises either one, matching the first-attempt check
This commit is contained in:
Thibault Serot 2026-11-04 13:15:43 +11:00
commit 3e38e88260
1061 changed files with 86560 additions and 17326 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/scripts/run_integration.sh
View file

@ -128,6 +128,8 @@ fi
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
export UV_OFFLINE=true
if [ "$mode" = parity ]; then
export INTEGRATION_WORKERS=1
else

13
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -6,12 +6,13 @@
## Handwritten 1-3 sentence description
<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
plain sentences proves the author understands what it does, and human-written technical English is far easier
for a reviewer to parse than AI-written English
An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
Tags still present at review time mean the summary is AI-written and has not been rewritten yet
<!-- The author writes a plain summary in their own words before requesting review.
Wrap newly AI-written summary text in the make-me-human-readable tags below, including backticks.
The author reviews or rewrites the draft and removes the tags.
When updating an existing PR, preserve the author's wording and removed tags.
Changes elsewhere in the body do not make an unchanged summary an AI draft again.
Add tags again only around newly AI-written replacement summary text, never merely to match this template.
This preserves author review across later updates (2026-11-02, #49779).
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No bullets, tables, or code names here. Everything else goes inside the Details block below -->

4
.github/workflows/compat-matrix-image.yml vendored
View file

@ -4,7 +4,7 @@ on:
pull_request:
paths:
- tests/e2e/claude_code/cron_vm/**
- tests/e2e/claude_code/pr_gate_version_resolver.py
- tests/e2e/cli_harness/npm_version_resolver.py
- .github/workflows/compat-matrix-image.yml
workflow_dispatch:
@ -36,7 +36,7 @@ jobs:
whoami
gh --version
uv --version
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
/opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
/tmp/claude-cli/claude --version
'

30
.github/workflows/deploy-dev-on-main.yml vendored
View file

@ -1,30 +0,0 @@
name: Deploy main to gateway-dev
on:
push:
branches: [main]
permissions: {}
concurrency:
group: deploy-dev-dispatch
cancel-in-progress: true
jobs:
dispatch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
id: token
with:
app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
owner: BerriAI
repositories: litellm-ops
permission-actions: write
- name: Dispatch the litellm-ops dev deploy
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

1
.github/workflows/osv-scan.yml vendored
View file

@ -5,6 +5,7 @@ on:
branches:
- main
- "litellm_**"
- "rc/**"
schedule:
- cron: "23 6 * * *"
workflow_dispatch:

2
.github/workflows/test-linting.yml vendored
View file

@ -208,7 +208,7 @@ jobs:
exit 0
fi
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

1
.github/workflows/test-unit.yml vendored
View file

@ -441,6 +441,7 @@ jobs:
tests/unit/ocr
tests/unit/passthrough
tests/unit/rag
tests/unit/cookbook
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search

85
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -1,85 +0,0 @@
name: "Weekly Load Anomaly Check"
on:
schedule:
- cron: "0 12 * * 6"
workflow_dispatch:
permissions:
contents: read
jobs:
weekly-load-anomaly:
if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-weekly-anomaly-check
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Run the weekly session anomaly test
env:
E2E_WEEKLY_ANOMALY: "1"
run: |
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

1
README.md
View file

@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | |

245
cookbook/gpt_realtime_translate.py Normal file
View file

@ -0,0 +1,245 @@
#!/usr/bin/env python3
import argparse
import asyncio
import base64
import json
import os
import sys
import wave
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlencode, urlsplit, urlunsplit
import websockets
from websockets.asyncio.client import ClientConnection
SAMPLE_RATE = 24_000
CHANNELS = 1
SAMPLE_WIDTH = 2
CHUNK_DURATION_SECONDS = 0.1
CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
RESPONSE_TIMEOUT_SECONDS = 30.0
AUDIO_EVENT_TYPES = frozenset(
{
"session.output_audio.delta",
"response.audio.delta",
"response.output_audio.delta",
}
)
TRANSCRIPT_EVENT_TYPES = frozenset(
{
"session.output_transcript.delta",
"response.text.delta",
"response.output_audio_transcript.delta",
}
)
@dataclass(frozen=True, slots=True)
class Settings:
input_wav: Path
output_wav: Path
base_url: str
model: str
target_language: str
trailing_silence_seconds: float
api_key: str
def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
sys.stdout.write(f"{message}{end}")
if flush:
sys.stdout.flush()
def write_stderr(message: str) -> None:
sys.stderr.write(f"{message}\n")
def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
parser = argparse.ArgumentParser(
description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
)
parser.add_argument("input_wav", type=Path)
parser.add_argument("--output", type=Path, default=Path("translated.wav"))
parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
parser.add_argument("--target-language", default="fr")
parser.add_argument("--trailing-silence", type=float, default=1.5)
parsed = parser.parse_args(argv)
api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
if not api_key:
return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
if parsed.trailing_silence < 0:
return "--trailing-silence must be zero or greater"
return Settings(
input_wav=parsed.input_wav,
output_wav=parsed.output,
base_url=parsed.base_url,
model=parsed.model,
target_language=parsed.target_language,
trailing_silence_seconds=parsed.trailing_silence,
api_key=api_key,
)
def translation_url(base_url: str, model: str) -> str | None:
parsed = urlsplit(base_url.rstrip("/"))
scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
if not scheme or not parsed.netloc:
return None
base_path = parsed.path.rstrip("/")
realtime_path = (
f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
)
return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
def read_pcm16_wav(path: Path) -> bytes | str:
try:
with wave.open(str(path), "rb") as source:
actual_format = (
source.getnchannels(),
source.getsampwidth(),
source.getframerate(),
source.getcomptype(),
)
expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
if actual_format != expected_format:
return (
f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
)
return source.readframes(source.getnframes())
except (OSError, EOFError, wave.Error) as exc:
return f"Unable to read {path}: {exc}"
def audio_chunks(audio: bytes) -> Iterator[bytes]:
return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
def audio_message(audio: bytes) -> str:
return json.dumps(
{
"type": "session.input_audio_buffer.append",
"audio": base64.b64encode(audio).decode("ascii"),
}
)
async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
await connection.send(
json.dumps(
{
"type": "session.update",
"session": {"audio": {"output": {"language": target_language}}},
}
)
)
while True:
raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type == "session.created":
write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
if event_type == "session.updated":
return None
if event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
async def send_audio(
connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
) -> None:
silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
try:
for chunk in audio_chunks(pcm + silence):
await connection.send(audio_message(chunk))
await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
await connection.send(json.dumps({"type": "session.close"}))
finally:
finished.set()
async def receive_translation(
connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
) -> str | None:
audio_received = asyncio.Event()
try:
with wave.open(str(output_path), "wb") as output:
output.setnchannels(CHANNELS)
output.setsampwidth(SAMPLE_WIDTH)
output.setframerate(SAMPLE_RATE)
write_stdout("Translation: ", end="", flush=True)
while True:
try:
raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
except TimeoutError:
if not sender_finished.is_set():
continue
return (
f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type in AUDIO_EVENT_TYPES:
output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
audio_received.set()
elif event_type in TRANSCRIPT_EVENT_TYPES:
write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
elif event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
elif event_type == "session.closed":
write_stdout()
return None if audio_received.is_set() else "The translation stream ended without translated audio"
except (OSError, wave.Error) as exc:
return f"Unable to write {output_path}: {exc}"
async def translate(settings: Settings, pcm: bytes) -> str | None:
url = translation_url(settings.base_url, settings.model)
if not url:
return f"Invalid --base-url: {settings.base_url}"
sender_finished = asyncio.Event()
try:
async with websockets.connect(
url,
additional_headers={"Authorization": f"Bearer {settings.api_key}"},
proxy=None,
open_timeout=20,
close_timeout=5,
) as connection:
configuration_error = await configure_session(connection, settings.target_language)
if configuration_error:
return configuration_error
async with asyncio.TaskGroup() as tasks:
receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
return receiver.result()
except Exception as exc:
return f"Translation failed: {type(exc).__name__}: {exc}"
def main(argv: Sequence[str] | None = None) -> int:
settings = parse_args(argv)
if isinstance(settings, str):
write_stderr(settings)
return 2
pcm = read_pcm16_wav(settings.input_wav)
if isinstance(pcm, str):
write_stderr(pcm)
return 2
error = asyncio.run(translate(settings, pcm))
if error:
write_stderr(error)
return 1
write_stdout(f"Translated audio: {settings.output_wav}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

184
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -3,9 +3,11 @@
import asyncio
import base64
import copy
import json
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta
from itertools import chain
from types import MappingProxyType
@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
import litellm
from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import (
@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
request_tags_from_metadata,
)
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
custom_llm_provider: ReadOnly[str | None]
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
@dataclass(frozen=True, slots=True)
class _ProviderUpload:
model_id: str
provider_file_id: str
upload_call_id: str | None = None
def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
file_hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
file_hidden_params.get("model_file_id_mapping") or {}
)
upload_call_id: Final = file_hidden_params.get("litellm_call_id")
return tuple(
_ProviderUpload(
model_id=model_id,
provider_file_id=provider_file_id,
upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
)
for model_id, provider_file_id in model_file_id_mapping.items()
)
def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
upload: Final = uploads[index]
other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
return all(other.model_id != upload.model_id for other in other_uploads)
def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
return None
return carried
async def _delete_provider_file(
llm_router: Router,
model_id: str,
provider_file_id: str,
request_data: Mapping[str, object],
) -> None:
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
delete_data: Final = {
**{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, delete_data),
)
def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
async def _delete_redundant_upload(
llm_router: Router,
upload: _ProviderUpload,
delete_request: Mapping[str, object],
parent_call_id: str | None,
) -> None:
try:
with internal_sub_call(parent_call_id):
await _delete_provider_file(
llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
)
except Exception as error:
verbose_logger.warning(
f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
async def _delete_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
delete_request: Mapping[str, object],
) -> None:
await asyncio.gather(
*(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
)
def _start_deleting_redundant_uploads(
llm_router: Router,
uploads: Sequence[_ProviderUpload],
create_file_request: CreateFileRequest,
) -> None:
if not uploads:
return
delete_request: Final = MappingProxyType(
{k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
)
with post_response_phase():
spawn_logging_task(
asyncio.get_running_loop(),
_delete_redundant_uploads(llm_router, uploads, delete_request),
task_name="managed file repeat upload cleanup",
)
def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
return prisma_client.db
@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
target_model_names_list=target_model_names_list,
)
## STORE MODEL MAPPINGS IN DB
model_mappings: Dict[str, str] = {}
for file_object in responses:
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
model_mappings.update(model_file_id_mapping)
uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
per_target_model_ids: Final = frozenset(
model_id
for model_id in {upload.model_id for upload in uploads}
if _keeps_one_upload_per_target(llm_router, model_id)
)
kept_uploads: Final = tuple(
upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
)
model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
await self.store_unified_file_id(
file_id=response.id,
@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
_start_deleting_redundant_uploads(
llm_router,
tuple(
upload
for upload in uploads
if upload not in kept_uploads and upload.model_id not in per_target_model_ids
),
create_file_request,
)
# Emit Prometheus metrics for managed file creation
prom_logger = self._get_prometheus_logger()
@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
if not specific_model_file_id_mapping:
return
filtered_data: Final = {
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
credentials = llm_router.registry().credentials(model_id, form="with_provider")
delete_data = {
**filtered_data,
**(
{"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
if credentials is not None
else {}
),
}
await llm_router.afile_delete(
model=model_id,
file_id=model_file_id,
**cast(_RouterFileCallKwargs, delete_data),
await _delete_provider_file(llm_router, model_id, model_file_id, data)
unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
delete_call_id: Final = data.get("litellm_call_id")
if unrecorded_upload is not None:
await _delete_redundant_upload(
llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
)
async def afile_content(

18
litellm-rust/crates/callbacks-legacy-python/python_contract.json
View file

@ -33,6 +33,17 @@
"api_key",
"additional_args"
],
"record_response": [
"logger",
"status",
"headers",
"body"
],
"record_api_window": [
"logger",
"start",
"end"
],
"defers_async_logging": [
"logger"
],
@ -91,6 +102,13 @@
"logger",
"head"
],
"stamp_stream_head": [
"head",
"logger",
"kwargs",
"start_time",
"end_time"
],
"stream_success": [
"logger",
"url_route",

117
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -57,6 +57,8 @@ pub struct LegacyLogging {
asynchronous: bool,
internal: bool,
cache_key: Option<String>,
/// When the provider request went out, until its window is recorded (Messages only).
api_started: Option<f64>,
}
fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
@ -87,6 +89,60 @@ impl LegacyLogging {
asynchronous,
internal: false,
cache_key: None,
api_started: None,
}
}
/// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
/// its own argument, so the logger never sees the caller's raw metadata, only the validated
/// request metadata merged in through `litellm_params`. Python updates the logger once, before
/// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
/// logged metadata is not rebuilt from what the call wrote into it since.
fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult<Bound<'py, PyDict>> {
let kwargs = self.call.kwargs().bind(py).copy()?;
if self.operation != LoggingOperation::Messages {
return Ok(kwargs);
}
let consumed: &[&str] = if refresh {
&["metadata", "litellm_metadata"]
} else {
&["metadata"]
};
for name in consumed {
if kwargs.contains(name)? {
kwargs.del_item(name)?;
}
}
Ok(kwargs)
}
/// Python's Messages handler hands the logger its request params, validated `metadata`
/// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
fn logged_request_params(
&self,
context: &RequestContext,
) -> Option<serde_json::Map<String, Value>> {
if self.operation != LoggingOperation::Messages {
return None;
}
let mut params = context
.optional_params
.as_object()
.cloned()
.unwrap_or_default();
params.insert(
"aanthropic_messages".to_owned(),
Value::Bool(self.asynchronous),
);
Some(params)
}
/// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
/// client is untimed.
fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
match self.api_started.take() {
Some(start) => self.logger()?.record_api_window(py, start, end),
None => Ok(()),
}
}
@ -111,6 +167,16 @@ impl LegacyLogging {
}
}
/// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
match self.operation {
LoggingOperation::Messages => Some(""),
LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
context.api_key.as_ref().map(|api_key| api_key.expose())
}
}
}
fn stream_billing(&self) -> Option<PassThroughStream> {
match self.operation {
LoggingOperation::Messages => Some(PassThroughStream {
@ -377,7 +443,13 @@ impl LegacyLogging {
context: &RequestContext,
) -> PyResult<HookStep<Self, Box<WireRequest>>> {
let logger = self.logger()?;
logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
logger.update_from_kwargs(
py,
&self.logging_kwargs(py, false)?,
&wire,
context,
self.logged_request_params(context).as_ref(),
)?;
let body = to_py(py, &wire.body)?
.into_bound(py)
.cast_into::<PyDict>()?;
@ -400,11 +472,14 @@ impl LegacyLogging {
self.logger()?.pre_call(
py,
self.input_description(),
context.api_key.as_ref().map(|api_key| api_key.expose()),
self.logged_api_key(context),
&body,
&headers,
&wire.url,
)?;
if self.operation == LoggingOperation::Messages && self.asynchronous {
self.api_started = Some(epoch_seconds());
}
let headers = headers
.iter()
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
@ -455,7 +530,7 @@ impl LegacyLogging {
py,
(
&logger,
self.call.kwargs(),
self.logging_kwargs(py, true)?,
&facts.provider.model,
logger.getattr("optional_params")?,
params,
@ -483,6 +558,16 @@ impl LegacyLogging {
py: Python<'_>,
raw: &RawResponse,
) -> PyResult<HookStep<Self, ()>> {
if self.operation == LoggingOperation::Messages {
// Python's Messages handler never calls `post_call` at receipt: it stores the raw
// response, and logging replays it through `AnthropicConfig.transform_response`.
self.close_api_window(py, epoch_seconds())?;
if let Some(head) = &raw.head {
self.logger()?
.record_response(py, head.status, &head.headers, &raw.body)?;
}
return Ok(HookStep::Ready(()));
}
let api_key = self
.request
.as_ref()
@ -522,6 +607,9 @@ impl LegacyLogging {
) -> PyResult<HookStep<Self, ()>> {
self.end = Some(datetime(py, timing.end_time)?);
self.error = Some(error.clone_ref(py).into_value(py));
if self.logger.is_some() {
self.close_api_window(py, timing.end_time)?;
}
if self.stream.is_some() {
return self.stream_failure(py);
}
@ -543,6 +631,20 @@ impl LegacyLogging {
head.bind(py).set_item("cache_key", key)?;
head.bind(py).set_item("cache_hit", true)?;
}
self.close_api_window(py, epoch_seconds())?;
if self.operation == LoggingOperation::Messages {
// Python's `@client` stamps the call metadata onto the stream it hands back.
Streaming::StampHead.call(
py,
(
head.bind(py),
self.logger()?.object(py),
self.call.kwargs(),
&self.start,
datetime(py, epoch_seconds())?,
),
)?;
}
Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
self.stream = Some(DeliveredStream {
chunks: PyList::empty(py).unbind(),
@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
let step = logging
.before_provider_request(py, Box::new(wire), context)
.unwrap();
let raw = RawResponse {
body: "raw response".into(),
};
let raw = RawResponse::new("raw response".into());
assert!(matches!(
logging
.on_event(
@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
py,
&locals,
c"
assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
assert logger.calls[0][1] is head
chunks = logger.calls[1][1]
assert logger.calls[1][1] is head
chunks = logger.calls[2][1]
assert len(chunks) == 2
assert chunks[0] is first
assert chunks[1] is last

51
litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs
View file

@ -5,6 +5,7 @@
use litellm_host::interceptors::{RequestContext, WireRequest};
use litellm_host_python::to_py;
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
use serde_json::Value;
use crate::logger::PythonLogger;
use crate::python::{Logging, Wrapper};
@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
request_params: Option<&serde_json::Map<String, Value>>,
) -> PyResult<()>;
/// `Logging.pre_call`.
@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
url: &str,
) -> PyResult<()>;
/// Records the provider's raw HTTP response on the logger, as Python's Messages handler
/// does with `httpx_response` and `response_headers`.
fn record_response(
&self,
py: Python<'_>,
status: u16,
headers: &[(String, String)],
body: &str,
) -> PyResult<()>;
/// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
/// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
/// `Logging.post_call`.
fn post_call(
&self,
@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
request_params: Option<&serde_json::Map<String, Value>>,
) -> PyResult<()> {
let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
let optional_params = redact(
py,
&to_py(py, &context.optional_params)?
@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
&secret_fields,
)?;
let params = PyDict::new(py);
params.set_item(
"litellm_call_id",
kwargs.bind(py).get_item("litellm_call_id")?,
)?;
params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
params.set_item("api_base", &wire.url)?;
for name in ["logger_fn", "litellm_request_debug"] {
if let Some(value) = kwargs.bind(py).get_item(name)? {
if let Some(value) = kwargs.get_item(name)? {
params.set_item(name, value)?;
}
}
for name in custom_pricing_fields(py)? {
if let Some(value) = kwargs.bind(py).get_item(&name)?
if let Some(value) = kwargs.get_item(&name)?
&& !value.is_none()
{
params.set_item(name, value)?;
}
}
for (name, value) in request_params.into_iter().flatten() {
params.set_item(name, to_py(py, value)?)?;
}
Logging::Update.call(
py,
(
@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn record_response(
&self,
py: Python<'_>,
status: u16,
headers: &[(String, String)],
body: &str,
) -> PyResult<()> {
Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
Ok(())
}
fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
Ok(())
}
fn post_call(
&self,
py: Python<'_>,

6
litellm-rust/crates/callbacks-legacy-python/src/python.rs
View file

@ -44,6 +44,10 @@ pub(crate) enum Logging {
PreCall,
#[strum(serialize = "post_call")]
PostCall,
#[strum(serialize = "record_response")]
RecordResponse,
#[strum(serialize = "record_api_window")]
RecordApiWindow,
#[strum(serialize = "defers_async_logging")]
DefersAsync,
#[strum(serialize = "defer_success")]
@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
pub(crate) enum Streaming {
#[strum(serialize = "stream_opened")]
Opened,
#[strum(serialize = "stamp_stream_head")]
StampHead,
#[strum(serialize = "stream_success")]
Success,
#[strum(serialize = "stream_failure")]

7
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -65,6 +65,12 @@ FAKES = {
'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
original_response, api_key, additional_args
),
'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
response_headers=dict(headers)
),
'record_api_window': lambda logger, start, end: logger.model_call_details.update(
llm_api_duration_ms=(end - start) * 1000
),
'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@ -85,6 +91,7 @@ FAKES = {
),
'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
'stream_opened': lambda logger, head: logger.record('stream_opened', head),
'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
'stream_success', list(chunks)
),

2
litellm-rust/crates/cost/src/price.rs
View file

@ -151,6 +151,8 @@ pub enum Uplift {
Residency(Residency),
/// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
NonGlobalEndpoint,
/// OpenAI FedRAMP endpoint (gov.api.openai.com), from `fedramp_uplift_multiplier`
FedrampEndpoint,
/// Anthropic `speed: fast`, from `provider_specific_entry.fast`
Speed(Speed),
/// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main

14
litellm-rust/crates/host-http/tests/serve.rs
View file

@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(Bytes::from(wire.url)))
},
@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
move |_, _, route_hooks, _observations| async move {
if event {
route_hooks
.after_provider_response(RawResponse {
body: "response".into(),
})
.after_provider_response(RawResponse::new("response".into()))
.await?;
} else {
route_hooks
@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(json!({"url": wire.url})))
},
@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
None,
|_, _, route_hooks, _observations| async move {
route_hooks
.after_provider_response(RawResponse { body: "raw".into() })
.after_provider_response(RawResponse::new("raw".into()))
.await?;
Err(TestError::Provider)
},

8
litellm-rust/crates/host-native/tests/driver.rs
View file

@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
)
.await?;
route_hooks
.after_provider_response(RawResponse {
body: wire.url.clone(),
})
.after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(wire.url))
},
@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
"input",
Some(sender),
|_, _, interceptors, observers| async move {
let raw = RawResponse {
body: "provider response".into(),
};
let raw = RawResponse::new("provider response".into());
observers.unwrap().emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
));

2
litellm-rust/crates/host-python/tests/support/mod.rs
View file

@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
.before_provider_request(wire(), context())
.await?;
host.interceptors
.after_provider_response(RawResponse { body: "raw".into() })
.after_provider_response(RawResponse::new("raw".into()))
.await?;
Ok(format!("{projected}|{signed}|{}", wire.url))
})

24
litellm-rust/crates/host/src/interceptors.rs
View file

@ -28,6 +28,28 @@ pub struct RequestContext {
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawResponse {
pub body: String,
/// The provider's HTTP status and headers, when the route forwards them to logging.
pub head: Option<ResponseHead>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ResponseHead {
pub status: u16,
pub headers: Vec<(String, String)>,
}
impl RawResponse {
pub fn new(body: String) -> Self {
Self { body, head: None }
}
#[must_use]
pub fn with_head(self, head: ResponseHead) -> Self {
Self {
head: Some(head),
..self
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -186,7 +208,7 @@ mod tests {
.await?;
Interceptors::after_provider_response(
&channel.interceptors,
RawResponse { body: "raw".into() },
RawResponse::new("raw".into()),
)
.await?;
Ok((sent, ()))

4
litellm-rust/crates/host/tests/observation.rs
View file

@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
#[rstest]
fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
let mut raw = RawResponse {
body: "provider response".into(),
};
let mut raw = RawResponse::new("provider response".into());
let event: CallEvent<(), (), &RawResponse> =
CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
let expected = raw.clone();

76
litellm-rust/crates/http/src/error.rs
View file

@ -58,8 +58,20 @@ pub(crate) enum Repr {
Network(#[source] reqwest::Error),
#[error("upstream request timed out")]
Timeout(#[source] reqwest::Error),
/// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
/// the response head had already arrived.
#[error("upstream request timed out")]
ReadTimeout {
#[source]
elapsed: Elapsed,
body: bool,
},
#[error("upstream request failed with status {status}: {body}")]
Status { status: u16, body: UpstreamBody },
Status {
status: u16,
body: UpstreamBody,
headers: Vec<(String, String)>,
},
#[error("invalid WebSocket URL")]
WebSocketUrl(#[source] url::ParseError),
@ -136,13 +148,59 @@ impl Repr {
impl Error {
pub fn upstream(status: u16, body: String) -> Self {
Self::upstream_with_headers(status, body, Vec::new())
}
/// An upstream failure that keeps the provider's response headers (`retry-after`,
/// `request-id`) for the caller.
pub fn upstream_with_headers(
status: u16,
body: String,
headers: Vec<(String, String)>,
) -> Self {
Repr::Status {
status,
body: UpstreamBody(body),
headers,
}
.into()
}
/// The provider's response headers on an upstream failure; empty for every other error.
pub fn upstream_headers(&self) -> &[(String, String)] {
match &*self.0 {
Repr::Status { headers, .. } => headers,
_ => &[],
}
}
/// A timeout reached before the connection was established, as opposed to while reading.
pub fn is_connect_timeout(&self) -> bool {
matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
}
/// A read the caller timed itself (httpx's per-read timeout) ran out.
pub fn read_timeout(elapsed: Elapsed) -> Self {
Repr::ReadTimeout {
elapsed,
body: false,
}
.into()
}
/// A read of the response body the caller timed itself ran out, after the head arrived.
pub fn body_read_timeout(elapsed: Elapsed) -> Self {
Repr::ReadTimeout {
elapsed,
body: true,
}
.into()
}
pub fn is_body_read_timeout(&self) -> bool {
matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
}
pub fn computed_header(name: impl Into<String>) -> Self {
Repr::ComputedHeader(name.into()).into()
}
@ -169,6 +227,7 @@ impl Error {
match &*self.0 {
Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@ -227,6 +286,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@ -259,6 +319,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -295,6 +356,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -333,6 +395,7 @@ impl Error {
| Repr::Signature(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@ -356,7 +419,10 @@ impl Error {
pub fn is_timeout(&self) -> bool {
match &*self.0 {
Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::HandshakeTimeout(_)
| Repr::DownloadTimeout(_) => true,
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -392,7 +458,7 @@ impl Error {
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match &*self.0 {
Repr::Status { status, body } => Some((*status, &body.0)),
Repr::Status { status, body, .. } => Some((*status, &body.0)),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -408,6 +474,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
| Repr::WebSocketSchemeRewrite(_)
@ -431,7 +498,7 @@ impl Error {
pub fn status(&self) -> Option<u16> {
match &*self.0 {
Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@ -482,6 +549,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
| Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)

2
litellm-rust/crates/inference-chat/src/handler.rs
View file

@ -108,7 +108,7 @@ pub(super) async fn execute(
if !status.is_success() {
return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
}
let raw = RawResponse { body: text.clone() };
let raw = RawResponse::new(text.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

2
litellm-rust/crates/inference-messages/Cargo.toml
View file

@ -24,6 +24,7 @@ litellm-tracing.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
tokio.workspace = true
tracing.workspace = true
[dev-dependencies]
@ -34,5 +35,4 @@ litellm-host-native.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }
litellm-inference-testing.workspace = true
rstest.workspace = true
tokio.workspace = true
wiremock.workspace = true

74
litellm-rust/crates/inference-messages/src/common_utils.rs
View file

@ -35,6 +35,14 @@ impl MessagesProvider {
pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
self.config
}
/// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
pub(crate) fn accepts_service_tier(self) -> bool {
matches!(
self.provider,
LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
)
}
}
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@ -119,3 +127,69 @@ mod tests {
);
}
}
/// Python's `_merge_messages_header_sources`: later sources win per header name, except
/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
/// sorted into one lowercase `anthropic-beta` entry.
pub fn merge_messages_header_sources(
sources: impl IntoIterator<Item = Map<String, Value>>,
) -> Map<String, Value> {
let mut merged = Map::new();
let mut betas = std::collections::BTreeSet::new();
for (name, value) in sources.into_iter().flatten() {
if !name.eq_ignore_ascii_case("anthropic-beta") {
merged.insert(name, value);
continue;
}
let values: Vec<&str> = match &value {
Value::String(text) => vec![text.as_str()],
Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
_ => Vec::new(),
};
betas.extend(
values
.iter()
.flat_map(|value| value.split(','))
.map(str::trim)
.filter(|beta| !beta.is_empty())
.map(str::to_owned),
);
}
if !betas.is_empty() {
merged.insert(
"anthropic-beta".to_owned(),
Value::String(betas.into_iter().collect::<Vec<_>>().join(",")),
);
}
merged
}
#[cfg(test)]
mod merge_tests {
use serde_json::{Map, Value, json};
use super::merge_messages_header_sources;
fn map(value: Value) -> Map<String, Value> {
value.as_object().cloned().unwrap()
}
#[test]
fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
let merged = merge_messages_header_sources([
map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
map(json!({"anthropic-beta": 5})),
]);
assert_eq!(
Value::Object(merged),
json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
);
}
#[test]
fn no_beta_values_leave_no_beta_header() {
let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
assert!(merged.is_empty());
}
}

191
litellm-rust/crates/inference-messages/src/handler.rs
View file

@ -10,9 +10,9 @@ use litellm_core_utils::{
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
};
use litellm_host::interceptors::{
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
ResponseHead, ResultSource, WireRequest,
};
use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
messages::{
@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
transformation::BaseMessagesConfig,
},
};
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use litellm_llms_types::formats::messages::{
MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::Value;
use super::{
Error, MessagesCall, MessagesCallResponse, MessagesRoute,
common_utils::{MessagesProvider, string_headers},
common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
constants::MESSAGES_TIMEOUT_SECS,
route::Messages,
types::invalid_request,
@ -126,14 +128,18 @@ async fn provider_call(
MessagesRequest { model, ..body },
shaping.settings.reasoning_auto_summary,
)?;
let transform_context =
MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let forwarded = string_headers(Some(merge_messages_header_sources([
extra_headers.unwrap_or_default(),
scoped,
])))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
@ -144,16 +150,13 @@ async fn provider_call(
)?;
let body = config.request_body(
config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?,
config.transform_anthropic_messages_request(trimmed, &transform_context)?,
&validated.headers,
shaping.settings.drop_params,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
with_python_default_headers(validated.headers, config.default_headers()),
&body,
),
auth: validated.auth,
@ -184,7 +187,7 @@ async fn provider_call(
.before_provider_request(
WireRequest {
url,
headers: authenticated.headers,
headers: joined_header_values(authenticated.headers),
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
},
request_context,
@ -213,6 +216,64 @@ async fn provider_call(
})
}
/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
/// case into one lowercase header whose values are joined with `", "`.
fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
let mut positions: std::collections::HashMap<String, usize> =
std::collections::HashMap::with_capacity(headers.len());
for (name, value) in headers {
match positions.entry(name.to_ascii_lowercase()) {
std::collections::hash_map::Entry::Occupied(position) => {
let present = &mut joined[*position.get()].1;
present.push_str(", ");
present.push_str(&value);
}
std::collections::hash_map::Entry::Vacant(position) => {
position.insert(joined.len());
joined.push((name, value));
}
}
}
joined
}
/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
fn with_python_default_headers(
headers: Vec<(String, String)>,
defaults: &[(&str, &str)],
) -> Vec<(String, String)> {
let missing: Vec<(String, String)> = defaults
.iter()
.filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
.map(|(name, value)| ((*name).to_string(), (*value).to_string()))
.collect();
headers.into_iter().chain(missing).collect()
}
/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
/// and sampling params before `additional_drop_params` and the environment check run.
fn requested_optional_params(
request: MessagesRequest,
provider: MessagesProvider,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
request, context,
)?;
if provider.accepts_service_tier() {
return Ok(request);
}
Ok(MessagesRequest {
params: MessagesOptionalParams {
service_tier: None,
..request.params
},
..request
})
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],
@ -245,6 +306,7 @@ async fn call_provider(
} = request;
let provider_name = provider.as_str();
log_request_body(provider_name, stream, &wire.body);
let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
let response = send(
http,
Authenticated {
@ -254,10 +316,11 @@ async fn call_provider(
&wire.url,
&wire.body,
timeout,
stream,
)
.await?;
if !response.status().is_success() {
return Err(provider_error(response).await);
return Err(provider_error(response, timeout).await);
}
let config = provider.config();
if stream {
@ -265,11 +328,24 @@ async fn call_provider(
response,
config.stream_decoder(),
provider_name,
timeout,
));
}
let head = ResponseHead {
status: response.status().as_u16(),
headers: litellm_http::request::header_pairs(response.headers()),
};
let status = head.status;
let text = response.text().await.map_err(litellm_http::Error::from)?;
log_response_body(&text);
context.response_received(&text).await?;
context
.raw_response_received(RawResponse::new(text.clone()).with_head(head))
.await?;
if serde_json::from_str::<serde::de::IgnoredAny>(&text).is_err() {
// Python raises `AnthropicError(status_code=<the response's>, message=<its text>)` for a
// body it cannot parse, and the public mapping classifies it by that status.
return Err(litellm_http::Error::upstream(status, text).into());
}
decode_response(config, &identity.model, &text)
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
}
@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
))
}
/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
async fn send(
http: &litellm_http::Client,
authenticated: Authenticated,
url: &str,
body: &Value,
timeout: Option<Duration>,
timeout: Duration,
stream: bool,
) -> Result<reqwest::Response, Error> {
let request = outbound_request(
authenticated,
url,
body,
Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
)?;
Ok(litellm_inference::outbound::send(request, http).await?)
let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
if !stream {
return Ok(litellm_inference::outbound::send(request, http).await?);
}
tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
.await
.map_err(litellm_http::Error::read_timeout)?
.map_err(Error::from)
}
async fn provider_error(response: reqwest::Response) -> Error {
/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
fn timed_chunks(
response: reqwest::Response,
timeout: Duration,
) -> impl futures_util::Stream<Item = Result<Bytes, litellm_http::Error>> + Send + 'static {
futures_util::stream::try_unfold(response, move |mut response| async move {
let chunk = tokio::time::timeout(timeout, response.chunk())
.await
.map_err(litellm_http::Error::body_read_timeout)?
.map_err(litellm_http::Error::from)?;
Ok(chunk.map(|chunk| (chunk, response)))
})
}
/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
/// its head arrives, so an error body that stalls would otherwise hold the call forever.
async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
let status = response.status().as_u16();
match response.text().await {
Ok(text) => {
let headers = litellm_http::request::header_pairs(response.headers());
match tokio::time::timeout(timeout, response.text()).await {
Ok(Ok(text)) => {
log_error_body(status, &text);
litellm_http::Error::upstream(status, text).into()
litellm_http::Error::upstream_with_headers(status, text, headers).into()
}
Err(error) => litellm_http::Error::from(error).into(),
Ok(Err(error)) => litellm_http::Error::from(error).into(),
Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
}
}
@ -329,22 +428,22 @@ fn streaming_response(
response: reqwest::Response,
decoder: Option<StreamDecoder>,
provider: &'static str,
timeout: Duration,
) -> MessagesCallResponse {
let headers = response
.headers()
.iter()
.filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
.collect();
let chunks = timed_chunks(response, timeout)
.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
let chunks = match decoder {
None => futures_util::stream::try_unfold(response, move |mut response| async move {
let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
Ok(chunk.map(|chunk| {
log_chunk(provider, "provider_response", &chunk);
(chunk, response)
}))
})
.boxed(),
Some(decode) => decoded_chunks(response, decode, provider),
None => chunks.map_err(Error::from).boxed(),
Some(decode) => decoded_chunks(
chunks.map_err(std::io::Error::other).boxed(),
decode,
provider,
),
};
MessagesCallResponse::Stream {
head: super::route::MessagesStreamHead { headers },
@ -353,15 +452,10 @@ fn streaming_response(
}
fn decoded_chunks(
response: reqwest::Response,
bytes: ByteStream,
decode: StreamDecoder,
provider: &'static str,
) -> BoxStream<'static, Result<Bytes, Error>> {
let bytes: ByteStream = response
.bytes_stream()
.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
.map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
.boxed();
futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
let Some(event) = events.try_next().await? else {
return Ok(None);
@ -964,9 +1058,12 @@ mod tests {
.send()
.await
.unwrap();
let MessagesCallResponse::Stream { mut chunks, .. } =
streaming_response(response, Some(streaming::event_stream), "test")
else {
let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
response,
Some(streaming::event_stream),
"test",
Duration::from_secs(5),
) else {
panic!("a streaming response returns chunks");
};

1
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -14,6 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use common_utils::merge_messages_header_sources;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,

14
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
/// Python adds a default unless that exact lowercase name is present, then folds the casings
/// into one header, so a capitalised caller header is sent beside the default.
async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
call: MessagesCall,
#[case] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(MessagesCall {
@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
.await;
let request = only_request(&upstream).await;
assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
assert_eq!(
request.header_values("anthropic-version"),
["2024-01-01, 2023-06-01"]
);
assert_eq!(
request.header_values("content-type"),
["application/json; charset=utf-8"]
["application/json; charset=utf-8, application/json"]
);
}

25
litellm-rust/crates/inference-messages/tests/messages/response.rs
View file

@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
}
#[rstest]
#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
#[case::not_a_message(json_response(json!({"unexpected": true})))]
#[tokio::test]
async fn an_unreadable_success_body_is_an_invalid_response(
call: MessagesCall,
#[case] response: ResponseTemplate,
) {
let upstream = upstream([response]).await;
async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
let error = run(MessagesCall {
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect_err("an unparseable body fails");
assert_eq!(error.upstream_response(), Some((200, "not json")));
}
#[rstest]
#[tokio::test]
async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
let error = run(MessagesCall {
api_key: Some("sk".into()),

43
litellm-rust/crates/inference-messages/tests/messages/stream.rs
View file

@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
.unwrap();
}
/// Answers with an error head whose promised body never arrives.
async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let connection = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0; 4096];
let _ = socket.read(&mut request).await;
socket
.write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
.await
.unwrap();
let _ = socket.read_to_end(&mut Vec::new()).await;
});
(base, connection)
}
#[rstest]
#[tokio::test]
async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
let (base, connection) = stalling_error_upstream().await;
let host = RecordingStreamHost::new(
MessagesCall {
timeout: Some(Duration::from_millis(300)),
..streaming(call, base)
},
usize::MAX,
);
let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
.await
.expect("the stalled error body gives up within the timeout")
.expect_err("a stalled error body fails the call");
assert!(error.dispatched(), "{error:?}");
assert_eq!(error.upstream_response(), None);
assert!(host.seen.into_inner().unwrap().is_empty());
tokio::time::timeout(Duration::from_secs(5), connection)
.await
.expect("timing out closes the upstream connection")
.unwrap();
}
#[rstest]
#[tokio::test]
async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {

4
litellm-rust/crates/inference-ocr/src/handler.rs
View file

@ -91,9 +91,7 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
}
fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
let raw = RawResponse {
body: String::from_utf8_lossy(body).into_owned(),
};
let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
if let Some(observers) = self.observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

2
litellm-rust/crates/inference-responses/src/handler.rs
View file

@ -114,7 +114,7 @@ pub(super) async fn execute(
});
}
let body = response.text().await.map_err(litellm_http::Error::from)?;
let raw = RawResponse { body: body.clone() };
let raw = RawResponse::new(body.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

8
litellm-rust/crates/inference/src/context.rs
View file

@ -42,9 +42,11 @@ impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
}
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
let raw = RawResponse {
body: body.to_owned(),
};
self.raw_response_received(RawResponse::new(body.to_owned()))
.await
}
pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },

28
litellm-rust/crates/inference/src/error.rs
View file

@ -146,6 +146,34 @@ impl Error {
}
}
/// The provider's response headers on an upstream failure; empty for every other error.
pub fn upstream_headers(&self) -> &[(String, String)] {
match self {
Self::Llm(error) => error.upstream_headers(),
Self::Http(error) => error.upstream_headers(),
_ => &[],
}
}
/// A read of a streamed body timed out after the response head arrived.
pub fn body_read_timeout(&self) -> bool {
match self {
Self::Llm(error) => error.is_body_read_timeout(),
Self::Http(error) => error.is_body_read_timeout(),
_ => false,
}
}
/// `Some(connect)` when the call timed out, where `connect` says the connection was never
/// established; `None` for every other error.
pub fn timeout(&self) -> Option<bool> {
match self {
Self::Llm(error) => error.is_timeout(),
Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
_ => None,
}
}
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match self {
Self::Llm(error) => error.upstream_response(),

7
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct EnabledThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
/// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
#[serde(
default,
skip_serializing_if = "Option::is_none",
deserialize_with = "crate::recognized::present"
)]
pub budget_tokens: Option<Recognized<u64>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display: Option<Recognized<ThinkingDisplay>>,

36
litellm-rust/crates/llms-types/src/formats/messages/response.rs
View file

@ -10,14 +10,29 @@ pub struct MessagesResponse {
pub content: Vec<Value>,
pub stop_reason: Option<String>,
pub stop_sequence: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
/// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
#[serde(
default,
deserialize_with = "present_value",
skip_serializing_if = "Option::is_none"
)]
pub usage: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(
default,
deserialize_with = "present_value",
skip_serializing_if = "Option::is_none"
)]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
fn present_value<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Value>, D::Error> {
<Value as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
}
#[cfg(test)]
mod tests {
use super::*;
@ -73,4 +88,21 @@ mod tests {
assert_eq!(body.get("usage").cloned(), usage);
assert_eq!(body.get("container").cloned(), container);
}
#[test]
fn explicit_null_usage_and_container_survive_a_round_trip() {
let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
"content": [], "stop_reason": null, "stop_sequence": null,
"usage": null, "container": null});
let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
}
#[test]
fn absent_usage_and_container_stay_absent() {
let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
"content": [], "stop_reason": null, "stop_sequence": null});
let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses");
assert_eq!(serde_json::to_value(parsed).expect("serializable"), body);
}
}

293
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -124,12 +124,86 @@ pub enum AnthropicBeta {
ToolSearchTool20251019,
#[strum(serialize = "inline-tools-2026-09-15")]
InlineTools20260915,
#[strum(serialize = "afk-mode-2026-01-31")]
AfkMode20260131,
#[strum(serialize = "agent-memory-2026-07-22")]
AgentMemory20260722,
#[strum(serialize = "analytics-usage-cost-2026-08-24")]
AnalyticsUsageCost20260824,
#[strum(serialize = "cache-diagnosis-2026-04-07")]
CacheDiagnosis20260407,
#[strum(serialize = "ce-plugins-2026-09-01")]
CePlugins20260901,
#[strum(serialize = "ce-user-management-2026-07-13")]
CeUserManagement20260713,
#[strum(serialize = "code-execution-2025-05-22")]
CodeExecution20250522,
#[strum(serialize = "computer-use-2024-10-22")]
ComputerUse20241022,
#[strum(serialize = "dev-full-thinking-2025-05-14")]
DevFullThinking20250514,
#[strum(serialize = "dreaming-2026-04-21")]
Dreaming20260421,
#[strum(serialize = "experimental-claude-tag-2026-10-21")]
ExperimentalClaudeTag20261021,
#[strum(serialize = "extended-cache-ttl-2025-04-11")]
ExtendedCacheTtl20250411,
#[strum(serialize = "fallback-credit-2026-06-01")]
FallbackCredit20260601,
#[strum(serialize = "fallback-credit-2026-07-01")]
FallbackCredit20260701,
#[strum(serialize = "managed-agents-2026-04-01")]
ManagedAgents20260401,
#[strum(serialize = "mcp-client-2026-09-15")]
McpClient20260915,
#[strum(serialize = "mcp-tunnels-2026-06-22")]
McpTunnels20260622,
#[strum(serialize = "message-batches-2024-09-24")]
MessageBatches20240924,
#[strum(serialize = "message-threads-2026-08-12")]
MessageThreads20260812,
#[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
MidConversationSystemClearAt20260821,
#[strum(serialize = "model-context-window-exceeded-2025-08-26")]
ModelContextWindowExceeded20250826,
#[strum(serialize = "output-300k-2026-03-24")]
Output300k20260324,
#[strum(serialize = "pdfs-2024-09-25")]
Pdfs20240925,
#[strum(serialize = "prompt-caching-2024-07-31")]
PromptCaching20240731,
#[strum(serialize = "redact-thinking-2026-02-12")]
RedactThinking20260212,
#[strum(serialize = "server-side-fallback-2026-06-01")]
ServerSideFallback20260601,
#[strum(serialize = "server-side-fallback-2026-07-01")]
ServerSideFallback20260701,
#[strum(serialize = "spend-limit-reads-2026-09-26")]
SpendLimitReads20260926,
#[strum(serialize = "structured-outputs-2025-12-15")]
StructuredOutputs20251215,
#[strum(serialize = "task-budgets-2026-03-13")]
TaskBudgets20260313,
#[strum(serialize = "telemetry-destinations-2026-08-11")]
TelemetryDestinations20260811,
#[strum(serialize = "thinking-token-count-2026-05-13")]
ThinkingTokenCount20260513,
#[strum(serialize = "token-counting-2024-11-01")]
TokenCounting20241101,
#[strum(serialize = "user-profiles-2026-03-24")]
UserProfiles20260324,
#[strum(serialize = "user-profiles-2026-08-18")]
UserProfiles20260818,
#[strum(serialize = "user-profiles-2026-09-04")]
UserProfiles20260904,
#[strum(serialize = "webhooks-2026-10-07")]
Webhooks20261007,
#[strum(default, transparent)]
Other(String),
}
impl AnthropicBeta {
pub const KNOWN: [Self; 40] = [
pub const KNOWN: [Self; 77] = [
Self::Oauth20250420,
Self::WebFetch20250910,
Self::WebSearch20250305,
@ -170,6 +244,43 @@ impl AnthropicBeta {
Self::ToolExamples20251029,
Self::ToolSearchTool20251019,
Self::InlineTools20260915,
Self::AfkMode20260131,
Self::AgentMemory20260722,
Self::AnalyticsUsageCost20260824,
Self::CacheDiagnosis20260407,
Self::CePlugins20260901,
Self::CeUserManagement20260713,
Self::CodeExecution20250522,
Self::ComputerUse20241022,
Self::DevFullThinking20250514,
Self::Dreaming20260421,
Self::ExperimentalClaudeTag20261021,
Self::ExtendedCacheTtl20250411,
Self::FallbackCredit20260601,
Self::FallbackCredit20260701,
Self::ManagedAgents20260401,
Self::McpClient20260915,
Self::McpTunnels20260622,
Self::MessageBatches20240924,
Self::MessageThreads20260812,
Self::MidConversationSystemClearAt20260821,
Self::ModelContextWindowExceeded20250826,
Self::Output300k20260324,
Self::Pdfs20240925,
Self::PromptCaching20240731,
Self::RedactThinking20260212,
Self::ServerSideFallback20260601,
Self::ServerSideFallback20260701,
Self::SpendLimitReads20260926,
Self::StructuredOutputs20251215,
Self::TaskBudgets20260313,
Self::TelemetryDestinations20260811,
Self::ThinkingTokenCount20260513,
Self::TokenCounting20241101,
Self::UserProfiles20260324,
Self::UserProfiles20260818,
Self::UserProfiles20260904,
Self::Webhooks20261007,
];
pub fn as_str(&self) -> &str {
@ -195,10 +306,9 @@ impl AnthropicBeta {
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 => {
Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -207,98 +317,99 @@ impl AnthropicBeta {
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260904 => &[
Self::AfkMode20260131
| Self::CePlugins20260901
| Self::CeUserManagement20260713
| Self::Compact20260904
| Self::Dreaming20260421
| Self::ExperimentalClaudeTag20261021
| Self::ExtendedCacheTtl20250411
| Self::FallbackCredit20260601
| Self::FallbackCredit20260701
| Self::InlineTools20260915
| Self::ManagedAgents20260401
| Self::McpTunnels20260622
| Self::MidConversationSystemClearAt20260821
| Self::ModelContextWindowExceeded20250826
| Self::Oauth20250420
| Self::Pdfs20240925
| Self::PromptCaching20240731
| Self::PromptCachingScope20260105
| Self::RedactThinking20260212
| Self::ServerSideFallback20260601
| Self::SpendLimitReads20260926
| Self::StructuredOutputs20251215
| Self::TaskBudgets20260313
| Self::TelemetryDestinations20260811
| Self::ThinkingTokenCount20260513
| Self::TokenCounting20241101
| Self::UserProfiles20260324
| Self::UserProfiles20260818
| Self::UserProfiles20260904 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Bash20241022
Self::AgentMemory20260722
| Self::Bash20241022
| Self::Bash20250124
| Self::DevFullThinking20250514
| Self::McpServers20251204
| Self::StructuredOutput20240301
| Self::TextEditor20241022
| Self::TextEditor20250124 => BetaProvider::VARIANTS,
Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::CodeExecution20250825
Self::AnalyticsUsageCost20260824
| Self::CacheDiagnosis20260407
| Self::CodeExecution20250522
| Self::CodeExecution20250825
| Self::ComputerUse20241022
| Self::FastMode20260201
| Self::FilesApi20250414
| Self::McpClient20250404
| Self::McpClient20251120
| Self::PromptCachingScope20260105
| Self::McpClient20260915
| Self::MessageBatches20240924
| Self::MessageThreads20260812
| Self::Output300k20260324
| Self::ServerSideFallback20260701
| Self::Skills20251002
| Self::WebFetch20250910 => &[
| Self::WebFetch20250910
| Self::Webhooks20261007 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ClaudeCode20250219
| Self::PerTurnControl20260701
| Self::ToolExamples20251029 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::Compact20260112
| Self::ContextManagement20250627
| Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::Effort20251124
| Self::FineGrainedToolStreaming20250514
| Self::ThinkingBindingControls20260801
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]
}
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::MidConversationToolChanges20260701 => &[
Self::InterleavedThinking20250514
| Self::Output128k20250219
| Self::TokenEfficientTools20250219
| Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
],
Self::PerTurnControl20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Databricks,
],
Self::InlineTools20260915 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
Self::Other(_) => BetaProvider::VARIANTS,
}
}
@ -451,7 +562,44 @@ mod tests {
AnthropicBeta::TokenEfficientTools20250219,
AnthropicBeta::ToolExamples20251029,
AnthropicBeta::ToolSearchTool20251019,
AnthropicBeta::InlineTools20260915
AnthropicBeta::InlineTools20260915,
AnthropicBeta::AfkMode20260131,
AnthropicBeta::AgentMemory20260722,
AnthropicBeta::AnalyticsUsageCost20260824,
AnthropicBeta::CacheDiagnosis20260407,
AnthropicBeta::CePlugins20260901,
AnthropicBeta::CeUserManagement20260713,
AnthropicBeta::CodeExecution20250522,
AnthropicBeta::ComputerUse20241022,
AnthropicBeta::DevFullThinking20250514,
AnthropicBeta::Dreaming20260421,
AnthropicBeta::ExperimentalClaudeTag20261021,
AnthropicBeta::ExtendedCacheTtl20250411,
AnthropicBeta::FallbackCredit20260601,
AnthropicBeta::FallbackCredit20260701,
AnthropicBeta::ManagedAgents20260401,
AnthropicBeta::McpClient20260915,
AnthropicBeta::McpTunnels20260622,
AnthropicBeta::MessageBatches20240924,
AnthropicBeta::MessageThreads20260812,
AnthropicBeta::MidConversationSystemClearAt20260821,
AnthropicBeta::ModelContextWindowExceeded20250826,
AnthropicBeta::Output300k20260324,
AnthropicBeta::Pdfs20240925,
AnthropicBeta::PromptCaching20240731,
AnthropicBeta::RedactThinking20260212,
AnthropicBeta::ServerSideFallback20260601,
AnthropicBeta::ServerSideFallback20260701,
AnthropicBeta::SpendLimitReads20260926,
AnthropicBeta::StructuredOutputs20251215,
AnthropicBeta::TaskBudgets20260313,
AnthropicBeta::TelemetryDestinations20260811,
AnthropicBeta::ThinkingTokenCount20260513,
AnthropicBeta::TokenCounting20241101,
AnthropicBeta::UserProfiles20260324,
AnthropicBeta::UserProfiles20260818,
AnthropicBeta::UserProfiles20260904,
AnthropicBeta::Webhooks20261007
)]
beta: AnthropicBeta,
) {
@ -551,7 +699,7 @@ mod tests {
BetaProvider::Anthropic,
Some(AnthropicBeta::Oauth20250420)
)]
#[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
#[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
#[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
fn on_resolves_known_spellings_held_as_other(
#[case] raw: &str,
@ -580,12 +728,11 @@ mod tests {
#[test]
fn unknown_values_are_kept_verbatim() {
let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
assert_eq!(
parsed,
AnthropicBeta::Other("claude-code-20250219".to_string())
let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
assert!(
matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
);
assert_eq!(parsed.to_string(), "claude-code-20250219");
assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
}
#[test]

10
litellm-rust/crates/llms-types/src/recognized.rs
View file

@ -7,6 +7,16 @@ pub enum Recognized<T> {
Unrecognized(Value),
}
/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay
/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`.
pub fn present<'de, D, T>(deserializer: D) -> Result<Option<Recognized<T>>, D::Error>
where
D: serde::Deserializer<'de>,
Recognized<T>: serde::Deserialize<'de>,
{
<Recognized<T> as serde::Deserialize<'de>>::deserialize(deserializer).map(Some)
}
impl<T> Recognized<T> {
pub fn known(&self) -> Option<&T> {
match self {

5
litellm-rust/crates/llms/src/anthropic/beta_headers.rs
View file

@ -96,7 +96,10 @@ mod tests {
}
#[rstest]
#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
#[case::azure_ai(
BetaProvider::AzureAi,
"advisor-tool-2026-03-01,example-beta-2099-01-01"
)]
#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
#[case::vertex_ai(BetaProvider::VertexAi, " , ")]
fn filter_removes_header_when_nothing_survives(

57
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -154,11 +154,42 @@ pub fn complete_anthropic_url(
) -> String {
let api_base = resolve_anthropic_api_base(api_base, env_lookup);
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH) {
ensure_api_route(&api_base, MESSAGES_PATH)
}
/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds.
fn url_path(url: &str) -> &str {
let before_fragment = url.split('#').next().unwrap_or_default();
let before_query = before_fragment.split('?').next().unwrap_or_default();
match before_query.split_once("://") {
Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]),
None => before_query,
}
}
/// Python's `_strip_trailing_slashes_keeping_the_scheme`.
fn strip_trailing_slashes(api_base: &str) -> String {
let stripped = api_base.trim_end_matches('/');
if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') {
return api_base.to_string();
}
format!("{api_base}{MESSAGES_PATH}")
match stripped.split_once('?') {
Some((before, query)) if url_path(stripped).ends_with('/') => {
format!("{}?{query}", before.trim_end_matches('/'))
}
_ => stripped.to_string(),
}
}
/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as
/// it is, query string included; anything else gets the route appended to the whole string.
fn ensure_api_route(api_base: &str, route: &str) -> String {
let base = strip_trailing_slashes(api_base);
let route = route.trim_start_matches('/');
if url_path(&base).ends_with(&format!("/{route}")) {
return base;
}
format!("{}/{route}", strip_trailing_slashes(&base))
}
pub fn existing_betas(headers: &[(String, String)]) -> BetaSet {
@ -1922,4 +1953,24 @@ mod tests {
expected
);
}
#[rstest]
#[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")]
#[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")]
#[case::route_present(
"https://api.example.com/v1/messages/",
"https://api.example.com/v1/messages"
)]
#[case::route_and_query(
"https://api.example.com/v1/messages?beta=true",
"https://api.example.com/v1/messages?beta=true"
)]
#[case::query_without_route(
"https://api.example.com/?x=1",
"https://api.example.com?x=1/v1/messages"
)]
#[case::scheme_only("https://", "https:///v1/messages")]
fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) {
assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url);
}
}

233
litellm-rust/crates/llms/src/anthropic/messages/thinking.rs
View file

@ -2,8 +2,8 @@ use litellm_llms_types::{
formats::{
chat_completions::ReasoningEffort,
messages::{
EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig,
ThinkingDisplay,
AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
ThinkingConfig, ThinkingDisplay,
},
},
recognized::Recognized,
@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti
fn effort_for_budget(
budgets: &ThinkingBudgets,
budget_tokens: u64,
budget_tokens: i64,
capabilities: &MessagesModelCapabilities,
) -> EffortLevel {
if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh {
let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier);
if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh {
return EffortLevel::Xhigh;
}
if budget_tokens >= budgets.high {
if reaches(budgets.high) {
return EffortLevel::High;
}
if budget_tokens >= budgets.medium {
if reaches(budgets.medium) {
return EffortLevel::Medium;
}
EffortLevel::Low
@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) -
}
}
/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string
/// as a budget, and nothing else.
fn numeric_budget_tokens(budget: &Recognized<u64>) -> Option<i64> {
let value = match budget {
Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)),
Recognized::Unrecognized(value) => value,
};
match value {
Value::Number(number) => number
.as_i64()
.or_else(|| number.as_f64().map(|float| float.trunc() as i64)),
Value::String(text) => python_int(text),
Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None,
}
}
/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits.
fn python_int(text: &str) -> Option<i64> {
let trimmed = text.trim();
let digits = trimmed.trim_start_matches(['+', '-']);
let well_formed = trimmed.len() - digits.len() <= 1
&& !digits.is_empty()
&& !digits.starts_with('_')
&& !digits.ends_with('_')
&& !digits.contains("__")
&& digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_');
if !well_formed {
return None;
}
let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect();
cleaned.parse::<i64>().ok().or_else(|| {
Some(if cleaned.starts_with('-') {
i64::MIN
} else {
i64::MAX
})
})
}
fn python_type_name(value: &Value) -> &'static str {
match value {
Value::Null => "NoneType",
Value::Bool(_) => "bool",
Value::Number(number) if number.is_f64() => "float",
Value::Number(_) => "int",
Value::String(_) => "str",
Value::Array(_) => "list",
Value::Object(_) => "dict",
}
}
fn kept_display(
display: Option<&Recognized<ThinkingDisplay>>,
context: &ThinkingContext,
) -> Option<Recognized<ThinkingDisplay>> {
match display? {
Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => {
display.cloned()
}
Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => {
display.cloned()
}
Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None,
}
}
/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive
/// keys always, and any other key unless `drop_params` is set.
fn carried_thinking_keys(
extra: &serde_json::Map<String, Value>,
context: &ThinkingContext,
) -> serde_json::Map<String, Value> {
extra
.iter()
.filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params)
.map(|(key, value)| (key.clone(), value.clone()))
.collect()
}
/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`.
fn translate_legacy_thinking_for_adaptive_model(
request: MessagesRequest,
context: &ThinkingContext,
) -> MessagesRequest {
) -> Result<MessagesRequest, Error> {
let capabilities = &context.capabilities;
if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking {
return request;
return Ok(request);
}
let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else {
return request;
return Ok(request);
};
let budget = enabled
.budget_tokens
.as_ref()
.and_then(Recognized::known)
.copied()
.unwrap_or(0);
let level = effort_for_budget(&context.budgets, budget, capabilities);
MessagesRequest {
let budget = match &enabled.budget_tokens {
None => Some(0),
Some(budget) => numeric_budget_tokens(budget),
};
if budget.is_none() && !context.drop_params {
let actual = match &enabled.budget_tokens {
Some(Recognized::Unrecognized(value)) => python_type_name(value),
Some(Recognized::Known(_)) | None => "int",
};
return Err(Error::InvalidRequest(
crate::ErrorDetail::NonNumericThinkingBudget { actual },
));
}
let thinking = ThinkingConfig::Adaptive(AdaptiveThinking {
display: kept_display(enabled.display.as_ref(), context),
extra: carried_thinking_keys(&enabled.extra, context),
});
let output_config = match budget {
Some(budget) => with_default_effort(
request.params.output_config,
effort_for_budget(&context.budgets, budget, capabilities),
),
None => request.params.output_config,
};
Ok(MessagesRequest {
params: MessagesOptionalParams {
thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))),
output_config: with_default_effort(request.params.output_config, level),
thinking: Some(Recognized::Known(thinking)),
output_config,
..request.params
},
..request
}
})
}
fn translate_adaptive_effort_for_non_adaptive_model(
@ -337,7 +435,7 @@ pub fn translate_thinking(
) -> Result<MessagesRequest, Error> {
let request = translate_reasoning_effort(request, context)?;
let request = drop_disabled_thinking(request, context);
let request = translate_legacy_thinking_for_adaptive_model(request, context);
let request = translate_legacy_thinking_for_adaptive_model(request, context)?;
let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?;
Ok(drop_incompatible_temperature_for_thinking(request, context))
}
@ -363,6 +461,7 @@ mod tests {
ThinkingContext {
capabilities,
budgets: ThinkingBudgets::default(),
..ThinkingContext::default()
}
}
@ -1239,10 +1338,102 @@ mod tests {
let context = ThinkingContext {
capabilities,
budgets: overridden_budgets(overrides),
..ThinkingContext::default()
};
assert_eq!(
translate_thinking(request(input), &context),
Ok(request(expected))
);
}
use serde_json::json;
fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext {
ThinkingContext {
drop_params,
keeps_display_updates,
..context(opus_4_7())
}
}
#[rstest]
#[case::float_budget(json!(8192.0), "xhigh")]
#[case::float_below_a_tier(json!(1023.9), "low")]
#[case::string_budget(json!(" 16_000 "), "xhigh")]
#[case::negative_budget(json!(-5), "low")]
fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) {
let translated = translate_thinking(
request(
json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}),
),
&legacy_context(false, false),
);
assert_eq!(
translated,
Ok(request(json!({
"max_tokens": 64000,
"thinking": {"type": "adaptive"},
"output_config": {"effort": effort},
})))
);
}
#[rstest]
#[case::text(json!("lots"), "str")]
#[case::list(json!([1]), "list")]
#[case::null(Value::Null, "NoneType")]
#[case::boolean(json!(true), "bool")]
fn a_non_numeric_legacy_budget_is_rejected_unless_dropped(
#[case] budget: Value,
#[case] actual: &'static str,
) {
let fields =
json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}});
assert_eq!(
translate_thinking(request(fields.clone()), &legacy_context(false, false)),
Err(Error::InvalidRequest(
crate::ErrorDetail::NonNumericThinkingBudget { actual }
))
);
assert_eq!(
translate_thinking(request(fields), &legacy_context(true, false)),
Ok(request(
json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}})
))
);
}
#[rstest]
#[case::summarized("summarized", false, Some("summarized"))]
#[case::updates_kept("updates", true, Some("updates"))]
#[case::updates_dropped("updates", false, None)]
#[case::unknown("verbose", true, None)]
fn legacy_display_survives_only_where_python_keeps_it(
#[case] display: &str,
#[case] keeps_display_updates: bool,
#[case] kept: Option<&str>,
) {
let translated = translate_thinking(
request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})),
&legacy_context(false, keeps_display_updates),
)
.unwrap();
let thinking = serde_json::to_value(translated.params.thinking).unwrap();
assert_eq!(thinking.get("display").and_then(Value::as_str), kept);
}
#[rstest]
#[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))]
#[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))]
fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) {
let translated = translate_thinking(
request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})),
&legacy_context(drop_params, false),
)
.unwrap();
assert_eq!(
serde_json::to_value(translated.params.thinking).unwrap(),
expected
);
}
}

28
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
transform_messages_request(request, context)
transform_messages_request(request, &context.keeping_display_updates())
}
fn secret_names(&self) -> &'static [&'static str] {
@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
uses_thinking_block_binding(params)
.then_some(AnthropicBeta::ThinkingBindingControls20260801),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`.
fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool {
match params.thinking.as_ref() {
Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => {
thinking.extra.contains_key("block_binding")
}
Some(Recognized::Unrecognized(Value::Object(thinking))) => {
thinking.contains_key("block_binding")
}
Some(Recognized::Unrecognized(_)) | None => false,
}
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error
})
}
fn drop_unsupported_params(
/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`.
pub fn drop_unsupported_params(
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
@ -838,6 +860,8 @@ mod tests {
low: 2000,
..ThinkingBudgets::default()
},
drop_params: true,
keeps_display_updates: false,
},
drop_params: true,
}

2
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
.capabilities
.supports_mid_conversation_system,
)),
context,
&context.keeping_display_updates(),
)
}

19
litellm-rust/crates/llms/src/base_llm/messages/context.rs
View file

@ -110,6 +110,11 @@ impl ThinkingBudgets {
pub struct ThinkingContext {
pub capabilities: MessagesModelCapabilities,
pub budgets: ThinkingBudgets,
/// `drop_params` as Python's thinking translation reads it.
pub drop_params: bool,
/// The provider accepts `display: "updates"` on thinking translated from the legacy shape
/// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`).
pub keeps_display_updates: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct MessagesTransformContext {
@ -130,8 +135,22 @@ impl MessagesTransformContext {
thinking: ThinkingContext {
capabilities,
budgets: ThinkingBudgets::from_lookup(env),
drop_params,
keeps_display_updates: false,
},
drop_params,
}
}
/// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`.
#[must_use]
pub fn keeping_display_updates(&self) -> Self {
Self {
thinking: ThinkingContext {
keeps_display_updates: true,
..self.thinking
},
..*self
}
}
}

22
litellm-rust/crates/llms/src/error.rs
View file

@ -92,6 +92,24 @@ impl Error {
}
}
pub fn upstream_headers(&self) -> &[(String, String)] {
match self {
Self::Http(error) => error.upstream_headers(),
_ => &[],
}
}
pub fn is_body_read_timeout(&self) -> bool {
matches!(self, Self::Http(error) if error.is_body_read_timeout())
}
pub fn is_timeout(&self) -> Option<bool> {
match self {
Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
_ => None,
}
}
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match self {
Self::Http(error) => error.upstream_response(),
@ -158,6 +176,10 @@ pub enum ErrorDetail {
value: String,
hint: String,
},
#[error(
"thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy"
)]
NonNumericThinkingBudget { actual: &'static str },
#[error(
"Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params"
)]

2
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = transform_messages_request(request, context)?;
let request = transform_messages_request(request, &context.keeping_display_updates())?;
let request = strip_cache_control_scope(normalize_system_role_messages(
request,
context

10
litellm-rust/crates/model-catalog/src/lower.rs
View file

@ -104,6 +104,7 @@ impl From<&EntryWire> for Adjustments {
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
(Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
(Uplift::FedrampEndpoint, wire.fedramp_uplift),
]
.into_iter()
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
@ -746,7 +747,8 @@ mod tests {
fn uplifts_are_adjustments_not_prices() {
let wire = wire(json!({
"regional_endpoint_uplift_multiplier": 1.1,
"regional_processing_uplift_multiplier_eu": "1.2"
"regional_processing_uplift_multiplier_eu": "1.2",
"fedramp_uplift_multiplier": 1.1
}));
let adjustments = Adjustments::from(&wire);
@ -764,6 +766,12 @@ mod tests {
Some(1.2)
);
assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
assert_eq!(
adjustments
.get(&Uplift::FedrampEndpoint)
.map(Multiplier::get),
Some(1.1)
);
}
#[rstest]

4
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -96,6 +96,7 @@ pub struct ModelInfoFields {
pub output_cost_per_token_ultrafast: Option<f64>,
pub regional_processing_uplift_multiplier_eu: Option<f64>,
pub regional_processing_uplift_multiplier_us: Option<f64>,
pub fedramp_uplift_multiplier: Option<f64>,
pub regional_endpoint_uplift_multiplier: Option<f64>,
pub output_cost_per_audio_token: Option<f64>,
pub output_cost_per_character: Option<f64>,
@ -186,6 +187,9 @@ pub struct ModelInfoFields {
pub supports_image_size: Option<bool>,
pub supported_audio_formats: Option<Value>,
pub token_count_framing: Option<i64>,
pub token_count_tool_framing: Option<i64>,
pub token_count_responses_framing: Option<i64>,
pub token_count_responses_tool_framing: Option<i64>,
pub token_count_model: Option<Value>,
pub token_count_tokenizer: Option<Value>,
pub vertex_ai_audio_api: Option<Value>,

2
litellm-rust/crates/model-catalog/src/schema.rs
View file

@ -29,6 +29,8 @@ struct PriceBlocks {
regional_processing_uplift_multiplier_eu: Option<Multiplier>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
regional_processing_uplift_multiplier_us: Option<Multiplier>,
/// Multiplier applied to all token costs when served from the OpenAI FedRAMP endpoint (e.g. 1.10 = +10%).
fedramp_uplift_multiplier: Option<Multiplier>,
/// USD cost per web search query, keyed by search context size.
search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.

6
litellm-rust/crates/model-catalog/src/vocabulary.rs
View file

@ -93,6 +93,12 @@ pub enum Endpoint {
Realtime,
#[serde(rename = "/v1/realtime/transcription_sessions")]
RealtimeTranscriptionSessions,
#[serde(rename = "/v1/realtime/translations")]
RealtimeTranslations,
#[serde(rename = "/v1/realtime/translations/calls")]
RealtimeTranslationCalls,
#[serde(rename = "/v1/realtime/translations/client_secrets")]
RealtimeTranslationClientSecrets,
#[serde(rename = "/v1/videos")]
Videos,
#[serde(rename = "/v1/ocr")]

31
litellm-rust/crates/model-catalog/src/wire/attributes.rs
View file

@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
use crate::vocabulary::{
AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
};
use crate::wire::lax::Count;
use crate::wire::lax::{Count, NumericCount};
macro_rules! attributes {
($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@ -93,6 +93,10 @@ attributes! {
search_free_requests: SearchFreeRequests,
/// URL of the provider pricing/model page this entry was taken from.
source: String,
/// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
successor_model: String,
/// URL of the provider page or feed `deprecation_date` was read from.
deprecation_source: String,
/// Audio container formats the model can return.
supported_audio_formats: Vec<AudioFormat>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
@ -104,9 +108,14 @@ attributes! {
/// Cloud regions the model is available in ('global' or region ids).
supported_regions: Vec<String>,
thinking_always_on: bool,
token_count_framing: u64 as Count,
/// Whether the model thinks when the request carries no thinking param.
thinking_on_by_default: bool,
token_count_framing: u64 as NumericCount,
token_count_model: TokenCountModel,
token_count_responses_framing: u64 as NumericCount,
token_count_responses_tool_framing: u64 as NumericCount,
token_count_tokenizer: TokenCountTokenizer,
token_count_tool_framing: u64 as NumericCount,
/// Provider default tokens-per-minute limit.
tpm: u64 as Count,
use_openai_responses_path: bool,
@ -298,10 +307,28 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
#[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
#[case::deprecation_source("deprecation_source", json!("https://developers.openai.com/api/docs/deprecations"), true)]
#[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
#[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
#[case::framing("token_count_framing", json!(12), true)]
#[case::framing_written_as_text("token_count_framing", json!("12"), false)]
#[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
#[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
#[case::tool_framing("token_count_tool_framing", json!(214), true)]
#[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
#[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
#[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
#[case::responses_framing("token_count_responses_framing", json!(38), true)]
#[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
#[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
#[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
fn each_attribute_reads_its_own_type(
#[case] key: &str,
#[case] value: Value,

2
litellm-rust/crates/model-catalog/src/wire/entry.rs
View file

@ -24,6 +24,7 @@ pub struct EntryWire {
pub regional_endpoint_uplift: Option<Multiplier>,
pub residency_uplift_eu: Option<Multiplier>,
pub residency_uplift_us: Option<Multiplier>,
pub fedramp_uplift: Option<Multiplier>,
pub unknown: Fields,
pub problems: Vec<Error>,
}
@ -57,6 +58,7 @@ impl EntryWire {
"regional_processing_uplift_multiplier_us" => {
self.residency_uplift_us = reader.value(path(), value)
}
"fedramp_uplift_multiplier" => self.fedramp_uplift = reader.value(path(), value),
_ => self.read_open_key(reader, key, value),
}
}

55
litellm-rust/crates/model-catalog/src/wire/lax.rs
View file

@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
pub(crate) struct Count;
pub(crate) struct NumericCount;
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
deserializer.deserialize_any(Self)
@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
}
}
impl<'de> DeserializeAs<'de, u64> for NumericCount {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
deserializer.deserialize_any(Self)
}
}
impl Visitor<'_> for NumericCount {
type Value = u64;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a whole number of zero or more")
}
fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
Ok(value)
}
fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
}
fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
integral_float(value)
.and_then(|whole| u64::try_from(whole).ok())
.ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
}
}
fn integer_string(value: &str) -> Option<i64> {
let integer = match value.split_once('.') {
Some((integer, fraction)) => {
@ -99,6 +129,12 @@ mod tests {
.map(DeserializeAsWrap::into_inner)
}
fn numeric_count(value: Value) -> Option<u64> {
DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
.ok()
.map(DeserializeAsWrap::into_inner)
}
use serde::Deserialize;
#[rstest]
@ -125,4 +161,23 @@ mod tests {
) {
assert_eq!(count(written), expected);
}
#[rstest]
#[case::integer(json!(214), Some(214))]
#[case::zero(json!(0), Some(0))]
#[case::whole_float(json!(214.0), Some(214))]
#[case::negative_zero(json!(-0.0), Some(0))]
#[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
#[case::negative(json!(-1), None)]
#[case::negative_whole_float(json!(-1.0), None)]
#[case::fraction(json!(1.5), None)]
#[case::string(json!("214"), None)]
#[case::boolean(json!(true), None)]
#[case::list(json!([214]), None)]
fn numeric_counts_accept_only_whole_json_numbers(
#[case] written: Value,
#[case] expected: Option<u64>,
) {
assert_eq!(numeric_count(written), expected);
}
}

59
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -23,6 +23,17 @@ pyo3::create_exception!(
);
const MESSAGES_REQUEST_ERROR: &str = "messages_request_error";
/// The rejection is a parameter the model does not support, which Python raises as
/// `UnsupportedParamsError`.
const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param";
/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or
/// `Body` for a streamed body read that timed out after the head arrived.
const MESSAGES_TIMEOUT: &str = "messages_timeout";
/// No credential reached the provider, which Python raises as `AuthenticationError`.
const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential";
/// Python refuses the request only once its logger knows the resolved model and provider (the
/// provider config's request transform), so the failure logs them.
const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection";
pub(crate) fn upstream_error(
py: Python<'_>,
@ -78,17 +89,57 @@ pub(crate) fn route_failure(
return Ok(raised);
}
if let Some((status, body)) = error.upstream_response() {
return public(upstream_error(py, status, body, &HeaderMap::new())?);
let headers = error
.upstream_headers()
.iter()
.filter_map(|(name, value)| {
Some((
reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?,
reqwest::header::HeaderValue::from_str(value).ok()?,
))
})
.collect::<HeaderMap>();
return public(upstream_error(py, status, body, &headers)?);
}
let native = match (marker, error.rejected_request()) {
(RequestMarker::Messages, Some(rejection)) => {
let native = PyValueError::new_err(Report::new(rejection).to_string());
native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?;
let value = native.value(py);
value.setattr(MESSAGES_REQUEST_ERROR, true)?;
if matches!(
rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
Some(
litellm_llms::ErrorDetail::UnsupportedParameter { .. }
| litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }
)
) {
value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?;
}
if matches!(
rejection.downcast_ref::<litellm_llms::ErrorDetail>(),
Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. })
) {
value.setattr(MESSAGES_LATE_REJECTION, true)?;
}
native
}
(RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => {
by_fault(error.is_request(), &error)
(RequestMarker::Messages, None) => {
let native = by_fault(error.is_request(), &error);
let value = native.value(py);
if let Some(connect) = error.timeout() {
let phase = match (connect, error.body_read_timeout()) {
(true, _) => "Connection",
(false, true) => "Body",
(false, false) => "Read",
};
value.setattr(MESSAGES_TIMEOUT, phase)?;
}
if error.is_missing_credential() {
value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
}
native
}
(RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
};
public(native)
}

21
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -34,11 +34,9 @@ fn merge_headers(
forwarded: Option<Map<String, Value>>,
extra_headers: Option<Map<String, Value>>,
) -> Option<Map<String, Value>> {
let merged: Map<String, Value> = forwarded
.into_iter()
.flatten()
.chain(extra_headers.into_iter().flatten())
.collect();
let merged = litellm_inference_messages::merge_messages_header_sources(
forwarded.into_iter().chain(extra_headers),
);
(!merged.is_empty()).then_some(merged)
}
@ -107,10 +105,19 @@ impl MessagesPythonHost {
.map(|value| python_timeout_seconds(py, value.unbind()))
.transpose()?
.flatten();
let custom_llm_provider = string("custom_llm_provider")?;
let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
let api_key = string("api_key")?;
let api_base = string("api_base")?;
// Python's handler resolves a bare model name (`claude-sonnet-4-5`) through
// `get_llm_provider`'s model lists, which the native prefix lookup does not carry.
let custom_llm_provider = match string("custom_llm_provider")? {
Some(provider) => Some(provider),
None => py
.import(ROUTE_HOST_MODULE)?
.getattr("inferred_provider")?
.call1((&model, api_base.as_deref()))?
.extract()?,
};
let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?;
let extra_headers = self.merged_headers(py, arguments)?;
let provider_specific_header = self.provider_specific_header(py, arguments)?;
let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;

13
litellm-rust/crates/python-bridge/src/routes/messages/mod.rs
View file

@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
)
}
/// Python's sync Messages handler raises for every native provider before it sends anything
/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies
/// that until both sides serve sync calls.
const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls";
#[pyfunction]
pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_messages(py, call, false)
#[pyo3(signature = (call))]
pub(crate) fn messages(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
drop(call);
Err(pyo3::exceptions::PyValueError::new_err(
SYNC_NOT_IMPLEMENTED,
))
}
#[pyfunction]

26
litellm/__init__.py
View file

@ -8,7 +8,7 @@ _apply_process_defaults()
import threading
import os
from collections.abc import Mapping, Sequence
from collections.abc import Sequence
from typing import (
Any,
Callable,
@ -688,6 +688,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
streamlake_models: Final[Set[str]] = set()
wally_models: Final[Set[str]] = set()
zerogpu_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
@ -984,6 +985,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
kyma_models.add(key)
elif value.get("litellm_provider") == "streamlake":
streamlake_models.add(key)
elif value.get("litellm_provider") == "wally":
wally_models.add(key)
elif value.get("litellm_provider") == "zerogpu":
zerogpu_models.add(key)
elif value.get("litellm_provider") == "v0":
@ -1071,7 +1074,6 @@ _populate_provider_model_sets(model_cost)
# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting
azure_llms = {
"gpt-35-turbo": "azure/gpt-35-turbo",
"gpt-35-turbo-16k": "azure/gpt-35-turbo-16k",
"gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct",
"azure/gpt-41": "gpt-4.1",
"azure/gpt-41-mini": "gpt-4.1-mini",
@ -1166,6 +1168,7 @@ model_list = list(
| openinfer_models
| kyma_models
| streamlake_models
| wally_models
| zerogpu_models
| v0_models
| morph_models
@ -1299,6 +1302,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"kyma": kyma_models,
"streamlake": streamlake_models,
"wally": wally_models,
"zerogpu": zerogpu_models,
"v0": v0_models,
"morph": morph_models,
@ -1562,7 +1566,9 @@ from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
arealtime_calls,
arealtime_translation_calls as arealtime_translation_calls,
)
from .responses.main import _aresponses_websocket
@ -1754,6 +1760,9 @@ if TYPE_CHECKING:
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)
from .llms.parallel_ai.decisions.transformation import (
ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
)
from .llms.openrouter.decisions.transformation import (
OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
)
@ -1808,6 +1817,9 @@ if TYPE_CHECKING:
from .llms.vertex_ai.rerank.transformation import (
VertexAIRerankConfig as VertexAIRerankConfig,
)
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.fireworks_ai.rerank.transformation import (
FireworksAIRerankConfig as FireworksAIRerankConfig,
)
@ -1853,9 +1865,6 @@ if TYPE_CHECKING:
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig as VertexGeminiConfig,
@ -2014,6 +2023,9 @@ if TYPE_CHECKING:
from .llms.xai.responses.transformation import (
XAIResponsesAPIConfig as XAIResponsesAPIConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
)
@ -2041,9 +2053,6 @@ if TYPE_CHECKING:
from .llms.gemini.interactions.transformation import (
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
OpenAIOSeriesConfig as OpenAIO1Config,
@ -2374,6 +2383,7 @@ if TYPE_CHECKING:
supports_web_search: Callable[..., bool]
supports_tool_search: Callable[..., bool]
supports_url_context: Callable[..., bool]
supports_continuation: Callable[..., bool]
supports_response_schema: Callable[..., bool]
supports_parallel_function_calling: Callable[..., bool]
supports_vision: Callable[..., bool]

12
litellm/_internal_context.py
View file

@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
internal_parent_request_id.reset(parent_token)
@contextmanager
def billed_separately() -> Generator[None]:
"""A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
token: Final = is_internal_call.set(False)
parent_token: Final = internal_parent_request_id.set(None)
try:
yield
finally:
is_internal_call.reset(token)
internal_parent_request_id.reset(parent_token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

4
litellm/_lazy_imports_registry.py
View file

@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
"supports_web_search",
"supports_tool_search",
"supports_url_context",
"supports_continuation",
"supports_response_schema",
"supports_parallel_function_calling",
"supports_vision",
@ -162,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"ParallelAIDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
@ -488,6 +490,7 @@ _UTILS_IMPORT_MAP: Final = {
"supports_web_search": (".utils", "supports_web_search"),
"supports_tool_search": (".utils", "supports_tool_search"),
"supports_url_context": (".utils", "supports_url_context"),
"supports_continuation": (".utils", "supports_continuation"),
"supports_response_schema": (".utils", "supports_response_schema"),
"supports_parallel_function_calling": (
".utils",
@ -738,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),

175
litellm/anthropic_beta_headers_config.json
View file

@ -37,7 +37,47 @@
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
"cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": "code-execution-2025-05-22",
"computer-use-2024-10-22": "computer-use-2024-10-22",
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"message-batches-2024-09-24": "message-batches-2024-09-24",
"message-threads-2026-08-12": "message-threads-2026-08-12",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"output-300k-2026-03-24": "output-300k-2026-03-24",
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04",
"webhooks-2026-10-07": "webhooks-2026-10-07",
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"azure_ai": {
"advisor-tool-2026-03-01": null,
@ -45,23 +85,21 @@
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"compact-2026-09-04": "compact-2026-09-04",
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@ -71,9 +109,54 @@
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
"web-search-2025-03-05": "web-search-2025-03-05",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
"cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": "code-execution-2025-05-22",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2024-10-22": "computer-use-2024-10-22",
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"message-batches-2024-09-24": "message-batches-2024-09-24",
"message-threads-2026-08-12": "message-threads-2026-08-12",
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"oauth-2025-04-20": "oauth-2025-04-20",
"output-300k-2026-03-24": "output-300k-2026-03-24",
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04",
"webhooks-2026-10-07": "webhooks-2026-10-07",
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"bedrock_converse": {
"advisor-tool-2026-03-01": null,
@ -196,34 +279,92 @@
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": null,
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"agent-memory-2026-07-22": null,
"analytics-usage-cost-2026-08-24": null,
"cache-diagnosis-2026-04-07": null,
"ce-plugins-2026-09-01": null,
"ce-user-management-2026-07-13": null,
"claude-code-20250219": "claude-code-20250219",
"code-execution-2025-05-22": null,
"compact-2026-09-04": null,
"computer-use-2024-10-22": null,
"dev-full-thinking-2025-05-14": null,
"dreaming-2026-04-21": null,
"experimental-claude-tag-2026-10-21": null,
"extended-cache-ttl-2025-04-11": null,
"fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
"fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
"managed-agents-2026-04-01": null,
"mcp-client-2026-09-15": null,
"mcp-tunnels-2026-06-22": null,
"message-batches-2024-09-24": null,
"message-threads-2026-08-12": null,
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
"oauth-2025-04-20": null,
"output-300k-2026-03-24": null,
"pdfs-2024-09-25": "pdfs-2024-09-25",
"prompt-caching-2024-07-31": null,
"server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
"server-side-fallback-2026-07-01": null,
"spend-limit-reads-2026-09-26": null,
"task-budgets-2026-03-13": "task-budgets-2026-03-13",
"telemetry-destinations-2026-08-11": null,
"thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
"token-counting-2024-11-01": "token-counting-2024-11-01",
"tool-examples-2025-10-29": "tool-examples-2025-10-29",
"user-profiles-2026-03-24": null,
"user-profiles-2026-08-18": null,
"user-profiles-2026-09-04": null,
"webhooks-2026-10-07": null,
"afk-mode-2026-01-31": "afk-mode-2026-01-31",
"redact-thinking-2026-02-12": null,
"structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
},
"vertex_ai_header_only": {
"ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
"ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
"compact-2026-09-04": "compact-2026-09-04",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"dreaming-2026-04-21": "dreaming-2026-04-21",
"experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
"extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"managed-agents-2026-04-01": "managed-agents-2026-04-01",
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
"oauth-2025-04-20": "oauth-2025-04-20",
"prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
"spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
"telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
"user-profiles-2026-03-24": "user-profiles-2026-03-24",
"user-profiles-2026-08-18": "user-profiles-2026-08-18",
"user-profiles-2026-09-04": "user-profiles-2026-09-04"
},
"databricks": {
"advisor-tool-2026-03-01": null,

50
litellm/anthropic_beta_headers_manager.py
View file

@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final, Literal, TypeAlias
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
UnlistedBetas: TypeAlias = Literal["drop", "forward"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
@ -280,12 +282,13 @@ def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
unlisted: UnlistedBetas = "drop",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
This function:
1. Only allows headers that are present in the provider's mapping keys
1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
2. Filters out headers with null values (unsupported)
3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
@ -293,6 +296,7 @@ def filter_and_transform_beta_headers(
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
Returns:
List of filtered and transformed beta headers for the provider
@ -307,6 +311,10 @@ def filter_and_transform_beta_headers(
for header in beta_headers:
header = header.strip()
if header not in provider_mapping and unlisted == "forward":
filtered_headers.add(header)
continue
# Check if header is in the mapping
if header not in provider_mapping:
verbose_logger.debug(
@ -388,9 +396,38 @@ def get_provider_beta_header(
return provider_mapping[anthropic_beta_header]
FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
"services.ai.azure.com",
"openai.azure.com",
"cognitiveservices.azure.com",
)
def is_foundry_api_base(api_base: str | None) -> bool:
"""Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
if not api_base:
return False
host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
"""
Pick the beta allowlist row for one deployment, not one provider string.
An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
"""
if provider == "anthropic" and is_foundry_api_base(api_base):
return "azure_ai"
return provider
def update_headers_with_filtered_beta(
headers: dict,
provider: str,
api_base: str | None = None,
unlisted: UnlistedBetas = "drop",
) -> dict:
"""
Update headers dict by filtering and transforming anthropic-beta header values.
@ -399,6 +436,8 @@ def update_headers_with_filtered_beta(
Args:
headers: Request headers dict (will be modified in place)
provider: Provider name
api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
unlisted: Whether a beta missing from the row is dropped or forwarded as sent
Returns:
Updated headers dict
@ -413,7 +452,8 @@ def update_headers_with_filtered_beta(
# Filter and transform based on provider
filtered_beta_values: Final = filter_and_transform_beta_headers(
beta_headers=beta_values,
provider=provider,
provider=resolve_beta_provider(provider, api_base),
unlisted=unlisted,
)
# Update or remove the header
@ -430,6 +470,7 @@ def update_request_with_filtered_beta(
headers: dict,
request_data: dict,
provider: str,
api_base: str | None = None,
) -> tuple[dict[str, str], dict[str, object]]:
"""
Update both headers and request body beta fields based on provider support.
@ -439,11 +480,12 @@ def update_request_with_filtered_beta(
headers: Request headers dict (will be modified in place)
request_data: Request body dict (will be modified in place)
provider: Provider name
api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
Returns:
Tuple of (updated headers, updated request_data)
"""
headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
existing_body_betas: Final = request_data.get("anthropic_beta")
if not existing_body_betas:
@ -451,7 +493,7 @@ def update_request_with_filtered_beta(
filtered_body_betas: Final = filter_and_transform_beta_headers(
beta_headers=existing_body_betas,
provider=provider,
provider=resolve_beta_provider(provider, api_base),
placement="body",
)

10
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
# Source: https://platform.claude.com/docs/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
400: "invalid_request_error",
401: "authentication_error",
@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
status_code: int,
message: str,
request_id: str | None = None,
error_type: AnthropicErrorType | None = None,
) -> AnthropicErrorResponse:
"""
Create an Anthropic-formatted error response dict.
@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
"request_id": "req_..."
}
"""
error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
response: Final[AnthropicErrorResponse] = {
"type": "error",
"error": {
"type": error_type,
"type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
"message": message,
},
}
@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
status_code: int,
raw_message: str,
request_id: str | None = None,
error_type: AnthropicErrorType | None = None,
) -> AnthropicErrorResponse:
"""
Transform an error message to Anthropic format.
@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
status_code: HTTP status code
raw_message: Raw error message (may be JSON string or plain text)
request_id: Optional request ID to include
error_type: Anthropic error type to use instead of the one the status maps to
Returns:
AnthropicErrorResponse dict
@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
status_code=status_code,
message=message,
request_id=request_id,
error_type=error_type,
)

6
litellm/anthropic_interface/exceptions/exceptions.py
View file

@ -6,15 +6,18 @@ from typing import Literal
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
# Known Anthropic error types
# Source: https://docs.anthropic.com/en/api/errors
# Source: https://platform.claude.com/docs/en/api/errors
AnthropicErrorType = Literal[
"invalid_request_error",
"authentication_error",
"billing_error",
"permission_error",
"not_found_error",
"conflict_error",
"request_too_large",
"rate_limit_error",
"api_error",
"timeout_error",
"overloaded_error",
]
@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
type: AnthropicErrorType
message: str
provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
litellm_error_type: NotRequired[ReadOnly[str]]
litellm_call_id: NotRequired[ReadOnly[str]]

586
litellm/blog_posts.json
View file

@ -1,10 +1,588 @@
{
"posts": [
{
"title": "Realtime WebRTC HTTP Endpoints",
"description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.",
"date": "2026-03-12",
"url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints"
"title": "Release cadence changes and an LTS version pilot",
"description": "Starting the week of November 2, 2026, release candidates and stable releases ship on Tuesday mornings, and LiteLLM is piloting a long-term support line on 1.108 for one month.",
"date": "2026-11-02",
"url": "https://docs.litellm.ai/blog/release-cadence-and-lts-pilot",
"tags": [
"release",
"support"
]
},
{
"title": "How to build an agent with grounding on LiteLLM",
"description": "Build a grounded chat agent in four steps with LiteLLM's AI Gateway. Connect your evidence, call a grounding guardrail, and show its verdict in chat.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/grounded-agent",
"tags": [
"engineering",
"guardrails",
"agents"
]
},
{
"title": "Introducing LiteLLM’s Unified Cost Accounting Engine",
"description": "Consistent cost reporting, independently scalable accounting, and spend analysis by team, user, model, or custom dimensions.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/cost-infrastructure-makeover",
"tags": [
"engineering",
"reliability"
]
},
{
"title": "What we are changing after October 21",
"description": "Why the controls built after March did not stop the v1.107.0 release, and the identity, credential and detection changes we are making in response.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/security-changes-after-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Use Hex to analyze LiteLLM costs, then set budgets and routing",
"description": "Connect Hex to LiteLLM’s Postgres data, import a cost dashboard, then use the LiteLLM UI to set a budget or change model routing.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/hex-litellm-roi",
"tags": [
"observability",
"cost tracking",
"ai-gateway"
]
},
{
"title": "Introducing the LiteLLM Private Registry",
"description": "LiteLLM is committing to a private registry to provide early access to fixes for the most severe security vulnerabilities, ahead of public release.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/litellm-private-registry",
"tags": [
"security"
]
},
{
"title": "Morph open-weight models on LiteLLM",
"description": "Use Morph's Kimi, GLM, and DeepSeek models through LiteLLM with tool calling, structured output, streaming, and prompt-cache cost tracking.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/morph-provider",
"tags": []
},
{
"title": "Abliterated models on LiteLLM",
"description": "Call abliterated models from Refuseless and Abliteration through LiteLLM, with gateway access controls, streaming, tool calls, and cached-token pricing.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/abliterated-models",
"tags": [
"engineering"
]
},
{
"title": "GPT-6.1 Terra API Pricing and Day 0 Support on LiteLLM",
"description": "GPT-6.1 Terra costs $1 per 1M input tokens and $5 per 1M output tokens, half the price of GPT-6.1 Sol. LiteLLM supports it on day 0.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/gpt-6-1-terra",
"tags": [
"openai",
"gpt-6.1",
"gpt-6.1-terra",
"pricing",
"day 0 support",
"card_title:GPT-6.1 Terra",
"card_summary:Half the price of GPT-6.1 Sol at $1 / $5 per 1M tokens, 50.3% on Terminal-Bench 4.0, 1.05M context."
]
},
{
"title": "Rust Migration: /v1/messages end-to-end in Rust by Dec 1",
"description": "By December 1st, the proxy will handle Anthropic messages requests 20x faster",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/rust-migration-rerouted",
"tags": [
"rust",
"rust-migration",
"ai-gateway",
"engineering"
]
},
{
"title": "OpenCode Go is now a LiteLLM provider",
"description": "Use your OpenCode Go subscription through LiteLLM. Connect once, route models to the right API, preserve sessions, and track estimated usage.",
"date": "2026-10-24",
"url": "https://docs.litellm.ai/blog/opencode-go",
"tags": [
"providers",
"tutorial"
]
},
{
"title": "The attack chain behind the v1.107.0 LiteLLM release",
"description": "How an internal key committed to a public repository in May led to three stolen GitHub tokens and the compromised litellm v1.107.0 release.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Tokenize text across vLLM and Watsonx",
"description": "Choose token counting for prompt budgets or /tokenize to inspect tokens. Use one request format across vLLM and Watsonx through LiteLLM.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/unified-tokenize",
"tags": [
"engineering"
]
},
{
"title": "October Townhall Updates: 426 Bug Fixes, 88.4% Coverage, and the Decisions API",
"description": "A recap of the October LiteLLM town hall: security changes that need action before you upgrade, 426 bug fixes, test coverage at 88.4%, the Decisions API, Lens, Moyai, and LiteLLM Relay.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/october-townhall-updates",
"tags": [
"townhall",
"security",
"reliability",
"product"
]
},
{
"title": "Contained: Oct 21, 2026 LiteLLM release v1.107.0 Compromised",
"description": "LiteLLM release v1.107.0 was compromised by a malicious actor.",
"date": "2026-10-21",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "JevRank: decision models through LiteLLM’s rerank API",
"description": "Preview JevRank for low-cost reranking with Perplexity and TypeSafe decision models. Explore SciFact results, reproducible data and LiteLLM setup.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/jevrank",
"tags": [
"engineering",
"reranking",
"decisions"
]
},
{
"title": "Cache-Aware Routing for Self-Hosted Models",
"description": "Route each agent session to the same vLLM pod on Kubernetes to reuse cached prompt prefixes.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/cache-aware-routing",
"tags": [
"routing",
"kubernetes",
"vllm",
"ai-gateway"
]
},
{
"title": "Day 0 Support: TypeSafe Jev 1.14: Adds images, box questions, and long context",
"description": "TypeSafe Jev 1.14 reads images, answers box questions with coordinates and takes long-context requests. LiteLLM serves all three on /v1/systemone and /v1/decisions behind TypeSafe's opt-in headers.",
"date": "2026-10-15",
"url": "https://docs.litellm.ai/blog/typesafe_jev_images",
"tags": [
"typesafe",
"jev",
"decisions",
"day 0 support",
"card_title:Jev 1.14",
"card_summary:Jev 1.14 reads PNG and JPEG images, answers box questions with coordinates, and takes long-context requests on LiteLLM."
]
},
{
"title": "Decision models on LiteLLM: /v1/decisions and /v1/systemone",
"description": "Decision models return probabilities instead of text, which makes routing, triage and guardrail checks cheap and deterministic. LiteLLM serves nine providers of them behind one request shape.",
"date": "2026-10-14",
"url": "https://docs.litellm.ai/blog/decision-models",
"tags": [
"openai",
"typesafe",
"decisions",
"day 0 support",
"card_title:Decision Models",
"card_summary:Typed yes/no, choice and score answers as probabilities, from 9 providers on /v1/decisions and /v1/systemone."
]
},
{
"title": "Workshop: Build a self-hosted cloud agent.",
"description": "Build a self-hosted cloud agent with the LiteLLM team. A two-week remote workshop, October 19–30, to build a working v0 and deploy an internal beta on your infrastructure.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/cloud-agent-workshop",
"tags": [
"agents",
"infrastructure",
"announcement"
]
},
{
"title": "Gemini 4 Argon API Pricing and Day 0 Support on LiteLLM",
"description": "Gemini 4 Argon costs $2.00 per 1M input tokens and $10.00 per 1M output tokens until Nov 30. LiteLLM supports it on day 0 on AI Studio and Google Cloud.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/gemini-4-argon",
"tags": [
"gemini",
"gemini 4",
"gemini-4-argon",
"pricing",
"day 0 support",
"card_title:Gemini 4 Argon",
"card_summary:Google's new flagship: 77.9% on DeepSWE v1.1, 1M context, thinking levels, $2 / $10 per 1M tokens until Nov 30."
]
},
{
"title": "October Townhall: Product + Roadmap Updates",
"description": "Join the LiteLLM October townhall on Friday, 23 October at 7:30 AM PT to learn about LiteLLM's product updates and roadmap.",
"date": "2026-10-10",
"url": "https://docs.litellm.ai/blog/october-townhall-announcement",
"tags": [
"announcement",
"townhall"
]
},
{
"title": "Introducing Microsoft 365 Copilot in LiteLLM",
"description": "Use Microsoft 365 Copilot from your applications through LiteLLM, with access to your Microsoft 365 work data.",
"date": "2026-10-09",
"url": "https://docs.litellm.ai/blog/microsoft-365-copilot",
"tags": [
"announcement",
"ai-gateway"
]
},
{
"title": "Introducing litellm-core: the first step toward a leaner SDK",
"description": "Ongoing work to reduce LiteLLM's dependencies and installation footprint, improve runtime loading, and make imports faster",
"date": "2026-10-08",
"url": "https://docs.litellm.ai/blog/introducing-litellm-core",
"tags": []
},
{
"title": "Claude Haiku 5.5 API Pricing and Day 0 Support on LiteLLM",
"description": "Claude Haiku 5.5 costs $0.10 per 1M input tokens and $0.50 per 1M output tokens. LiteLLM supports it on day 0 on Anthropic, AWS, Google Cloud and Azure.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/claude-haiku-5-5",
"tags": [
"anthropic",
"claude",
"claude haiku",
"haiku 5.5",
"pricing",
"day 0 support",
"card_title:Claude Haiku 5.5",
"card_summary:90% cheaper than Haiku 4.5 on short prompts, 1M context, 72.4% on OSWorld 2.1, $0.10 / $0.50 per 1M tokens."
]
},
{
"title": "Open Sourcing Moyai: Self-Hosted Cloud Coding Agent",
"description": "Run cloud coding agents on your own infrastructure with Moyai. Choose your agent and models through LiteLLM, track costs, and follow the setup guide.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/moyai-open-source",
"tags": [
"agents",
"open-source",
"infrastructure"
]
},
{
"title": "Day 0 Support: Nano Banana 2.1",
"description": "Day 0 support for Gemini Nano Banana 2.1 on LiteLLM, on Google AI Studio and Gemini Enterprise Agent Platform, with image output at half Nano Banana 2's price.",
"date": "2026-10-06",
"url": "https://docs.litellm.ai/blog/nano_banana_2_1",
"tags": [
"gemini",
"gemini enterprise agent platform",
"nano banana",
"image generation",
"day 0 support",
"card_title:Nano Banana 2.1",
"card_summary:Gemini image model with image output at half Nano Banana 2's price: $30 / MTok image out, $1.50 text in."
]
},
{
"title": "How LiteLLM Lens finds repeated failures across 1,000s of agent traces",
"description": "Inside Lens: parallel trace review, Python tools for large traces, and investigations that connect repeated failures to source evidence.",
"date": "2026-10-05",
"url": "https://docs.litellm.ai/blog/lens-failure-patterns",
"tags": [
"lens",
"agent-tracing",
"engineering"
]
},
{
"title": "How we cut time to first byte by 94% for long prompts",
"description": "A 440k-token benchmark went from 553 ms to 35 ms median time to first byte. We removed unnecessary prompt-cache routing work before the model call.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/long-prompt-latency",
"tags": [
"performance",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we built our own internal Devin in 2 days",
"description": "How we built Moyai Devin with Render, Modal, Hermes, Temporal, and LiteLLM: durable sessions, parallel agents, Slack, and shared organization connections.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/internal-devin-two-days",
"tags": [
"engineering",
"agents",
"infrastructure",
"slack"
]
},
{
"title": "Adding Self-hosted Auto Router Classifiers: Laya & Nimble",
"description": "Use Laya or Bespoke Nimble to classify Auto Router requests on your own infrastructure. Control where classification runs, which model you serve, and how you provision it.",
"date": "2026-10-02",
"url": "https://docs.litellm.ai/blog/laya-nimble-classifiers",
"tags": [
"auto-router",
"product",
"ai-gateway"
]
},
{
"title": "Launching LiteLLM Lens",
"description": "LiteLLM Lens turns the traces flowing through your gateway into findings your agents can act on. Built for agent swarms generating 200K+ traces.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/litellm-lens-launch",
"tags": [
"lens",
"agent-tracing"
]
},
{
"title": "How we cut LiteLLM's Redis round trips per request by 64%",
"description": "A governed request to the LiteLLM AI Gateway waited on Redis 22 times. It now waits 8 times: one pipeline per Redis backend before the model call, one after.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/redis-request-pipelines",
"tags": [
"performance",
"redis",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we made the LiteLLM Usage page 120x faster",
"description": "From six minutes to 3.2 seconds: how moving aggregation into Postgres made LiteLLM's Usage page roughly 120x faster in our benchmark.",
"date": "2026-09-30",
"url": "https://docs.litellm.ai/blog/usage-page-bounded-loading",
"tags": [
"performance",
"admin-ui",
"postgres",
"engineering",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6.1 Sol",
"description": "Day 0 support for GPT-6.1 Sol on LiteLLM, with cached input at half GPT-6 Sol's price.",
"date": "2026-09-29",
"url": "https://docs.litellm.ai/blog/gpt_6_1_sol",
"tags": [
"openai",
"gpt-6",
"gpt-6.1-sol",
"completion",
"day 0 support",
"card_title:GPT-6.1 Sol",
"card_summary:GPT-6 Sol successor with cached input at half GPT-6 Sol's price, same request family and params."
]
},
{
"title": "Learn the LiteLLM gateway with a guided course",
"description": "Follow a request through the LiteLLM gateway, Router, and SDK. A guided course for teams deploying the gateway and contributors making code changes.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/litellm-gateway-course",
"tags": [
"proxy",
"product"
]
},
{
"title": "Day 0 Support: Claude Sonnet 5.5",
"description": "Day 0 support for Claude Sonnet 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/claude_sonnet_5_5",
"tags": [
"anthropic",
"claude",
"sonnet 5.5",
"day 0 support",
"card_title:Claude Sonnet 5.5",
"card_summary:Same $2 / $10 price as Sonnet 5, 30%+ faster, 70.6% on Terminal-Bench 4.0 (Sonnet 5: 10.3%), 1M context."
]
},
{
"title": "Introducing the LiteLLM ROI Calculator",
"description": "Compare each engineer's LiteLLM gateway spend with the estimated effort of their merged pull requests. Open source and self-hosted.",
"date": "2026-09-26",
"url": "https://docs.litellm.ai/blog/litellm-roi-calculator",
"tags": [
"cost",
"product",
"announcement"
]
},
{
"title": "Introducing LiteAgents",
"description": "Switch agent harnesses without rewriting your agent. Keep your tools and model configuration, use native harness controls, and add Temporal when you need durable runs.",
"date": "2026-09-25",
"url": "https://docs.litellm.ai/blog/liteagents-sdk",
"tags": [
"agents",
"sdk"
]
},
{
"title": "September Townhall Updates: 583 Bug Fixes, OCR on Rust, and 83.5% Coverage",
"description": "A recap of the September LiteLLM town hall: security and stability updates, OCR running on Rust by default, test coverage past the 80% target, and new Fusion and liteagents launches.",
"date": "2026-09-24",
"url": "https://docs.litellm.ai/blog/september-townhall-updates",
"tags": [
"townhall",
"reliability",
"product"
]
},
{
"title": "Day 0 support: Gemini 3.8 Flash TTS and Flash-Lite TTS",
"description": "Day 0 support for Gemini 3.8 Flash TTS and Flash-Lite TTS on LiteLLM's /v1/audio/speech, with launch pricing tracked through 2026.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/gemini_3_8_flash_tts",
"tags": [
"gemini",
"gemini-3.8-flash-tts",
"text to speech",
"day 0 support",
"card_title:Gemini 3.8 Flash TTS and Flash-Lite TTS",
"card_summary:Gemini speech models on /v1/audio/speech, Flash and Flash-Lite, with launch pricing tracked through 2026."
]
},
{
"title": "Introducing LiteAdmin MCP",
"description": "Give your agent tools to create keys, add models, and manage budgets with LiteAdmin MCP. Use the same connector through LiteAdmin, our Slack admin agent.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/litellm-admin-mcp",
"tags": [
"mcp",
"agents",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6 Sol and GPT-6 Luna",
"description": "Day 0 support for GPT-6 Sol and GPT-6 Luna on LiteLLM, at half the price of their GPT-5.6 counterparts.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/gpt_6_sol_luna",
"tags": [
"openai",
"gpt-6",
"gpt-6-sol",
"gpt-6-luna",
"completion",
"day 0 support",
"card_title:GPT-6 Sol and GPT-6 Luna",
"card_summary:Two GPT-6 variants at half the price of their GPT-5.6 counterparts, routed as the GPT-5 request family."
]
},
{
"title": "Day 0 Support: Claude Opus 5.5",
"description": "Day 0 support for Claude Opus 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/claude_opus_5_5",
"tags": [
"anthropic",
"claude",
"opus 5.5",
"day 0 support",
"card_title:Claude Opus 5.5",
"card_summary:20% cheaper than Opus 5 at $4 / $20 per 1M, 30% faster output, Terminal-Bench 4.0 66.4%, thinking always on."
]
},
{
"title": "Getting Started with Fireworks AI on LiteLLM",
"description": "A beginner tutorial: run an open model on Fireworks AI behind a LiteLLM gateway, call it with the OpenAI SDK you already use, then add a second model and a fallback.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/fireworks-getting-started",
"tags": [
"tutorial",
"providers",
"fireworks",
"getting-started"
]
},
{
"title": "Day 0 Support: Xiaomi MiMo V2.6",
"description": "Day 0 support for Xiaomi MiMo V2.6 Pro and Flash on LiteLLM, priced on the native route for the first time.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/mimo_v2_6",
"tags": [
"xiaomi",
"mimo",
"mimo-v2.6",
"day 0 support",
"card_title:Xiaomi MiMo V2.6",
"card_summary:Xiaomi's Pro ($0.435 / $0.87 per 1M) and Flash ($0.14 / $0.28), same rates as V2.5, priced on the native route."
]
},
{
"title": "Claude Code server-side auto mode through LiteLLM",
"description": "Anthropic is moving Claude Code auto mode's safety classifier server-side. LiteLLM's native /v1/messages route now forwards the safeguards contract to the Anthropic API, Bedrock InvokeModel, Bedrock Mantle and Vertex AI. Here is the contract, what changed, which releases carry it, and how to verify it.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/claude-code-server-side-auto-mode",
"tags": [
"announcement",
"claude-code",
"anthropic",
"ai-gateway"
]
},
{
"title": "Day 0 Support: Grok 4.7",
"description": "Day 0 support for Grok 4.7 on LiteLLM, at the same price as Grok 4.6.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/grok_4_7",
"tags": [
"xai",
"grok",
"grok-4.7",
"day 0 support",
"card_title:Grok 4.7",
"card_summary:SpaceXAI's update to Grok 4.6 at the same price and limits, better at checking its own work and long context."
]
},
{
"title": "JEV Classifier: 5.43x as Fast as Haiku, 96% Lower Cost",
"description": "JEV classified requests 5.43x as fast as Haiku by median latency in our AI Gateway benchmark. Explore the setup, cost savings and methodology.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/jev-auto-router-benchmark",
"tags": [
"routing",
"complexity-router",
"engineering",
"ai-gateway"
]
},
{
"title": "TypeSafe Jev on LiteLLM",
"description": "TypeSafe AI's Jev lands in LiteLLM v1.103.0-rc: call it through the proxy with logging and cost tracking.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/typesafe_jev",
"tags": [
"typesafe",
"jev",
"product"
]
}
]
}

21
litellm/caching/caching.py
View file

@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
raw_extra_body: Final = kwargs.get("extra_body")
if not isinstance(raw_extra_body, dict):
return ""
extra_body: Final = cast( # cast-ok: isinstance leaves the key and value types unknown
Mapping[str, object], raw_extra_body
)
return "".join(
f"extra_body.{param}: {extra_body[param]}"
for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
if extra_body.get(param)
)
@contextmanager
def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
"""The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@ -429,7 +443,10 @@ class Cache:
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
cache_key
+ _keyed_provider_params_in_extra_body(kwargs)
+ reasoning_summary_material
+ tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(

2
litellm/caching/dual_cache.py
View file

@ -673,7 +673,7 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
self.redis_cache.flush_cache()
def delete_cache(self, key):
def delete_cache(self, key: str) -> None:
"""
Delete a key from the cache
"""

22
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -11,7 +11,7 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
@ -20,15 +20,15 @@ if TYPE_CHECKING:
class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
model: str
messages: list
optional_params: dict
litellm_params: dict
headers: dict
model_response: "ModelResponse"
logging_obj: "LiteLLMLoggingObj"
custom_llm_provider: str
encoding: object
model: ReadOnly[str]
messages: ReadOnly[list]
optional_params: ReadOnly[dict[str, object]]
litellm_params: ReadOnly[dict[str, object]]
headers: ReadOnly[dict]
model_response: ReadOnly["ModelResponse"]
logging_obj: ReadOnly["LiteLLMLoggingObj"]
custom_llm_provider: ReadOnly[str]
encoding: ReadOnly[object]
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
request_data: Final = self.transformation_handler.transform_request(
model=model,
@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
try:
request_data: Final = self.transformation_handler.transform_request(

83
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import (
ResponseCustomToolCallOutputParam,
ResponseCustomToolCallParam,
)
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
from openai.types.responses.tool_param import FunctionToolParam
@ -33,11 +34,17 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
replay_items,
requested_phase,
)
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
prompt_cache_breakpoint_with_readable_mode,
responses_reasoning_items_from_thinking_blocks,
whole_prompt_cache_breakpoint_or_none,
with_prompt_cache_breakpoint,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@ -97,6 +104,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
)
def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
if target_trims_breakpoints:
return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
if not isinstance(value, dict):
return value
@ -775,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _normalize_tool_choice_for_responses_api(
self, tool_choice: _ToolChoiceT
) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
) -> (
_ToolChoiceT
| ToolChoiceFunctionParam
| ToolChoiceCustomParam
| ToolChoiceAllowedParam
| Literal["auto", "none", "required"]
):
"""Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
if not isinstance(tool_choice, dict):
return tool_choice
choice_type: Final = tool_choice.get("type")
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
return choice_type
if choice_type == "allowed_tools":
parsed: Final = parse_chat_allowed_tools_tool_choice(
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice
)
)
flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None
if flat is not None:
return flat
if choice_type not in ("function", "custom"):
return tool_choice
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@ -844,6 +872,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
keep_prompt_cache_breakpoints: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
@ -852,6 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
messages,
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_openai_history=replay_openai_history,
@ -868,6 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
@ -921,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content,
role,
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
}
)
@ -940,6 +972,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content,
"user", # Use "user" role to get input_* types
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
)
else:
# Fallback: convert unexpected types to input_text
@ -968,7 +1001,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": "assistant",
"content": self._convert_content_to_responses_format(
content, "assistant", drop_params=drop_params
content,
"assistant",
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
**_assistant_phase_field(msg, replay_openai_history, drop_params),
}
@ -1009,7 +1045,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, cast(str, role), drop_params=drop_params
content,
cast(str, role),
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
),
**_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
}
@ -1127,14 +1166,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params = self._extract_extra_body_params(optional_params)
provider: Final = _served_provider(litellm_params)
base_model: Final = litellm_params.get("base_model")
supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or (
isinstance(base_model, str)
and bool(base_model)
and supports_openai_prompt_cache_breakpoint(base_model, provider)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
supports_prompt_cache_breakpoint: Final = any(
supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
@ -1195,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
api_model: Final = model
from litellm.types.utils import CallTypes
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
request_data: Final = {
@ -1640,6 +1676,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
| None,
role: str,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
) -> list[dict[str, object]]:
"""Convert chat completion content to responses API format"""
from litellm.types.llms.openai import ChatCompletionImageObject
@ -1666,7 +1703,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if original_type == "text":
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(item.get("text", ""), role),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: text -> %s", converted)
@ -1679,14 +1718,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
cast(ChatCompletionImageObject, item), role
),
),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: image_url -> %s", converted)
elif original_type == "input_audio":
converted = with_prompt_cache_breakpoint(
{"type": "input_audio", "input_audio": item.get("input_audio")},
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: input_audio -> %s", converted)
@ -1702,7 +1745,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
_input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: file -> %s", converted)
@ -1726,7 +1771,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Default to input_text for unknown types
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(str(item.get("text", item)), role),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
_bridged_prompt_cache_breakpoint(
item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
),
)
result.append(converted)
verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted)

11
litellm/constants.py
View file

@ -135,6 +135,7 @@ ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
DEFAULT_MAX_REQUEST_HEADER_SIZE: Final = 16 * 1024
budget_reservation_disabled_info_emitted = False
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)
DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512)
@ -451,6 +452,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
{"response.completed", "response.incomplete", "response.failed"}
)
REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
# SSL/TLS cipher configuration for faster handshakes
# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
@ -699,6 +701,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
"DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400
@ -968,7 +973,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
OPENAI_TRANSCRIPTION_PARAMS: Final = [
"language",
"languages",
"keywords",
"response_format",
"stream",
"timestamp_granularities",
]
@ -1085,6 +1093,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.scx.ai/v1",
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://inference.runanywhere.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
@ -1172,6 +1181,7 @@ openai_compatible_providers: Final[list[str]] = [
"sail",
"reka",
"streamlake",
"wally",
]
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
@ -2060,6 +2070,7 @@ SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30
SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000
SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000
SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS: Final = 30
SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE: Final = 100
# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
# callers from forcing a DB query per request for unknown names, while bounding

381
litellm/cost_calculator.py
View file

@ -6,6 +6,7 @@ import time
from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -22,9 +23,14 @@ from litellm.constants import (
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
REALTIME_METERED_USAGE_KEY,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
from litellm.litellm_core_utils.decisions_pricing import (
decisions_pricing_fields,
decisions_request_price,
is_decisions_call_type,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
@ -43,6 +49,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_batch_cost_rates,
get_billable_input_tokens,
get_cost_per_unit,
get_modality_token_cost_key,
get_regional_uplift_multiplier,
get_service_tier_cost_key,
get_token_type_cost_breakdown,
@ -319,12 +327,41 @@ def _get_additional_costs(
return None
def _transcription_usage_has_token_details(
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second",
"output_cost_per_second",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
"input_cost_per_token",
"output_cost_per_token",
]
_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
)
_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_second",
"output_cost_per_second",
)
def _transcription_uses_token_pricing(
model: str,
custom_llm_provider: str | None,
usage_block: Usage | None,
) -> bool:
if usage_block is None:
return False
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
if (
_declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
):
return False
prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@ -338,13 +375,38 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
if pricing_model is None or custom_llm_provider is None:
return False
transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
transcription_metadata: Final = (
as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
)
return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
def _transcription_duration_for_cost(
completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
) -> float:
hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
measured_duration: Final = normalized_audio_duration_seconds(
hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
)
if measured_duration is not None and measured_duration > 0:
return measured_duration
provider_duration: Final = _get_transcription_usage_duration(completion_response)
if (
provider_duration is not None
and provider_duration > 0
and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
):
return provider_duration
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return 0.0
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -382,6 +444,14 @@ def _per_second_pricing_cost(
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds)
def _per_second_pricing_cost_from_info(
model_info: ModelInfoBase,
response_time_ms: float | None,
audio_seconds: float = 0.0,
) -> tuple[float, float] | None:
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
cost_per_second: Final = model_info.get("cost_per_second")
@ -404,7 +474,7 @@ def _per_second_pricing_cost(
)
verbose_logger.debug(
"For model=%s - cost_per_second: %s; response time: %s",
model,
model_info.get("key"),
resolved_cost_per_second,
response_time_ms,
)
@ -427,6 +497,7 @@ def cost_per_token(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
### NUMBER OF QUERIES ###
number_of_queries: int | None = None,
### USAGE OBJECT ###
@ -538,6 +609,20 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
if requested_model_info_override is not None:
per_second: Final = _per_second_pricing_cost_from_info(
requested_model_info_override, response_time_ms, audio_transcription_file_duration
)
if per_second is not None:
return per_second
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider or "",
service_tier=service_tier,
model_info=requested_model_info_override,
)
model_cost_ref: Final = litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
@ -604,6 +689,9 @@ def cost_per_token(
decisions_overrides: Final = decisions_pricing_fields(
litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
)
request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
if request_price is not None:
return request_price, 0.0
if decisions_overrides:
return generic_cost_per_token(
model=model,
@ -693,7 +781,7 @@ def cost_per_token(
vertex_location=vertex_location,
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -1039,20 +1127,19 @@ def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_nam
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
def declared_cost_map_pricing(key: str, model: str, custom_llm_provider: str | None) -> Mapping[str, object]:
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return unmapped_model_generalization(model, custom_llm_provider) or MappingProxyType({})
fill_missing: Final = mapped_model_fill_missing(key, entry)
return entry if fill_missing is None else MappingProxyType({**fill_missing, **entry})
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
return key is not None and _cost_map_entry_prices_anything(
declared_cost_map_pricing(key, model, custom_llm_provider)
)
def _served_or_priced_requested_model(
@ -1275,6 +1362,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
)
def _get_transcription_usage_duration(completion_response: object) -> float | None:
response_fields: Final = as_str_mapping(completion_response)
usage_object: Final = (
response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
)
usage_fields: Final = as_str_mapping(usage_object)
usage_type: Final = (
usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
)
if usage_type != "duration":
return None
seconds: Final = (
usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
)
return normalized_audio_duration_seconds(seconds)
def _extraction_usage(usage: ExtractionUsage) -> Usage:
return Usage(
prompt_tokens=usage.input_tokens,
@ -1668,6 +1772,7 @@ def completion_cost(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
optional_params: dict | None = None,
custom_pricing: bool | None = None,
base_model: str | None = None,
@ -1721,6 +1826,7 @@ def completion_cost(
quality=quality,
n=n,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
custom_cost_per_second=custom_cost_per_second,
optional_params=optional_params,
custom_pricing=custom_pricing,
@ -2000,20 +2106,8 @@ def completion_cost(
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
# the response attribute (for verbose_json responses that
# naturally include duration from the provider).
_hidden = getattr(completion_response, "_hidden_params", {}) or {}
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
audio_transcription_file_duration = _transcription_duration_for_cost(
completion_response, model, requested_model, custom_llm_provider
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -2123,6 +2217,7 @@ def completion_cost(
litellm_logging_obj=litellm_logging_obj,
custom_pricing_model=selected_model if custom_pricing else None,
base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
deployment_models=(base_model, requested_model),
)
elif call_type == _MCP_CALL_TYPE:
from litellm.proxy.mcp_server.cost_calculator import (
@ -2206,6 +2301,7 @@ def completion_cost(
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
prompt_characters=prompt_characters,
completion_characters=completion_characters,
cache_creation_input_tokens=cache_creation_input_tokens,
@ -2351,18 +2447,20 @@ def completion_cost(
raise e
def get_response_cost_from_hidden_params(
hidden_params: dict | BaseModel,
) -> float | None:
if isinstance(hidden_params, BaseModel):
_hidden_params_dict = hidden_params.model_dump()
else:
_hidden_params_dict = hidden_params
_ADDITIONAL_HEADERS: Final = TypeAdapter(Mapping[str, object])
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def get_response_cost_from_hidden_params(
hidden_params: Mapping[str, object] | BaseModel,
) -> float | None:
params: Final[Mapping[str, object]] = (
hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
)
additional_headers: Final = params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
headers: Final = _ADDITIONAL_HEADERS.validate_python(additional_headers)
return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
def _billable_header_cost(value: object) -> float | None:
@ -2438,6 +2536,7 @@ def response_cost_calculator(
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
) -> float:
"""
Returns
@ -2494,6 +2593,7 @@ def response_cost_calculator(
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
@ -3088,8 +3188,12 @@ def _batch_rate(
"input_cost_per_video_token_batches",
],
fallback: float,
*,
prompt_tokens: int = 0,
) -> float:
rate: Final = model_info.get(key)
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
rate: Final = model_info.get(key) if context_rate is None else context_rate
return fallback if rate is None else rate
@ -3239,7 +3343,8 @@ def _batch_prompt_cost(
)
return (
text_tokens * input_rate
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ audio_tokens
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
@ -3401,8 +3506,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
def create_logging_realtime_object(
usage: Usage, results: OpenAIRealtimeStreamList
) -> LiteLLMRealtimeStreamLoggingObject:
translated_input_seconds, _ = _translation_session_seconds(results)
return LiteLLMRealtimeStreamLoggingObject(
usage=usage,
usage=usage.model_copy(update={"seconds": translated_input_seconds})
if translated_input_seconds > 0
else usage,
results=results,
)
@ -3487,6 +3595,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
class _LiveResponsePayload(BaseModel):
@ -3709,6 +3818,7 @@ def handle_realtime_stream_cost_calculation(
litellm_logging_obj: LitellmLoggingObject | None = None,
custom_pricing_model: str | None = None,
base_pricing_model: str | None = None,
deployment_models: Sequence[str | None] = (),
) -> float:
"""
Handles the cost calculation for realtime stream responses.
@ -3753,6 +3863,13 @@ def handle_realtime_stream_cost_calculation(
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
else 0.0
)
translation_cost: Final = handle_realtime_translation_cost_calculation(
results=results,
custom_llm_provider=custom_llm_provider,
litellm_model_name=litellm_model_name,
deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
session_model=received_model,
)
live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
live_session_started: Final = any(result["type"] == "session.started" for result in results)
live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@ -3809,12 +3926,19 @@ def handle_realtime_stream_cost_calculation(
input_cost: Final = input_cost_per_token + live_response_input_cost
output_cost: Final = output_cost_per_token + live_response_output_cost
total_cost: Final = (
input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
input_cost
+ output_cost
+ transcription_cost
+ translation_cost
+ live_voice_cost
+ audio_duration_cost
+ text_message_cost
)
additional_costs: Final = tuple(
(cost_name, cost)
for cost_name, cost in (
("transcription_cost", transcription_cost),
("translation_cost", translation_cost),
("live_voice_cost", live_voice_cost),
("audio_duration_cost", audio_duration_cost),
("text_message_cost", text_message_cost),
@ -3835,6 +3959,55 @@ def handle_realtime_stream_cost_calculation(
return total_cost
def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
usage_events: Final = tuple(
usage
for result in results
if (event := as_str_mapping(result)) is not None
and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
and (usage := as_str_mapping(event.get("usage"))) is not None
)
input_seconds: Final = sum(
seconds
for usage in usage_events
if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
)
output_seconds: Final = sum(
output_duration
for usage in usage_events
if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
)
return float(input_seconds), float(output_seconds)
def _translation_per_second_rates(
models: Sequence[str | None], custom_llm_provider: str
) -> tuple[float | None, float | None]:
model_infos: Final = tuple(
_lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
)
return (
_first_declared_transcription_rate(("input_cost_per_second",), model_infos),
_first_declared_transcription_rate(("output_cost_per_second",), model_infos),
)
def handle_realtime_translation_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
litellm_model_name: str,
deployment_pricing_models: Sequence[str | None] = (),
session_model: str | None = None,
) -> float:
input_seconds, output_seconds = _translation_session_seconds(results)
if input_seconds <= 0 and output_seconds <= 0:
return 0.0
input_rate, output_rate = _translation_per_second_rates(
(*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
)
return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
def handle_realtime_transcription_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
@ -3850,57 +4023,77 @@ def handle_realtime_transcription_cost_calculation(
- {"type": "duration", "seconds": <float>} → priced via input_cost_per_second
- {"type": "tokens", "input_tokens": ...} → priced via input/audio token cost
"""
completed_events: Final = [
cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
]
if not completed_events:
configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
clips: Final = tuple(
(as_str_mapping(result.get("usage")) or {}, configured)
for result, configured in zip(results, configured_models, strict=True)
if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
)
if not clips:
return 0.0
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
model_infos: Final = MappingProxyType(
{model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
)
override_info: Final = (
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
)
return sum(
(_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
0.0,
)
total_cost = 0.0
for event in completed_events:
usage = event.get("usage") or {}
total_cost += _transcription_usage_cost(usage, model_info, override_info)
return total_cost
_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
{"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
)
def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
value: Final = None if mapping is None else mapping.get(key)
return value if isinstance(value, str) and value else None
def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
def _session_transcription_model(result: Mapping[str, object]) -> str | None:
session: Final = _session_of(result)
return None if session is None else _configured_transcription_model(session)
def _newer_model(current: str | None, newer: str | None) -> str | None:
return newer or current
def _get_transcription_model_name_from_results(
results: OpenAIRealtimeStreamList,
) -> str | None:
"""Resolve the ASR model from a transcription_session.* / session.* event."""
for result in results:
if result.get("type") in (
"transcription_session.created",
"transcription_session.updated",
"session.created",
"session.updated",
):
session = cast(dict, result).get("session", {}) or {}
transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
"transcription", {}
) or session.get("input_audio_transcription", {})
model = (transcription or {}).get("model") or session.get("model")
if model:
return model
return None
return next(
(model for result in results if (model := _str_field(_session_of(result), "model"))),
None,
)
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "output_cost_per_second":
return info.get("output_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "output_cost_per_audio_token":
return info.get("output_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
@ -3909,7 +4102,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3927,15 +4120,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
)
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)
def _first_declared_transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
) -> float | None:
rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
return next((rate for rate in rates if rate is not None), None)
def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
return _first_declared_transcription_rate(keys, infos) or 0.0
def _transcription_usage_cost(
usage: dict,
usage: Mapping[str, object],
model_info: ModelInfo | None,
override_info: ModelInfo | None = None,
) -> float:
@ -3944,13 +4141,13 @@ def _transcription_usage_cost(
usage_type: Final = usage.get("type")
if usage_type == "duration":
seconds: Final = usage.get("seconds") or 0.0
seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
if usage_type == "tokens":
input_token_details: Final = usage.get("input_token_details") or {}
audio_tokens: Final = input_token_details.get("audio_tokens") or 0
text_tokens: Final = input_token_details.get("text_tokens") or 0
output_tokens: Final = usage.get("output_tokens") or 0
input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
audio_cost: Final = float(audio_tokens) * _transcription_rate(
("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
)

10
litellm/decisions/call.py
View file

@ -13,6 +13,7 @@ from typing_extensions import assert_never
import litellm
from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -555,7 +556,7 @@ def prepare_call(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or call_api_base, dynamic_api_key or api_key
)
except ValueError as error:
except (ValueError, httpx.InvalidURL) as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
if resolved_api_base is None:
raise litellm.BadRequestError(
@ -613,6 +614,9 @@ def prepare_call(
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
"input_cost_per_query": (
kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
),
},
custom_llm_provider=provider,
)
@ -787,7 +791,9 @@ def _formatted_response(
case DecisionsRequestBody():
return ir_to_systemone_response(caller_response, caller_request)
case OpenAIDecisionRequestBody():
return ir_to_openai_response(caller_response, caller_request, call.requested_model)
return ir_to_openai_response(
caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
)
case _:
assert_never(call.request)

3
litellm/exceptions.py
View file

@ -867,6 +867,7 @@ class BadGatewayError(openai.APIStatusError):
class InternalServerError(openai.InternalServerError):
response: httpx.Response # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX response
request: httpx.Request # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX request
provider_status_code: int | None
def __init__(
self,
@ -878,8 +879,10 @@ class InternalServerError(openai.InternalServerError):
max_retries: int | None = None,
num_retries: int | None = None,
body: object | None = None,
provider_status_code: int | None = None,
):
self.status_code = 500
self.provider_status_code = provider_status_code
self.message = f"litellm.InternalServerError: {message}"
self.llm_provider = llm_provider
self.model = model

22
litellm/images/main.py
View file

@ -28,8 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.custom_llm import CustomLLM
from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@ -608,10 +607,7 @@ def image_generation(
)
elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider
# Get the Custom Handler
custom_handler: CustomLLM | None = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
if custom_handler is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@ -915,10 +911,7 @@ def image_edit(
# Check for custom provider
if custom_llm_provider in litellm._custom_providers:
custom_handler: CustomLLM | None = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider)
if custom_handler is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@ -995,10 +988,15 @@ def image_edit(
or custom_llm_provider == "azure"
or custom_llm_provider in litellm.openai_compatible_providers
):
dropped_form_params: Final = header_auth_params(custom_llm_provider)
image_edit_request_params.update(
flatten_form_field_values(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
{key: value for key, value in non_default_params.items() if key not in dropped_form_params},
(
{key: value for key, value in extra_body.items() if key not in dropped_form_params}
if isinstance(extra_body, dict)
else None
),
)
)

461
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -9,24 +9,29 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
"""
import bisect
import copy
import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_as_cache_control,
with_prompt_cache_breakpoint,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
@ -73,7 +78,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@ -85,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
CostMap: TypeAlias = Mapping[str, Mapping[str, object]]
def routed_model_info(value: object) -> Mapping[str, object] | None:
try:
return _MODEL_INFO_ADAPTER.validate_python(value)
except ValidationError:
return None
def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap:
import litellm
if model_cost is not None:
return model_cost
return cast(CostMap, litellm.model_cost) # cast-ok: the live map holds catalog entries
def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None:
declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
return declared if isinstance(declared, bool) else None
def routed_cost_map() -> CostMap:
from litellm.utils import catalog_with_runtime_registrations
return catalog_with_runtime_registrations()
def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None:
return None if model_info is None else routed_cost_map()
def _validated_object_mapping(value: object) -> dict[object, object] | None:
@ -295,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
if not isinstance(value, (list, tuple)):
return ()
@ -314,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non
return provider
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
def supports_openai_prompt_cache_breakpoint(
model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True
def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
def openai_prompt_cache_breakpoint_support(
model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool | None:
declared: Final = declared_prompt_cache_breakpoint(model_info)
if declared is not None:
return declared
model_cost: Final = _catalog_unless_unrouted(model_info)
hosted_flag: Final = (
None
if custom_llm_provider is None
else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model)
else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost)
)
if hosted_flag is not None:
return hosted_flag
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost)
if model_map_flag is not None:
return model_map_flag
fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
@ -335,6 +386,35 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
return _named_model_breakpoint_support(model)
class _DeclaredBaseModels(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
base_model: object = None
model_info: object = None
def names(self) -> tuple[str, ...]:
model_info: Final = _validated_object_mapping(self.model_info) or {}
candidates: Final = (self.base_model, model_info.get("base_model"))
return tuple(name for name in candidates if isinstance(name, str) and name)
def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None:
return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info)
def deployment_prompt_cache_breakpoint_support(
model: str, custom_llm_provider: str, litellm_params: object
) -> bool | None:
"""``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
model_info: Final = routed_model_info(declared.model_info)
supports: Final = (
openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info)
for name in dict.fromkeys((*declared.names(), model))
)
return next((support for support in supports if support is not None), None)
def _named_model_breakpoint_support(model: str) -> bool | None:
bare_model: Final = model.rsplit("/", 1)[-1].lower()
if _O_SERIES_PATTERN.match(bare_model) is not None:
@ -342,15 +422,14 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
if version_match is None:
return None
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
import litellm
entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None:
cost_map: Final = _live_cost_map_unless(model_cost)
entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1]))
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping))
return next((flag is True for flag in flags if flag is not None), None)
@ -366,7 +445,7 @@ def _hosted_openai_dialect_flag(
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
model_cost: CostMap | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -383,13 +462,7 @@ def _hosted_openai_dialect_flag(
row of the same name, so an exact entry only speaks for a deployment when it is
keyed for that deployment's provider.
"""
import litellm
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
cost_map: Final = _live_cost_map_unless(model_cost)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
@ -399,10 +472,16 @@ def _hosted_openai_dialect_flag(
return None
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider))
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]:
from litellm.utils import get_potential_model_names
names: Final = get_potential_model_names(model, provider)
candidates: Final = (
return (
names["combined_model_name"],
names["region_free_combined_model_name"],
names["split_model"],
@ -410,9 +489,60 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool:
entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider)))
return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries)
_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse"))
_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle"))
def _declared_hosted_flag(
declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object
) -> bool | None:
"""
A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the
request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes
a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep
the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass
the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost
map, or the deployment's own ``litellm_provider``, lists under that provider.
"""
if provider is None or provider == "openai":
return None
return (
declared
and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT
and (
provider in _PROVIDERS_HOSTING_OPENAIS_API
or deployment_provider == provider
or _cost_map_lists_model_for_provider(model, provider, model_cost)
)
and _sends_openai_shaped_chat(model, provider)
)
@lru_cache(maxsize=512)
def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.openai import OpenAIConfig
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
try:
served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
chat_config: Final = ProviderConfigManager.get_provider_chat_config(
model=served_model, provider=LlmProviders(provider)
)
except (BadRequestError, ValueError):
return False
return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig))
def targets_openai_api(api_base: object) -> bool:
@ -432,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
def _carries_cache_control(block: object) -> bool:
return _attribute_or_key(block, "cache_control") is not None
def _attribute_or_key(value: object, key: str) -> object | None:
if hasattr(value, key):
return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@ -511,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
model, custom_llm_provider
)
def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
return provider if isinstance(provider, str) else None
@ -554,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
def _carries_prompt_cache_breakpoint(block: object) -> bool:
return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") != "assistant"
def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and any(
_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
)
def _part_without_prompt_cache_breakpoint(part: object) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return {
key: value
for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
if key != "prompt_cache_breakpoint"
}
def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
if not _tool_result_carries_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
return cast( # cast-ok: the same tool message with its part marks removed
AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
)
def _created_cache_control(original: object, translated: object) -> bool:
return (
_attribute_or_key(original, "cache_control") is None
and _attribute_or_key(translated, "cache_control") is not None
)
def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
if translated is original:
return ()
original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
def _part_without_cache_control(part: object) -> object:
return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
def _message_dropping_created_cache_controls_between(
message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
) -> AllMessageValues:
if not any(created):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
return cast( # cast-ok: the same message with its middle translated marks removed
AllMessageValues,
{
**message,
"content": [
_part_without_cache_control(part)
if is_created and first_dropped <= ordinal < first_kept_after
else part
for part, is_created, ordinal in zip(parts, created, ordinals)
],
},
)
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
@ -643,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if not injection_points:
return model, messages, non_default_params
# Create a deep copy of messages to avoid modifying the original list
processed_messages = copy.deepcopy(messages)
message_points: Final = tuple(
cast(CacheControlMessageInjectionPoint, point)
for point in injection_points
@ -653,18 +859,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message")
custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0])
request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict
custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
model_info: Final = routed_model_info(request_params.get("model_info"))
openai_dialect: Final = (
stamped_dialect
if isinstance(stamped_dialect, bool)
else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model,
custom_llm_provider,
non_default_params.get("api_base") or non_default_params.get("base_url"),
non_default_params.get("prompt_cache_options"),
request_params.get("api_base") or request_params.get("base_url"),
request_params.get("prompt_cache_options"),
model_info,
)
)
processed_messages = copy.deepcopy(
[without_prompt_cache_breakpoints(message) for message in messages]
if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
else messages
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
@ -733,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object = None,
prompt_cache_options: object = None,
model_info: Mapping[str, object] | None = None,
) -> bool:
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
declared: Final = declared_prompt_cache_breakpoint(model_info)
hosted_flag: Final = (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base),
_catalog_unless_unrouted(model_info),
)
if declared is None
else _declared_hosted_flag(
declared,
model,
custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base),
routed_cost_map(),
None if model_info is None else model_info.get("litellm_provider"),
)
)
if hosted_flag is not None:
return hosted_flag
if not supports_openai_prompt_cache_breakpoint(model):
return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@ -805,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
@staticmethod
def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
return tuple(
tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
)
@staticmethod
def count_chat_external_cache_breakpoints(optional_params: object) -> int:
params: Final = _validated_object_mapping(optional_params) or {}
return AnthropicCacheControlHook.count_external_cache_breakpoints(
_as_object_list(params.get("tools")), params.get("cache_control")
)
@staticmethod
def count_external_cache_breakpoints_on_messages_route(
tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@ -960,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return message_count + content_count + tool_call_count
@staticmethod
def count_converse_cache_points(message: AllMessageValues) -> int:
content: Final = _as_object_list(_attribute_or_key(message, "content"))
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
if message.get("role") == "tool":
return int(_carries_cache_control(message) or marked_parts > 0)
return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
@staticmethod
def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
"""Return True if the message already carries any cache_control."""
@ -1018,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return renamed
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
@staticmethod
def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
"""Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
if not _may_carry_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content"))
if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
return message
return cast( # cast-ok: the same message with its part marks renamed
AllMessageValues,
{**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
)
@staticmethod
def prompt_cache_breakpoints_dropped_from_tool_results(
messages: Sequence[AllMessageValues], drop_params: bool, model: str
) -> tuple[AllMessageValues, ...]:
if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
return tuple(messages)
if not drop_params:
raise UnsupportedParamsError(
message=(
f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
@staticmethod
def prompt_cache_breakpoints_as_capped_cache_control(
messages: Sequence[AllMessageValues],
drop_params: bool,
sent_breakpoints: Callable[[AllMessageValues], int],
external_breakpoints: Callable[[], int],
) -> tuple[AllMessageValues, ...]:
"""``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
Under ``drop_params`` the first and last translated marks fill the slots the client's own
``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
turns stay cached, and without it a request over the cap is rejected."""
translated: Final = tuple(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
for message in messages
)
if all(after is before for before, after in zip(messages, translated)):
return translated
created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
created_marks: Final = sum(sum(flags) for flags in created)
if created_marks == 0:
return translated
external: Final = external_breakpoints()
breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
if excess <= 0:
return translated
if not drop_params:
raise UnsupportedParamsError(
message=(
f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
),
status_code=400,
)
verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
first_dropped: Final = (kept_marks + 1) // 2
first_kept_after: Final = created_marks - kept_marks // 2
return tuple(
_message_dropping_created_cache_controls_between(
message, flags, before, first_dropped, first_kept_after
)
for message, flags, before in zip(translated, created, marks_before)
)
def over_cap(kept_marks: int) -> bool:
return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
return capped_request(max(0, first_over_cap - 1))
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
if not _may_carry_prompt_cache_breakpoint(message):
@ -1157,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
model_info: Mapping[str, object] | None = None,
) -> Sequence[Mapping[str, object]]:
"""Carry onto the points what the prompt-management hook never receives.
@ -1168,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
across requests.
"""
with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
points, model, custom_llm_provider, api_base, prompt_cache_options
points, model, custom_llm_provider, api_base, prompt_cache_options, model_info
)
stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect)
if external_breakpoints == 0 and not stamped_earlier:
@ -1182,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
model_info: Mapping[str, object] | None = None,
) -> Sequence[Mapping[str, object]]:
if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(
model, custom_llm_provider
):
return points
return AnthropicCacheControlHook._stamped(
points,
"_litellm_openai_dialect",
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, prompt_cache_options
model, custom_llm_provider, api_base, prompt_cache_options, model_info
),
)
@ -1248,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
cache_control: object = None,
request_kwargs: object = None,
on_messages_route: bool = False,
model_info: Mapping[str, object] | None = None,
) -> list[CacheControlInjectionPoint]:
"""Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
@ -1265,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
return []
if not supports_anthropic_cache_control(model, custom_llm_provider):
if not supports_anthropic_cache_control(model, custom_llm_provider, model_info):
return []
if AnthropicCacheControlHook._request_has_cache_control(
@ -1343,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
tools: list | None = None,
enable_prompt_caching: bool | None = None,
api_base: object = None,
model_info: Mapping[str, object] | None = None,
) -> None:
"""For /chat/completions: resolve the injection points the request should carry.
@ -1357,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
if configured:
tools_keeping_marks: Final = tuple(
tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
)
tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
configured,
AnthropicCacheControlHook.count_external_cache_breakpoints(
@ -1369,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider,
api_base,
non_default_params.get("prompt_cache_options"),
model_info,
)
non_default_params["cache_control_injection_points"] = (
AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider)
@ -1385,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
enable_prompt_caching=enable_prompt_caching,
cache_control=non_default_params.get("cache_control"),
request_kwargs=non_default_params,
model_info=model_info,
)
if points:
non_default_params["cache_control_injection_points"] = points
@ -1503,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
cache_control=cache_control,
request_kwargs=kwargs,
on_messages_route=True,
model_info=routed_model_info(kwargs.get("model_info")),
)
if model is not None
else ()
@ -1511,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return messages, system
openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
model,
custom_llm_provider,
api_base,
kwargs.get("prompt_cache_options"),
routed_model_info(kwargs.get("model_info")),
)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
@ -1645,9 +1982,17 @@ def messages_within_cache_control_cap(
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
return messages_within_reserved_cache_control_cap(
messages,
AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
),
)
def messages_within_reserved_cache_control_cap(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:

50
litellm/integrations/azure_storage/azure_storage.py
View file

@ -1,7 +1,8 @@
import asyncio
import os
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from functools import cache
from typing import Final
@ -15,6 +16,7 @@ from litellm.constants import (
AZURE_STORAGE_MSFT_VERSION,
)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
from litellm.llms.custom_httpx.http_handler import (
@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
return f"{name.translate(_ADLS_SAFE_NAME)}.json"
@dataclass(frozen=True, slots=True)
class AzureStorageLogQueueItem:
payload: StandardLoggingPayload
file_name: str
def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
payload_id: Final = payload.get("id")
return AzureStorageLogQueueItem(
payload=payload,
file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
)
@cache
def _cached_credential_chain_token_provider() -> Callable[[], str]:
return get_azure_ad_token_provider(
@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
self.log_queue: list[StandardLoggingPayload] = []
self.log_queue: list[AzureStorageLogQueueItem] = []
super().__init__(**kwargs, flush_lock=self.flush_lock)
except Exception as e:
verbose_logger.exception(
@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
def azure_storage_blob_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log success events to Azure Blob Storage
@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log failure events to Azure Blob Storage
@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
len(self.log_queue),
)
for payload in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=payload)
for item in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
async def async_upload_payload_to_azure_blob_storage(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Blob Storage using a 3-step process:
1. Create file resource
@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
try:
if self.azure_storage_account_key:
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
else:
# Get a valid token instead of always requesting a new one
await self.set_valid_azure_ad_token()
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
payload_bytes: Final = json_payload.encode("utf-8")
filename: Final = adls_safe_file_name(payload.get("id"))
filename: Final = file_name or adls_safe_file_name(payload.get("id"))
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
# Execute the 3-step upload process
@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
return self._service_client
async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
async def upload_to_azure_data_lake_with_azure_account_key(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Data Lake using the Azure SDK
@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
try:
today: Final = datetime.now().strftime("%Y-%m-%d")
file_name: Final = adls_safe_file_name(payload.get("id"))
resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
await self._upload_bytes_with_account_key(
file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
except Exception as e:
verbose_logger.exception("Error occurred: %s", e)

7
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@ -134,7 +135,7 @@ def spend_log_row_from_payload(
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
trace_id, span_id = _find_traceparent(metadata, kwargs)
request_id = str(payload.get("id") or "")
response_id: Final = str(payload.get("id") or "")
redact = litellm.turn_off_message_logging is True
track_end_user: Final = not litellm.disable_end_user_cost_tracking
completion_start_ms = _to_ms(payload.get("completionStartTime"))
@ -157,8 +158,8 @@ def spend_log_row_from_payload(
)
)
return SpendLogRecord(
request_id=request_id,
response_id=strip_cache_hit_suffix(request_id),
request_id=logged_event_key(kwargs, response_id),
response_id=strip_cache_hit_suffix(response_id),
provider_request_id=(
get_provider_request_id(kwargs.get("response_headers"))
or get_provider_request_id(hidden_params.get("additional_headers"))

128
litellm/integrations/code_interpreter_interception/handler.py
View file

@ -10,6 +10,7 @@ import json
import time
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable
from pydantic import ValidationError
@ -73,7 +74,7 @@ class CodeInterpreterLogOutput(TypedDict):
class CodeInterpreterCall(TypedDict):
id: str
type: Literal["code_interpreter_call"]
status: Literal["completed"]
status: Literal["completed", "failed"]
code: str
container_id: str | None
outputs: list[CodeInterpreterLogOutput]
@ -141,6 +142,39 @@ class _SupportsOutput(Protocol):
_CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None]
@dataclass(frozen=True, slots=True)
class _ReadySandbox:
container: ContainerHandle
params: SandboxToolParams | None
@dataclass(frozen=True, slots=True)
class _SandboxUnavailable:
output: str
_Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable
@dataclass(frozen=True, slots=True)
class _ToolRun:
output: str
status: Literal["completed", "failed"]
_SANDBOX_UNAVAILABLE_OUTPUT: Final = "The code did not run because the code execution sandbox is unavailable."
_SANDBOX_LOST_RESULT_OUTPUT: Final = (
"The code execution sandbox failed while running the code, so the code may have run but its result is lost."
)
def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNAVAILABLE_OUTPUT) -> _SandboxUnavailable:
verbose_logger.warning(
"CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error
)
return _SandboxUnavailable(output=output)
def _output_item_type(item: object) -> object:
if isinstance(item, dict):
item_mapping: Final[dict[str, object]] = item
@ -189,6 +223,14 @@ def _extract_identity(kwargs: Mapping[str, object]) -> str:
return identity if isinstance(identity, str) else ""
def _bridged_from_responses(kwargs: Mapping[str, object], call_type: CallTypes | None) -> bool:
route: Final = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
return call_type in (CallTypes.completion, CallTypes.acompletion) and route in (
CallTypes.responses,
CallTypes.aresponses,
)
def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None:
if sandbox_tool_name is None:
return None
@ -261,7 +303,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
async def async_pre_call_deployment_hook(
self, kwargs: dict[str, object], call_type: CallTypes | None
) -> dict | None:
if not kwargs.get("_agentic_loop_depth"):
if not kwargs.get("_agentic_loop_depth") and not _bridged_from_responses(kwargs, call_type):
kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
kwargs.pop(_SANDBOX_KEY, None)
self._strip_interception_metadata(kwargs)
@ -451,16 +493,16 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = self._container_id(container)
container_id: Final = self._sandbox_container_id(sandbox)
input_list: Final = self._normalize_messages(messages)
code_interpreter_calls: Final[list[CodeInterpreterCall]] = []
for tool_call in tool_calls:
arguments = tool_call.get("arguments", "")
code = self._parse_code(arguments)
stdout = await self._run_tool_call(container=container, params=params, arguments=arguments)
tool_run = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
input_list.append(
{
"type": "function_call",
@ -473,22 +515,23 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
{
"type": "function_call_output",
"call_id": tool_call.get("call_id"),
"output": stdout,
"output": tool_run.output,
}
)
code_interpreter_calls.append(
{
"id": f"ci_{uuid.uuid4().hex}",
"type": "code_interpreter_call",
"status": "completed",
"status": tool_run.status,
"code": code,
"container_id": container_id,
"outputs": ([{"type": "logs", "logs": stdout}] if stdout else []),
"outputs": ([{"type": "logs", "logs": tool_run.output}] if tool_run.output else []),
}
)
except Exception:
await self._delete_container_for_cache_key(sandbox_key)
raise
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
optional_params: Final = anthropic_messages_optional_request_params
request_patch: Final = AgenticLoopRequestPatch(
@ -526,14 +569,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = self._container_id(container)
container_id: Final = self._sandbox_container_id(sandbox)
tool_results: Final = [
await self._build_chat_completion_tool_result(
container=container,
params=params,
sandbox=sandbox,
tool_call=tool_call,
container_id=container_id,
)
@ -544,6 +586,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
raise
tool_messages: Final = [result[0] for result in tool_results]
code_interpreter_calls: Final = [result[1] for result in tool_results]
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
request_patch: Final = AgenticLoopRequestPatch(
model=model,
@ -569,9 +612,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
)
@staticmethod
def _container_id(container: ContainerHandle) -> str | None:
container_id: Final[object] = getattr(container, "id", None)
return container_id if isinstance(container_id, str) else None
def _sandbox_container_id(sandbox: _Sandbox) -> str | None:
match sandbox:
case _ReadySandbox(container=container):
container_id: Final[object] = getattr(container, "id", None)
return container_id if isinstance(container_id, str) else None
case _SandboxUnavailable():
return None
@staticmethod
def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]:
@ -588,28 +635,27 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
async def _build_chat_completion_tool_result(
self,
container: ContainerHandle,
params: SandboxToolParams | None,
sandbox: _Sandbox,
tool_call: CodeExecutionToolCall,
container_id: str | None,
) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
arguments: Final = tool_call.get("arguments", "")
code: Final = self._parse_code(arguments)
stdout: Final = await self._run_tool_call(container=container, params=params, arguments=arguments)
tool_run: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex
return (
{
"role": "tool",
"tool_call_id": tool_call_id,
"content": stdout,
"content": tool_run.output,
},
{
"id": f"ci_{uuid.uuid4().hex}",
"type": "code_interpreter_call",
"status": "completed",
"status": tool_run.status,
"code": code,
"container_id": container_id,
"outputs": [{"type": "logs", "logs": stdout}] if stdout else [],
"outputs": [{"type": "logs", "logs": tool_run.output}] if tool_run.output else [],
},
)
@ -694,19 +740,55 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
except (json.JSONDecodeError, TypeError, AttributeError):
return ""
async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str:
async def _evict_sandbox_if_a_run_failed(
self, sandbox_key: str | None, sandbox: _Sandbox, calls: Sequence[CodeInterpreterCall]
) -> None:
if not sandbox_key or not isinstance(sandbox, _ReadySandbox):
return
if not any(call["status"] == "failed" for call in calls):
return
cached: Final = self._container_cache.get(sandbox_key)
if cached is None or cached[0] is not sandbox.container:
return
self._container_cache.pop(sandbox_key)
await self._delete_container(container=cached[0], params=cached[1])
async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun:
match sandbox:
case _SandboxUnavailable(output=output):
return _ToolRun(output=output, status="failed")
case _ReadySandbox(container=container, params=params):
match await self._run_tool_call(container=container, params=params, arguments=arguments):
case _SandboxUnavailable(output=output):
return _ToolRun(output=output, status="failed")
case str() as output:
return _ToolRun(output=output, status="completed")
async def _run_tool_call(
self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str
) -> str | _SandboxUnavailable:
try:
code: Final = json.loads(arguments).get("code", "") if arguments else ""
except (json.JSONDecodeError, TypeError):
return "[invalid tool arguments: could not parse code]"
result: Final = await self._run_code(container=container, params=params, code=code)
try:
result: Final = await self._run_code(container=container, params=params, code=code)
except Exception as e:
return _sandbox_unavailable("running the code", e, _SANDBOX_LOST_RESULT_OUTPUT)
if getattr(result, "error", None):
error: Final = result.error
message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error)
return f"[execution error] {message}"
return getattr(result, "stdout", "") or ""
async def _acquire_sandbox(self, cache_key: str | None, identity: str | None) -> _Sandbox:
try:
container, params = await self._get_or_create_container(cache_key=cache_key, identity=identity)
except Exception as e:
return _sandbox_unavailable("creating the sandbox", e)
return _ReadySandbox(container=container, params=params)
async def _get_or_create_container(
self,
cache_key: str | None,

13
litellm/integrations/custom_guardrail.py
View file

@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
enrich_http_exception_with_guardrail_context(result, self)
raise result
def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
return (
isinstance(request_data.get("guardrails"), list)
and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
)
async def async_post_call_success_deployment_hook(
self,
request_data: dict,
@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
"""
Allow modifying / reviewing the response just after it's received from the deployment.
"""
# should run guardrail
litellm_guardrails: Final = request_data.get("guardrails")
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
return None
if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
if not self.runs_post_call_deployment_hook(request_data):
return None
target: Final = self._deployment_hook_target()

5
litellm/integrations/deepeval/deepeval.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
to_zod_compatible_iso,
validate_environment,
)
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
# This file includes the custom callbacks for LiteLLM Proxy
@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
)
else:
output = str(standard_logging_object.get("error_string", ""))
logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
return BaseApiSpan(
uuid=standard_logging_object.get("id", uuid.uuid4()),
uuid=span_id,
name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
type=SpanApiType.LLM,

18
litellm/integrations/dynamodb.py
View file

@ -9,6 +9,8 @@ from typing import Final, Protocol
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
class _DynamoTable(Protocol):
@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
class DyanmoDBLogger:
# Class variables or attributes
def __init__(self):
def __init__(self, dynamodb: _DynamoResource | None = None):
# Instance variables
import boto3
self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
"dynamodb", region_name=os.environ["AWS_REGION_NAME"]
)
if litellm.dynamodb_table_name is None:
raise ValueError(
"LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@ -45,23 +49,23 @@ class DyanmoDBLogger:
_async_log_event = async_log_event
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
try:
print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
# construct payload to send to DynamoDB
# follows the same params as langfuse.py
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
messages: Final = kwargs.get("messages")
optional_params: Final = kwargs.get("optional_params", {})
call_type: Final = kwargs.get("call_type", "litellm.completion")
usage: Final = response_obj["usage"]
id: Final = response_obj.get("id", str(uuid.uuid4()))
response_id: Final = response_obj.get("id", str(uuid.uuid4()))
# Build the initial payload
payload: Final[dict[str, object]] = {
"id": id,
"id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
"call_type": call_type,
"startTime": start_time,
"endTime": end_time,

37
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
else:
await self._send_individual_logs(items_to_process)
def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
) -> str:
"""
Get the object name to use for the current payload
"""
current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
gcs_log_id: Final = metadata.get("gcs_log_id")
safe_log_id: Final = sanitize_cloud_object_component(
gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
)
if safe_log_id:
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
object_name = self._generate_failure_object_name(
request_date_str=current_date,
)
else:
object_name = self._generate_success_object_name(
request_date_str=current_date,
response_id=response_obj.get("id", ""),
)
# used for testing
_litellm_params: Final = kwargs.get("litellm_params", None) or {}
_metadata: Final = _litellm_params.get("metadata", None) or {}
if "gcs_log_id" in _metadata:
safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
if safe_log_id:
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
return object_name
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
response_id=logged_event_key(kwargs, response_id),
)
async def get_request_response_payload(
self,

11
litellm/integrations/openmeter.py
View file

@ -3,12 +3,15 @@
import json
import os
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
if len(missing_keys) > 0:
raise Exception(f"Missing keys={missing_keys} in environment.")
def _common_logic(self, kwargs: dict, response_obj):
def _common_logic(self, kwargs: Mapping[str, object], response_obj):
call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
dt: Final = get_utc_datetime().isoformat()
cost: Final = kwargs.get("response_cost", None)
@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
# If no user provided directly, try to get it from token user_id
if user_param is None:
# Check if user_id is available from the API key metadata
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {})
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
if user_api_key_user_id is not None:
@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
return {
"specversion": "1.0",
"type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
"id": call_id,
"id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
"time": dt,
"subject": subject,
"source": "litellm-proxy",

12
litellm/integrations/prometheus.py
View file

@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
async def async_post_call_failure_hook(
self,
request_data: dict,
request_data: dict[str, object],
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,
@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
try:
_tags: Final = StandardLoggingPayloadSetup.get_request_tags(
litellm_params=request_data,
proxy_server_request=request_data.get("proxy_server_request", {}),
proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
)
_metadata: Final = _logged_fields(request_data.get("metadata"))
model_id: Final = (
_logged_fields(_metadata.get("model_info")).get("id")
or _logged_fields(request_data.get("model_info")).get("id")
or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
)
_metadata: Final = request_data.get("metadata", {}) or {}
model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider: Final = self._extract_api_provider_from_request_data(
request_data

26
litellm/integrations/websearch_interception/handler.py
View file

@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
from typing_extensions import Never, ReadOnly
import litellm
from litellm._internal_context import billed_separately
from litellm._logging import verbose_logger
from litellm.anthropic_interface import messages as anthropic_messages
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
search_kwargs: Final = MappingProxyType(
{**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
)
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
with billed_separately():
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
# Format using transformation function
search_result_text: Final = WebSearchTransformation.format_search_response(result)

36
litellm/litellm_core_utils/agentic_followup_failure.py Normal file
View file

@ -0,0 +1,36 @@
from dataclasses import dataclass
from typing import Final, TypeAlias
from litellm.types.utils import Usage
ServedRound: TypeAlias = tuple[Usage, float]
_MARK_ATTRIBUTE: Final = "_litellm_agentic_followup_mark"
@dataclass(frozen=True, slots=True)
class _FollowUpMark:
served_rounds: tuple[ServedRound, ...]
def _mark_of(error: BaseException) -> _FollowUpMark | None:
mark: Final[object] = getattr(error, _MARK_ATTRIBUTE, None)
return mark if isinstance(mark, _FollowUpMark) else None
def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None:
setattr(error, _MARK_ATTRIBUTE, _FollowUpMark(served_rounds))
def is_agentic_followup_failure(error: BaseException) -> bool:
return _mark_of(error) is not None
def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]:
mark: Final = _mark_of(error)
return () if mark is None else mark.served_rounds
def carry_agentic_followup_mark(source: BaseException, mapped: BaseException) -> None:
if mapped is not source and is_agentic_followup_failure(source):
mark_agentic_followup_failure(mapped, rounds_served_before(source))

107
litellm/litellm_core_utils/agentic_loop_refusal.py Normal file
View file

@ -0,0 +1,107 @@
import asyncio
from collections.abc import Awaitable, Iterable, Mapping
from itertools import chain
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import (
ChatCompletionMessageCustomToolCall,
ChatCompletionMessageToolCall,
Choices,
ModelResponse,
)
def _strings(values: Iterable[object]) -> frozenset[str]:
return frozenset(value for value in values if isinstance(value, str))
def _claimed_calls(tool_calls: object) -> tuple[object, ...]:
wrapped: Final = message_field(tool_calls, "tool_calls")
return parts_of(tool_calls if wrapped is None else wrapped)
async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[object, ...]:
try:
should_run, tool_calls = await gate
except Exception as e:
verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed with %s", type(e).__name__)
return ()
return _claimed_calls(tool_calls) if should_run else ()
async def every_claimed_tool_call(
refused_tool_calls: object, other_gates: Iterable[Awaitable[tuple[bool, object]]]
) -> Mapping[str, tuple[object, ...]]:
claims: Final = await asyncio.gather(*(_calls_a_gate_claims(gate) for gate in other_gates))
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
def _call_ids(call: object) -> frozenset[str]:
return _strings((message_field(call, "id"), message_field(call, "call_id")))
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
calls: Final = _claimed_calls(tool_calls)
ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls))
names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call))
return ids, names
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names
def is_refused_item(
item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str]
) -> bool:
if message_field(item, "type") != item_type:
return False
return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names)
def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
kept: Final = [
item
for item in parts_of(message_field(response, "output"))
if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names)
]
return response.model_copy(update={"output": kept})
def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
name: Final = message_field(message_field(call, "function"), "name")
return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names)
def _at_position(
call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, position: int
) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall:
return call if message_field(call, "index") is None else call.model_copy(update={"index": position})
def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object:
if not isinstance(choice, Choices) or not choice.message.tool_calls:
return choice
surviving: Final = (
call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names)
)
kept: Final = [_at_position(call, position) for position, call in enumerate(surviving)]
finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason
return choice.model_copy(
update={
"message": choice.message.model_copy(update={"tool_calls": kept or None}),
"finish_reason": finish_reason,
}
)
def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
return response.model_copy(
update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]}
)

111
litellm/litellm_core_utils/allowed_tools_tool_choice.py Normal file
View file

@ -0,0 +1,111 @@
from collections.abc import Mapping
from typing import Final, Literal
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm.exceptions import BadRequestError
ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
'{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
)
class ChatAllowedTools(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
mode: Literal["auto", "required"]
tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
class ChatAllowedToolsToolChoice(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
type: Literal["allowed_tools"]
allowed_tools: ChatAllowedTools
_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
_SELECTOR: Final = TypeAdapter(dict[str, object])
class _ToolChoiceType(BaseModel):
model_config = ConfigDict(frozen=True)
type: str
_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType)
def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
try:
return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools"
except ValidationError:
return False
def raise_if_allowed_tools_tool_choice_unsupported(
tool_choice: object,
custom_llm_provider: str,
model: str,
) -> None:
"""Providers outside the set would silently send the request with no tool restriction at all, so the call stops
here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
return
raise BadRequestError(
message=(
"tool_choice type 'allowed_tools' is only supported for "
f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
"drop_params does not drop it, since that would let the model call tools the request excluded. "
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
),
model=model,
llm_provider=custom_llm_provider,
)
def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
try:
return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
except ValidationError:
return None
def _has_name(selector: Mapping[str, object]) -> bool:
name: Final = selector.get("name")
return isinstance(name, str) and name != ""
def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
tool_type: Final = tool.get("type")
if tool_type not in ("function", "custom"):
return tool
if tool_type not in tool:
return tool if _has_name(tool) else None
try:
nested: Final = _SELECTOR.validate_python(tool[tool_type])
except ValidationError:
return None
outer: Final = {key: value for key, value in tool.items() if key != tool_type}
if not _has_name(nested) or outer.keys() & nested.keys():
return None
return {**outer, **nested}
def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None:
"""The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or
carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser."""
flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools)
if any(tool is None for tool in flattened):
return None
return ToolChoiceAllowedParam(
type="allowed_tools",
mode=choice.allowed_tools.mode,
tools=[dict(tool) for tool in flattened if tool is not None],
)

341
litellm/litellm_core_utils/audio_utils/transcription_streaming.py Normal file
View file

@ -0,0 +1,341 @@
from __future__ import annotations
import datetime
import json
import traceback
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
from typing import Final, Protocol
import anyio
from openai import AsyncStream, Stream
from openai.types.audio import (
TranscriptionStreamEvent,
TranscriptionTextDeltaEvent,
TranscriptionTextDoneEvent,
)
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
)
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
_STREAM_FLAG: Final = TypeAdapter(bool)
def transcription_stream_flag(value: object) -> bool | None:
try:
return _STREAM_FLAG.validate_python(value)
except ValidationError:
return None
def transcription_stream_unset(value: object) -> bool:
return value is None or value == ""
def transcription_stream_requested(value: object) -> bool:
return transcription_stream_flag(value) is True
TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
model: Final = call_details.get("model")
custom_llm_provider: Final = call_details.get("custom_llm_provider")
extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
def map_exception(exception: Exception) -> Exception:
try:
mapped: Final[object] = exception_type(
model=model if isinstance(model, str) else None,
original_exception=exception,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
extra_kwargs=extra_kwargs,
)
except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception
return mapped_exception
return mapped if isinstance(mapped, Exception) else exception
return map_exception
def _guarded_transcription_event_payloads(
response: TranscriptionResponse,
) -> tuple[dict[str, object], dict[str, object]]:
text: Final = response.text or ""
usage: Final = response.usage
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
return (
{"type": "transcript.text.delta", "delta": text},
{
"type": "transcript.text.done",
"text": text,
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
},
)
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
return (
*(
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
for event in _guarded_transcription_event_payloads(response)
),
TRANSCRIPTION_STREAM_DONE_FRAME,
)
class GuardedTranscriptionReplay:
def __init__(self, response: TranscriptionResponse) -> None:
self.response: Final = response
self._hidden_params: dict[str, object] = response.hidden_params
def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
return self._events()
async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
delta, done = _guarded_transcription_event_payloads(self.response)
yield TranscriptionTextDeltaEvent.model_construct(**delta)
yield TranscriptionTextDoneEvent.model_construct(**done)
class TranscriptionStreamLogging(Protocol):
def success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
async def async_success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def handle_sync_success_callbacks_for_async_calls(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
*,
for_async_request: bool = False,
) -> None: ...
async def async_failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
class _TranscriptionEventCollector:
def __init__(self, duration: float | None) -> None:
self.duration = duration
self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event
self.done_event: TranscriptionTextDoneEvent | None = None
def add(self, event: TranscriptionStreamEvent) -> None:
if isinstance(event, TranscriptionTextDeltaEvent):
self.text_deltas.append(event.delta)
elif isinstance(event, TranscriptionTextDoneEvent):
self.done_event = event
def response(self) -> TranscriptionResponse:
done_event: Final = self.done_event
response: Final = TranscriptionResponse(
text=done_event.text if done_event is not None else "".join(self.text_deltas),
usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
languages=getattr(done_event, "languages", None) if done_event is not None else None,
)
if self.duration is not None:
response.set_audio_transcription_duration(self.duration)
return response
def _billed_usage(response: TranscriptionResponse) -> Usage:
usage: Final[object] = getattr(response, "usage", None)
token_usage: Final = (
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
if isinstance(usage, TranscriptionUsageTokensObject)
else None
)
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
def _record_billable_partial_spend(
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
) -> None:
partial_response: Final = collector.response()
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
if partial_cost <= 0:
return
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
def __init__(
self,
stream: Stream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
def _logging_iterator(
self, source_iterator: Iterator[TranscriptionStreamEvent]
) -> Iterator[TranscriptionStreamEvent]:
try:
for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
raise mapped_exception from exception
finally:
self._finalize()
def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
self._logging_obj.success_handler(
self._collector.response(),
self._start_time,
datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract
)
def close(self) -> None:
try:
super().close()
finally:
self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
def __init__(
self,
stream: AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
async def _logging_iterator(
self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
) -> AsyncIterator[TranscriptionStreamEvent]:
try:
async for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(
mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
)
await self._logging_obj.async_failure_handler(
mapped_exception,
traceback.format_exc(),
self._start_time,
end_time,
)
raise mapped_exception from exception
finally:
await self._finalize()
async def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
response: Final = self._collector.response()
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.handle_sync_success_callbacks_for_async_calls(
result=response,
start_time=self._start_time,
end_time=end_time,
)
with anyio.CancelScope(shield=True):
await self._logging_obj.async_success_handler(
result=response,
start_time=self._start_time,
end_time=end_time,
)
async def close(self) -> None:
try:
await super().close()
finally:
await self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
def wrap_transcription_stream(
stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
if isinstance(stream, AsyncStream):
return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

10
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -21,6 +21,16 @@ from litellm.types.files import (
from litellm.types.utils import FileTypes
def normalized_audio_duration_seconds(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
try:
seconds: Final = float(value)
except OverflowError:
return None
return seconds if math.isfinite(seconds) and seconds >= 0 else None
@dataclass
class ProcessedAudioFile:
"""

385
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -1,28 +1,35 @@
# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
import json
from collections.abc import Mapping
from collections.abc import Awaitable, Iterator, Mapping, Sequence
from itertools import chain
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
from pydantic import TypeAdapter, ValidationError
from litellm._internal_context import internal_sub_call
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopLimitError,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
converted_stream_requested,
is_interception_internal_key,
stashed_stream_options,
@ -30,6 +37,8 @@ from litellm.types.integrations.custom_logger import (
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
_DispatchResult: Final = TypeVar("_DispatchResult")
_FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
(
"acompletion",
@ -43,22 +52,41 @@ _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
)
async def bill_first_round_on_failure(
dispatch: Awaitable[_DispatchResult], *, logging_obj: object, response: object, stream: bool
) -> _DispatchResult:
try:
return await dispatch
except Exception as error:
if stream or not isinstance(logging_obj, LiteLLMLoggingObject):
mark_agentic_followup_failure(error, rounds_served_before(error))
raise
this_round: Final = logging_obj.served_round(response)
served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,)))
mark_agentic_followup_failure(error, served_rounds)
logging_obj.record_served_rounds_for_failure(served_rounds)
raise
def _gate_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_should_run_agentic_loop
func: Final = type(callback).async_should_run_agentic_loop
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return type(callback).async_should_run_agentic_loop is not CustomLogger.async_should_run_agentic_loop
def _build_plan_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_build_agentic_loop_plan
func: Final = type(callback).async_build_agentic_loop_plan
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return type(callback).async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
def _chat_build_plan_overridden(callback: CustomLogger) -> bool:
return (
type(callback).async_build_chat_completion_agentic_loop_plan
is not CustomLogger.async_build_chat_completion_agentic_loop_plan
)
def _post_hook_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_post_agentic_loop_response_hook
func: Final = type(callback).async_post_agentic_loop_response_hook
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return (
type(callback).async_post_agentic_loop_response_hook is not CustomLogger.async_post_agentic_loop_response_hook
)
_STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
@ -75,14 +103,14 @@ def _coerce_int(value: object, default: int) -> int:
return int(value) if isinstance(value, (int, str)) else default
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, tuple[str, ...]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
fingerprints: Final = tuple(str(fp) for fp in raw_fingerprints) if isinstance(raw_fingerprints, list) else ()
return depth, max_loops, fingerprints
@ -95,16 +123,16 @@ def _fingerprint_tools(tool_calls: object) -> str:
def _check_agentic_loop_safety(
tool_calls: object,
fingerprints: list[str],
fingerprints: Sequence[str],
depth: int,
max_loops: int,
model: str,
) -> str:
fingerprint: Final = _fingerprint_tools(tool_calls)
if fingerprint in fingerprints:
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@ -133,6 +161,83 @@ def _converted_stream_replay(
)
def _end_refused_turn(
finalized: ModelResponse,
*,
kwargs: Mapping[str, object],
depth: int,
model: str,
custom_llm_provider: str,
logging_obj: object,
) -> ModelResponse | CustomStreamWrapper:
replay: Final = _converted_stream_replay(
finalized,
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
return finalized if replay is None else replay
def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]:
return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)]
def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]:
import litellm
dynamic: Final = getattr(logging_obj, "dynamic_success_callbacks", None) or []
return tuple(callback for callback in (*litellm.callbacks, *dynamic) if isinstance(callback, CustomLogger))
async def _refused_chat_turn(
callbacks: Sequence[CustomLogger],
refusing: CustomLogger,
refused_tool_calls: object,
*,
response: ModelResponse,
model: str,
messages: list[dict[str, object]],
tools: list[dict[str, object]] | None,
custom_llm_provider: str,
kwargs: Mapping[str, object],
) -> ModelResponse:
surface_kwargs: Final = {
**kwargs,
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
"custom_llm_provider": custom_llm_provider,
}
def other_gates() -> Iterator[Awaitable[tuple[bool, object]]]:
for callback in callbacks:
if callback is refusing:
continue
yield callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(kwargs),
)
if _gate_overridden(callback):
yield callback.async_should_run_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(surface_kwargs),
)
claimed: Final = await every_claimed_tool_call(refused_tool_calls, other_gates())
return finalize_refused_chat_turn(response, claimed)
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
metadata: Final = kwargs_for_followup.get("litellm_metadata")
return MappingProxyType(
@ -163,6 +268,12 @@ def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]:
}
def _leave_pricing_to_the_request(response: object) -> None:
hidden_params: Final = get_hidden_params(response)
if hidden_params is not None and not served_from_cache(response):
hidden_params.pop("response_cost", None)
async def _execute_chat_completion_agentic_plan(
*,
plan: AgenticLoopPlan,
@ -174,7 +285,7 @@ async def _execute_chat_completion_agentic_plan(
custom_llm_provider: str,
depth: int,
max_loops: int,
fingerprints: list[str],
fingerprints: Sequence[str],
fingerprint: str,
) -> object:
import litellm
@ -204,12 +315,13 @@ async def _execute_chat_completion_agentic_plan(
)
try:
response_followup = await litellm.acompletion(
model=full_model_name,
messages=patch.messages,
**optional_params_for_followup,
**kwargs_for_followup,
)
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None):
response_followup = await litellm.acompletion(
model=full_model_name,
messages=patch.messages,
**optional_params_for_followup,
**kwargs_for_followup,
)
if _post_hook_overridden(callback):
try:
response_followup = await callback.async_post_agentic_loop_response_hook(
@ -224,6 +336,7 @@ async def _execute_chat_completion_agentic_plan(
model,
str(e),
)
_leave_pricing_to_the_request(response_followup)
replay: Final = _converted_stream_replay(
response_followup,
kwargs=kwargs,
@ -250,23 +363,18 @@ async def maybe_run_chat_completion_agentic_loop(
*,
response: ModelResponse,
model: str,
messages: list,
optional_params: dict,
messages: list[dict[str, object]],
optional_params: dict[str, object],
kwargs: dict[str, object],
logging_obj: object,
custom_llm_provider: str,
stream: bool,
) -> ModelResponse | CustomStreamWrapper | None:
import litellm
callbacks: Final = litellm.callbacks + (getattr(logging_obj, "dynamic_success_callbacks", None) or [])
callbacks: Final = _custom_loggers(logging_obj)
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
tools: Final = optional_params.get("tools", [])
tools: Final = _request_tools(optional_params)
for callback in callbacks:
if not isinstance(callback, CustomLogger):
continue
if not _gate_overridden(callback):
continue
@ -295,29 +403,39 @@ async def maybe_run_chat_completion_agentic_loop(
if not should_run:
continue
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
try:
if not _build_plan_overridden(callback):
return await callback.async_run_agentic_loop(
tools=tool_calls,
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return _end_refused_turn(
await _refused_chat_turn(
callbacks,
callback,
tool_calls,
response=response,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
),
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
plan = await callback.async_build_agentic_loop_plan(
if not _build_plan_overridden(callback):
return await callback.async_run_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
@ -329,31 +447,38 @@ async def maybe_run_chat_completion_agentic_loop(
kwargs=hook_kwargs,
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
return response
if not plan.run_agentic_loop:
continue
plan = await callback.async_build_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
except Exception as e:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s",
str(e),
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
return response
if not plan.run_agentic_loop:
continue
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
return _converted_stream_replay(
response,
@ -363,3 +488,113 @@ async def maybe_run_chat_completion_agentic_loop(
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
async def run_chat_completion_agentic_hooks(
*,
response: ModelResponse,
model: str,
messages: list[dict[str, object]],
optional_params: dict[str, object],
kwargs: dict[str, object],
logging_obj: LiteLLMLoggingObject,
custom_llm_provider: str,
stream: bool,
) -> object | None:
callbacks: Final = _custom_loggers(logging_obj)
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
tools: Final = _request_tools(optional_params)
hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider}
for callback in callbacks:
try:
should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
except Exception:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop"
)
continue
if not should_run:
continue
try:
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return await _refused_chat_turn(
callbacks,
callback,
tool_calls,
response=response,
model=model,
messages=messages,
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
if not _chat_build_plan_overridden(callback):
return await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
plan = await callback.async_build_chat_completion_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
verbose_logger.debug(
"Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason
)
return response
if not plan.run_agentic_loop:
continue
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=hook_kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
return None

16
litellm/litellm_core_utils/core_helpers.py
View file

@ -4,7 +4,7 @@ import copy
import logging
import math
import re
from collections.abc import Collection, Iterable, Iterator, Mapping
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@ -97,13 +97,17 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
return float(seconds / denominator)
_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
_LAX_BOOL: Final = TypeAdapter(bool)
def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def as_object_list(value: object) -> Sequence[object] | None:
return value if isinstance(value, list) else None
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:
@ -156,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]:
return iter_items(value) if value else ()
def normalize_drop_params(value: object) -> bool | None:
def lax_bool(value: object) -> bool | None:
if value is None or isinstance(value, bool):
return value
try:
return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
except ValidationError:
return None
def normalize_drop_params(value: object) -> bool | None:
return lax_bool(value)
def drops_unsupported_params(drop_params: object) -> bool:
return litellm.drop_params is True or normalize_drop_params(drop_params) is True

12
litellm/litellm_core_utils/decisions_pricing.py
View file

@ -12,6 +12,7 @@ from typing import Final
DECISIONS_PRICING_SUFFIX: Final = "_decisions"
DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
def is_decisions_call_type(call_type: str) -> bool:
@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
}
def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
return None
rates: Final = {**model_info, **overrides}
request_price: Final = rates.get("input_cost_per_query")
if not isinstance(request_price, (int, float)):
return None
bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
"""Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}

10
litellm/litellm_core_utils/deployment_pricing.py
View file

@ -1,4 +1,4 @@
from collections.abc import Iterator, Mapping
from collections.abc import Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import Final
@ -25,6 +25,14 @@ def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
def deployment_backend_model(deployment: object) -> str:
return next(iter(deployment_pricing_models(deployment)), "")
def deployment_backend_models(deployments: Iterable[object]) -> tuple[str, ...]:
return tuple(map(deployment_backend_model, deployments))
def _named_models(*candidates: object) -> Iterator[str]:
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)

4
litellm/litellm_core_utils/error_normalization.py
View file

@ -133,6 +133,8 @@ _STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType(
}
)
_OVERLOADED_STATUS_CODE: Final = "529"
_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = (
TypeError,
KeyError,
@ -201,6 +203,8 @@ def normalize_error(exc: Exception | None, status_code: str, message: str) -> st
)
if by_message is not None:
return by_message
if status_code == _OVERLOADED_STATUS_CODE:
return PROVIDER_OVERLOADED
by_class: Final = _classify_by_class(exc)
if by_class is not None:
return by_class

11
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import _ENABLE_SECRET_REDACTION, redact_string, verbose_logger
from litellm.litellm_core_utils.agentic_followup_failure import carry_agentic_followup_mark
from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
@ -107,6 +108,7 @@ class ExceptionCheckers:
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
"prompt is too long",
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:
@ -604,6 +606,11 @@ def _map_openai_exception(
)
def _anthropic_overload_status(original_exception: _ProviderHTTPException) -> int | None:
upstream_status: Final = getattr(original_exception, "status_code", None)
return 529 if isinstance(upstream_status, int) and upstream_status >= 500 else None
def _map_anthropic_exception(
*,
model: str,
@ -629,6 +636,7 @@ def _map_anthropic_exception(
message=f"AnthropicError - {error_str}",
model=model,
llm_provider="anthropic",
provider_status_code=_anthropic_overload_status(original_exception),
)
if "Invalid API Key" in error_str:
raise AuthenticationError(
@ -693,6 +701,7 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
response=getattr(original_exception, "response", None),
provider_status_code=original_exception.status_code,
)
elif original_exception.status_code == 502:
raise BadGatewayError(
@ -2894,6 +2903,7 @@ def exception_type(
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request
)
except Exception as e:
carry_agentic_followup_mark(source=original_exception, mapped=e)
# LOGGING
exception_logging(
logger_fn=None,
@ -2919,6 +2929,7 @@ def exception_type(
model="",
)
setattr(raised_exc, "litellm_response_headers", litellm_response_headers)
carry_agentic_followup_mark(source=original_exception, mapped=raised_exc)
raise raised_exc

4
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -33,6 +33,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
)
PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header"
AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset(
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"}
)
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset(
{
"token_exchange_endpoint",
@ -94,6 +97,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
| ANTHROPIC_WIF_KWARGS_KEYS
| OPENAI_WIF_KWARGS_KEYS
| OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS
| AGENTIC_LOOP_CONTROL_KWARGS_KEYS
| frozenset(CustomPricingLiteLLMParams.model_fields)
)

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
declared: Final = declared_provider(model, custom_llm_provider)
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None

27
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -26,7 +26,12 @@ from litellm.constants import (
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
NON_INFERENCE_CALL_TYPES,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@ -35,6 +40,8 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
return is_unbilled_non_inference_call(call_type, metadata, response)
def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
)
def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
call_type: Final = kwargs.get("call_type")
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = (
as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
)
if not event_id or not settles_background_response(
call_type if isinstance(call_type, str) else "", metadata, kwargs
):
return event_id
return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

314
litellm/litellm_core_utils/litellm_logging.py
View file

@ -19,9 +19,11 @@ from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
from openai import Stream as OpenAIStream
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -52,8 +54,11 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
declared_cost_map_pricing,
get_response_cost_from_hidden_params,
get_usage_object,
pricing_entry_for_cost_calc,
select_model_name_for_cost_calc,
@ -72,6 +77,8 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.langtrace import langtrace_trace_endpoint
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
from litellm.litellm_core_utils.classifier_logging import (
classifier_audit_fields,
classifier_input_snapshot,
@ -97,6 +104,7 @@ from litellm.litellm_core_utils.hidden_params import (
get_hidden_params,
served_from_cache,
set_hidden_param,
set_hidden_params,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
@ -122,6 +130,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
diagnose_zero_cost,
zero_cost_warning,
)
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler
from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
@ -142,7 +151,7 @@ from litellm.litellm_core_utils.served_output_texts import (
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
@ -211,7 +220,10 @@ from litellm.types.utils import (
batch_chat_usage,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
from litellm.utils import (
get_base_model_from_metadata,
print_verbose,
)
from ..integrations.argilla import ArgillaLogger
from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
@ -300,6 +312,11 @@ class _EnterpriseAlertingLoggers(NamedTuple):
smtp_email: type[CustomLogger]
class BridgedChatRequest(NamedTuple):
optional_params: Mapping[str, object]
litellm_params: Mapping[str, object]
@functools.cache
def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers:
try:
@ -580,6 +597,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
_REDACTED_RESULT: Final = TypeAdapter(object)
_LoggedResponse: Final = TypeVar("_LoggedResponse")
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -706,6 +724,7 @@ class Logging(LiteLLMLoggingBaseClass):
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
custom_model_info: ModelInfo | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -872,6 +891,8 @@ class Logging(LiteLLMLoggingBaseClass):
# The model the proxy shows the client on streamed chunks. The logged streamed response carries it
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.bridged_chat_request: BridgedChatRequest | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -918,12 +939,18 @@ class Logging(LiteLLMLoggingBaseClass):
# post_call guardrails have run; the @client decorator then stores the
# enqueue closure here instead of firing it immediately.
self.defer_async_logging: bool = False
self.drops_stream_token_ids: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self._stream_failure_claimed: bool = False
self._stash_owed_on_success: bool = False
self._stash_billed_by_continuation: bool = False
self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
self._failed_attempt_ids_owed_on_success: tuple[str, ...] = ()
self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = ()
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
@ -2128,6 +2155,9 @@ class Logging(LiteLLMLoggingBaseClass):
if resolved_cache_hit is True:
return 0.0
if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
return None
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):
@ -2205,6 +2235,7 @@ class Logging(LiteLLMLoggingBaseClass):
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"requested_model_info_override": self.custom_model_info if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
@ -2296,12 +2327,19 @@ class Logging(LiteLLMLoggingBaseClass):
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True)
def _price_without_recording(
self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool
) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
previous_failure_debug_info: Final = self.model_call_details.get(debug_key)
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
cost: Final = self.response_cost_calculator(
result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost
)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
@ -2401,6 +2439,10 @@ class Logging(LiteLLMLoggingBaseClass):
additional_headers=additional_headers,
)
def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None:
"""Price this call from its own full price entry instead of ``litellm.model_cost``."""
self.custom_model_info = custom_model_info
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
@ -2566,6 +2608,33 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
if (
cost != 0
or self._provider_reported_cost == 0
or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0
):
return True
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
return True
params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {})
if is_unbilled_non_inference_call(
payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result
):
return True
if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
return True
passthrough: Final = _LITELLM_PARAMS.validate_python(
self._model_call_details_view.get("passthrough_logging_payload") or {}
)
if passthrough.get("cost_per_request") is not None:
return True
model_info: Final = payload["model_map_information"]["model_map_value"]
if model_info is None:
return False
prices: Final = declared_cost_map_pricing(model_info["key"], model_info["key"], payload["custom_llm_provider"])
return any(value is not None and ("cost" in key or "pricing" in key) for key, value in prices.items())
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
@ -2618,6 +2687,7 @@ class Logging(LiteLLMLoggingBaseClass):
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
price_known=self._response_price_is_known(prepared[2], raw_cost, payload),
)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
@ -2721,27 +2791,43 @@ class Logging(LiteLLMLoggingBaseClass):
own usage while the request's failure log still bills every attempt."""
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
billed_by_continuation: Final = self._stash_billed_by_continuation
owed_on_success: Final = self._stash_owed_on_success
self._stash_billed_by_continuation = False
self._stash_owed_on_success = False
captured_attempt_ids: Final = (
self.capture_failed_attempt_usage(usage)
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None
else ()
)
self.model_call_details.pop("_accounting_capture", None)
self.model_call_details.pop("_accounting_pricing_snapshot", None)
self.model_call_details.pop("_accounting_usage_observed", None)
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
if not isinstance(usage, Usage):
return
attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0)
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
attempt,
)
if not owed_on_success or billed_by_continuation:
return
self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt)
self._failed_attempt_ids_owed_on_success = (*self._failed_attempt_ids_owed_on_success, *captured_attempt_ids)
def capture_failed_attempt_usage(self, usage: Usage) -> None:
def mark_stash_billed_by_continuation(self) -> None:
self._stash_billed_by_continuation = True
def capture_failed_attempt_usage(self, usage: Usage) -> tuple[str, ...]:
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
capture: Final = type(self).usage_capture_callback
if capture is None:
return
return ()
attempt: Final = self.attempt_scoped_copy()
attempt.prepared_cost = None
attempt.finalized_cost = None
@ -2776,11 +2862,13 @@ class Logging(LiteLLMLoggingBaseClass):
datetime.datetime.now(datetime.timezone.utc),
False,
)
if isinstance(captured, CapturedUsage):
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
if not isinstance(captured, CapturedUsage):
return ()
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
return tuple(write.event.event_id for write in captured.writes)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
@ -2788,20 +2876,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
def _add_continued_partial_spend(self, response: object) -> Usage | None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
return own_usage if isinstance(own_usage, Usage) else None
combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage])
self.model_call_details["combined_usage_object"] = combined_usage
if not isinstance(own_cost, (int, float)):
return
return combined_usage
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
@ -2811,6 +2896,27 @@ class Logging(LiteLLMLoggingBaseClass):
"mid_stream_partial_cost": partial_cost,
},
)
return combined_usage
def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None:
own_cost: Final = self.model_call_details.get("response_cost")
if own_usage is None or not isinstance(own_cost, (int, float)):
return
failed_attempts: Final = self._failed_attempts_owed_on_success
if not failed_attempts:
return
failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts)
self.model_call_details.pop("failed_attempts_usage", None)
self.failed_attempts_billed_on_success = failed_attempts
self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
if self._failed_attempt_ids_owed_on_success:
self.model_call_details["_accounting_included_attempt_ids"] = self._failed_attempt_ids_owed_on_success
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
)
self.model_call_details["response_cost"] = own_cost + failed_attempts_cost
if self.cost_breakdown is not None:
self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost)
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
@ -2818,6 +2924,24 @@ class Logging(LiteLLMLoggingBaseClass):
self._stream_failure_claimed = True
return True
def served_round(self, response: object) -> ServedRound | None:
usage: Final = get_usage_object(completion_response=response)
if usage is None:
return None
price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False)
return usage, price.cost or 0.0
def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None:
if not served_rounds:
return
self._stash_owed_on_success = True
self.record_partial_usage_for_failure(
served_rounds[0][0]
if len(served_rounds) == 1
else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]),
sum(cost for _, cost in served_rounds),
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -3122,9 +3246,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("cache_hit") is True:
self.model_call_details["response_cost"] = 0.0
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
@ -3134,6 +3260,7 @@ class Logging(LiteLLMLoggingBaseClass):
pass
else:
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
if not build_logging_payload:
return None
@ -3228,6 +3355,40 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
return self._bridged_chat_response(response, request)
def _as_served_response(
self, response: _LoggedResponse
) -> _LoggedResponse | ModelResponse | TextCompletionResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
served: Final = self._bridged_chat_response(response, request)
if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
return served
return self._as_text_completion_priced_as_chat(served)
def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse:
text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion(
served, TextCompletionResponse()
)
set_hidden_params(text_completion, dict(get_hidden_params(served) or {}))
return text_completion
def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse:
translated: Final = self._translate_responses_api_response_to_model_response(response, request)
served: Final = translated.model_copy(
update={
"id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id)
}
)
set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})})
return served
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
@ -3237,6 +3398,23 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"id": response_id})
def _with_client_facing_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.client_facing_response_id
if response_id is None or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _client_facing_streaming_response(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
return self._with_client_facing_response_id(
self._with_continued_response_id(self._with_client_facing_stream_model(response))
)
def _success_handler_helper_fn(
self,
result: object = None,
@ -3292,6 +3470,8 @@ class Logging(LiteLLMLoggingBaseClass):
)
result = self.normalize_endpoint_logging_result(result)
if self.stream is not True:
result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@ -3606,6 +3786,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
start_time=start_time,
end_time=end_time,
@ -3620,7 +3801,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
) = None
if "complete_streaming_response" in self.model_call_details and not for_async_request:
return # break out of this.
@ -3636,19 +3822,21 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["complete_streaming_response"] = complete_streaming_response
self._surface_response_headers_from_result(complete_streaming_response)
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
@ -4175,6 +4363,7 @@ class Logging(LiteLLMLoggingBaseClass):
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
)
)
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result = self._success_handler_helper_fn(
start_time=start_time,
end_time=end_time,
@ -4194,9 +4383,7 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
complete_streaming_response: Final = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
@ -4218,7 +4405,7 @@ class Logging(LiteLLMLoggingBaseClass):
get_base_model_from_metadata(model_call_details=self.model_call_details)
# base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
@ -4233,21 +4420,23 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
try:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release)
verbose_logger.exception(
@ -5117,10 +5306,17 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
) -> (
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
):
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
return result
elif isinstance(
result,
@ -5191,6 +5387,13 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_chat_request(
self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> None:
self.bridged_chat_request = BridgedChatRequest(
MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params))
)
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
@ -5262,6 +5465,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
result = result.response
if isinstance(result, ResponsesAPIResponse):
bridged_request: Final = self.bridged_chat_request
if bridged_request is not None:
return self._bridged_chat_response(result, bridged_request)
return self._translate_responses_api_response_to_model_response(result)
raw_body: Final[object] = result
@ -5313,7 +5519,9 @@ class Logging(LiteLLMLoggingBaseClass):
set_response_cost_in_hidden_params(result, reported_cost)
return result
def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
def _translate_responses_api_response_to_model_response(
self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None
) -> ModelResponse:
"""
Convert a Responses API response into a ModelResponse for spend_logs.
@ -5336,8 +5544,8 @@ class Logging(LiteLLMLoggingBaseClass):
logging_obj=self,
request_data={},
messages=[],
optional_params={},
litellm_params={},
optional_params={} if request is None else dict(request.optional_params),
litellm_params={} if request is None else dict(request.litellm_params),
encoding=litellm.encoding,
)
translated.id = served_id or translated.id
@ -7705,6 +7913,20 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
)
def _with_failed_attempts_cost(breakdown: CostBreakdown, own_cost: float, failed_attempts_cost: float) -> CostBreakdown:
return breakdown | CostBreakdown(
total_cost=own_cost + failed_attempts_cost,
additional_costs={**breakdown.get("additional_costs", {}), "failed_attempts_cost": failed_attempts_cost},
)
def _cached_cost_breakdown(served: CostBreakdown | None, failed_attempts: tuple[ServedRound, ...]) -> CostBreakdown:
cached: Final = _cache_hit_cost_breakdown(served)
if not failed_attempts:
return cached
return _with_failed_attempts_cost(cached, 0.0, sum(cost for _, cost in failed_attempts))
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
finalized: Final = getattr(logging_obj, "finalized_cost", None)
@ -7898,7 +8120,7 @@ def get_standard_logging_object_payload(
if finalized_cost is not None
else cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
_cached_cost_breakdown(uncached_price.cost_breakdown, logging_obj.failed_attempts_billed_on_success)
if uncached_price is not None
else logging_obj.cost_breakdown
),

Some files were not shown because too many files have changed in this diff Show more