scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-23 00:41:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge remote-tracking branch 'origin/main' into litellm_mcp_client_allowlist

Browse source
This commit is contained in:
yassin 2026-09-17 21:46:21 +00:00
commit fb99e3dde3
48 changed files with 1564 additions and 401 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
db_scripts/partition_spend_logs.sql
View file

@ -53,6 +53,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_end_user_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_end_user_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_session_id_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_api_key_startTime_idx";
CREATE TABLE "LiteLLM_SpendLogs" (
LIKE "LiteLLM_SpendLogs_legacy" INCLUDING DEFAULTS INCLUDING GENERATED
@ -78,6 +80,9 @@ CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_end_user_idx"
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_session_id_idx"
ON "LiteLLM_SpendLogs" ("session_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
ON "LiteLLM_SpendLogs" ("api_key", "startTime");
-- Safety net: any row whose startTime has no explicit partition lands here so
-- writes never fail. The cleanup job never drops the DEFAULT partition.
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogs_pdefault"

5
db_scripts/unpartition_spend_logs.sql
View file

@ -40,6 +40,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_end_user_idx"
RENAME TO "LiteLLM_SpendLogs_partitioned_end_user_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
RENAME TO "LiteLLM_SpendLogs_partitioned_session_id_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
RENAME TO "LiteLLM_SpendLogs_partitioned_api_key_startTime_idx";
CREATE TABLE "LiteLLM_SpendLogs" (
LIKE "LiteLLM_SpendLogs_partitioned" INCLUDING DEFAULTS INCLUDING GENERATED
@ -60,6 +62,9 @@ CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_end_user_idx"
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_session_id_idx"
ON "LiteLLM_SpendLogs" ("session_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
ON "LiteLLM_SpendLogs" ("api_key", "startTime");
INSERT INTO "LiteLLM_SpendLogs"
SELECT * FROM "LiteLLM_SpendLogs_partitioned"
ON CONFLICT ("request_id") DO NOTHING;

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -678,6 +678,7 @@ model LiteLLM_SpendLogs {
@@index([end_user])
@@index([session_id])
@@index([litellm_call_id])
@@index([api_key, startTime])
}
model LiteLLM_BudgetWindowSpend {

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.98"
version = "0.4.99"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.98"
version = "0.4.99"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

12
litellm/caching/dual_cache.py
View file

@ -521,6 +521,18 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
await self.redis_cache.async_delete_cache(key)
async def async_delete_cache_keys(self, keys: Sequence[str]) -> None:
"""Batch twin of ``async_delete_cache``, chunked because Redis takes the
whole list as one DELETE command."""
if not keys:
return
for key in keys:
self.in_memory_cache.delete_cache(key)
if self.redis_cache is None:
return
for start in range(0, len(keys), DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE):
await self.redis_cache.delete_cache_keys(keys[start : start + DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE])
async def async_get_ttl(self, key: str) -> int | None:
"""
Get the remaining TTL of a key in in-memory cache or redis

2
litellm/constants.py
View file

@ -321,6 +321,8 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE: Final = ">=0.10.0,<0.12.0"
CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved"
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"

3
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -508,7 +508,8 @@ class AnthropicMessagesHandler(BaseTranslation):
chat_completion_compatible_request,
_tool_name_mapping,
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()),
preserve_midturn_system=True,
)
return chat_completion_compatible_request

36
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -118,6 +118,10 @@ from litellm.llms.anthropic.common_utils import (
from litellm.llms.anthropic.experimental_pass_through.context_management import (
PolyfillResult,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
convert_mid_conversation_system_turns,
is_system_role_message,
)
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
openai_chat_refusal_text,
refusal_stop_details,
@ -176,6 +180,7 @@ from litellm.types.llms.openai import (
ToolMessageContentPart,
)
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
from litellm.utils import supports_mid_conversation_system
from .streaming_iterator import AnthropicStreamWrapper
@ -186,6 +191,12 @@ if TYPE_CHECKING:
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
def target_supports_mid_conversation_system(model: str | None, custom_llm_provider: str | None) -> bool:
if not model:
return False
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
class AnthropicAdapter:
def __init__(self) -> None:
pass
@ -418,10 +429,28 @@ class LiteLLMAnthropicMessagesAdapter:
self,
messages: list[AllAnthropicPassThroughMessageValues],
model: str | None = None,
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
) -> list:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
for m in replayable_messages:
leading_count: Final = next(
(i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
len(replayable_messages),
)
trailing_messages: Final = replayable_messages[leading_count:]
keeps_midturn_system: Final = (
preserve_midturn_system
or not any(is_system_role_message(m) for m in trailing_messages)
or target_supports_mid_conversation_system(model, custom_llm_provider)
)
ordered_messages: Final = (
replayable_messages
if keeps_midturn_system
else (*replayable_messages[:leading_count], *convert_mid_conversation_system_turns(trailing_messages))
)
for m in ordered_messages:
user_message: ChatCompletionUserMessage | None = None
tool_message_list: list[ChatCompletionToolMessage] = []
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []
@ -494,7 +523,7 @@ class LiteLLMAnthropicMessagesAdapter:
if isinstance(m.get("content"), str):
assistant_message_str = str(m.get("content", ""))
elif isinstance(m.get("content"), list):
for content in m.get("content", []):
for content in cast(list, m.get("content", [])): # cast-ok: untrusted client payload
if isinstance(content, str):
assistant_message_str = str(content)
elif isinstance(content, dict):
@ -1154,6 +1183,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1175,6 +1205,8 @@ class LiteLLMAnthropicMessagesAdapter:
new_messages = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
preserve_midturn_system=preserve_midturn_system,
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)

6
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -765,7 +765,8 @@ def _count_effective_tokens(
messages=cast(
"list[AllAnthropicPassThroughMessageValues]",
messages_without_compaction,
)
),
preserve_midturn_system=True,
)
except Exception as e:
verbose_logger.debug(
@ -920,7 +921,8 @@ def _build_summary_messages(
messages=cast(
"list[AllAnthropicPassThroughMessageValues]",
stripped,
)
),
preserve_midturn_system=True,
)
except Exception as e:
verbose_logger.warning(

77
litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py Normal file
View file

@ -0,0 +1,77 @@
from collections.abc import Mapping, Sequence
from itertools import groupby
from typing import Final
CONVERTED_SYSTEM_NOTE: Final = (
"Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
)
def as_system_content_blocks(value: object) -> list[object]:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
def is_system_role_message(message: object) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def system_role_message_as_user(message: Mapping[str, object]) -> Mapping[str, object]:
return {
"role": "user",
"content": as_system_content_blocks(CONVERTED_SYSTEM_NOTE) + as_system_content_blocks(message.get("content")),
}
def opens_with_tool_results(message: object) -> bool:
if not isinstance(message, dict) or message.get("role") != "user":
return False
content: Final = message.get("content")
return (
isinstance(content, list)
and len(content) > 0
and isinstance(content[0], dict)
and content[0].get("type") == "tool_result"
)
def system_run_placed_after_tool_results(
system_run: Sequence[Mapping[str, object]], follower_run: Sequence[Mapping[str, object]]
) -> tuple[Mapping[str, object], ...]:
if follower_run and opens_with_tool_results(follower_run[0]):
return (follower_run[0], *system_run, *follower_run[1:])
return (*system_run, *follower_run)
def system_turns_after_tool_results(
messages: Sequence[Mapping[str, object]],
) -> tuple[Mapping[str, object], ...]:
runs: Final = tuple(tuple(run) for _, run in groupby(messages, key=is_system_role_message))
if not runs:
return ()
first_system_run: Final = 0 if is_system_role_message(runs[0][0]) else 1
paired_runs: Final = tuple(
(runs[i], runs[i + 1] if i + 1 < len(runs) else ()) for i in range(first_system_run, len(runs), 2)
)
return (
*(runs[0] if first_system_run else ()),
*(
m
for system_run, follower_run in paired_runs
for m in system_run_placed_after_tool_results(system_run, follower_run)
),
)
def convert_mid_conversation_system_turns(
messages: Sequence[Mapping[str, object]],
) -> tuple[Mapping[str, object], ...]:
return tuple(
system_role_message_as_user(m) if is_system_role_message(m) else m
for m in system_turns_after_tool_results(messages)
)

81
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -27,6 +27,11 @@ from ...common_utils import (
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
from .mid_conversation_system import (
as_system_content_blocks,
convert_mid_conversation_system_turns,
is_system_role_message,
)
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -151,73 +156,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
else:
return system_param
@staticmethod
def _as_system_content_blocks(value: object) -> list:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
@staticmethod
def _is_system_role_message(message: object) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
_CONVERTED_SYSTEM_NOTE: Final = (
"Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
)
def _system_role_message_as_user(self, message: Mapping) -> Mapping:
return {
"role": "user",
"content": self._as_system_content_blocks(self._CONVERTED_SYSTEM_NOTE)
+ self._as_system_content_blocks(message.get("content")),
}
@staticmethod
def _opens_with_tool_results(message: object) -> bool:
if not isinstance(message, dict) or message.get("role") != "user":
return False
content: Final = message.get("content")
return (
isinstance(content, list)
and len(content) > 0
and isinstance(content[0], dict)
and content[0].get("type") == "tool_result"
)
def _system_run_before(self, messages: Sequence, index: int) -> Sequence:
start: Final = next(
(j + 1 for j in range(index - 1, -1, -1) if not self._is_system_role_message(messages[j])),
0,
)
return messages[start:index]
def _system_run_end(self, messages: Sequence, index: int) -> int:
return next(
(j for j in range(index, len(messages)) if not self._is_system_role_message(messages[j])),
len(messages),
)
def _reordered_around_tool_results(self, messages: Sequence, index: int) -> tuple:
message: Final = messages[index]
if self._opens_with_tool_results(message):
return (message, *self._system_run_before(messages, index))
if not self._is_system_role_message(message):
return (message,)
run_end: Final = self._system_run_end(messages, index)
follower: Final = messages[run_end] if run_end < len(messages) else None
return () if self._opens_with_tool_results(follower) else (message,)
def _system_turns_after_tool_results(self, messages: Sequence) -> tuple:
return tuple(
message
for index in range(len(messages))
for message in self._reordered_around_tool_results(messages, index)
)
def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None:
"""Normalize ``role: "system"`` entries in ``messages`` per the Anthropic
``/v1/messages`` contract, which the first-party API, Bedrock Invoke,
@ -254,7 +192,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if not isinstance(messages, list):
return
leading_count: Final = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
(i for i, m in enumerate(messages) if not is_system_role_message(m)),
len(messages),
)
hoisted: Final = messages[:leading_count]
@ -265,10 +203,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self.custom_llm_provider,
key="supports_mid_conversation_system",
)
else [
self._system_role_message_as_user(m) if self._is_system_role_message(m) else m
for m in self._system_turns_after_tool_results(messages[leading_count:])
]
else list(convert_mid_conversation_system_turns(messages[leading_count:]))
)
if hoisted or remaining != messages:
anthropic_messages_request["messages"] = remaining
@ -278,7 +213,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
anthropic_messages_request.get("system"),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
for block in as_system_content_blocks(source)
]
filtered_system: Final = self._filter_billing_headers_from_system(system_content)
if filtered_system:

86
litellm/llms/bedrock/realtime/handler.py
View file

@ -6,11 +6,12 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
import asyncio
import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Mapping, MutableMapping
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, NoReturn, Protocol
from typing import Final, NoReturn, Protocol, runtime_checkable
from pydantic import JsonValue, TypeAdapter
@ -19,6 +20,8 @@ from litellm._logging import _redact_string, verbose_proxy_logger
from litellm.constants import (
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_DISTRIBUTION,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
@ -121,6 +124,38 @@ class BedrockBidirectionalStream(Protocol):
async def await_output(self) -> tuple[object, BedrockOutputStream]: ...
@runtime_checkable
class ClosableBedrockRuntimeClient(Protocol):
async def close(self) -> None: ...
def _installed_sdk_version() -> str | None:
try:
return importlib.metadata.version(BEDROCK_REALTIME_SDK_DISTRIBUTION)
except importlib.metadata.PackageNotFoundError:
return None
def _sdk_import_error(installed_version: str | None, cause: ImportError) -> ImportError:
install_hint: Final = "pip install 'litellm[bedrock-realtime]'"
requirement: Final = f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}"
verbose_proxy_logger.error("Bedrock Realtime: SDK import failed (installed=%s): %s", installed_version, cause)
if installed_version is None:
return ImportError(f"Missing aws_sdk_bedrock_runtime: {install_hint} ({requirement})")
return ImportError(
f"{BEDROCK_REALTIME_SDK_DISTRIBUTION} {installed_version} is installed but Bedrock realtime needs "
f"[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}: {install_hint}"
)
async def _close_bedrock_client(bedrock_client: object) -> None:
if not isinstance(bedrock_client, ClosableBedrockRuntimeClient):
return
with contextlib.suppress(Exception):
await bedrock_client.close()
verbose_proxy_logger.debug("Bedrock Realtime: closed SDK client")
@dataclass(frozen=True, slots=True)
class _BridgeOutcome:
logged_events: tuple[OpenAIRealtimeEvents, ...]
@ -199,8 +234,9 @@ async def _ack_session_update(
class BedrockRealtime(BaseAWSLLM):
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
def __init__(self):
def __init__(self, sdk_version_lookup: Callable[[], str | None] = _installed_sdk_version):
super().__init__()
self._sdk_version_lookup: Final = sdk_version_lookup
async def async_realtime(
self,
@ -234,14 +270,13 @@ class BedrockRealtime(BaseAWSLLM):
Various AWS authentication parameters
"""
try:
from aws_sdk_bedrock_runtime.client import (
BedrockRuntimeClient,
InvokeModelWithBidirectionalStreamOperationInput,
)
from aws_sdk_bedrock_runtime.config import Config
from smithy_aws_core.identity import StaticCredentialsResolver
except ImportError:
raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime")
from aws_sdk_bedrock_runtime.client import AsyncBedrockRuntimeClient
from aws_sdk_bedrock_runtime.config import AsyncBedrockRuntimeConfig
from aws_sdk_bedrock_runtime.models import InvokeModelWithBidirectionalStreamOperationInput
from smithy_aws_core.identity import AWSCredentialsIdentity, StaticCredentialsResolver
from smithy_http.aio.crt import AWSCRTHTTPClient
except ImportError as e:
raise _sdk_import_error(self._sdk_version_lookup(), e) from e
pending_session_update: Final = _pending_session_update(websocket.scope)
@ -285,22 +320,37 @@ class BedrockRealtime(BaseAWSLLM):
)
frozen_credentials: Final = await run_aws_signing(credentials.get_frozen_credentials)
# Initialize Bedrock client with aws_sdk_bedrock_runtime
config: Final = Config(
credentials_identity: Final = AWSCredentialsIdentity(
access_key_id=frozen_credentials.access_key,
secret_access_key=frozen_credentials.secret_key,
session_token=frozen_credentials.token,
)
config: Final = await AsyncBedrockRuntimeConfig.resolve(
endpoint_uri=endpoint_uri,
region=aws_region_name,
aws_access_key_id=frozen_credentials.access_key,
aws_secret_access_key=frozen_credentials.secret_key,
aws_session_token=frozen_credentials.token,
aws_credentials_identity_resolver=StaticCredentialsResolver(),
aws_credentials_identity_resolver=StaticCredentialsResolver(identity=credentials_identity),
transport=AWSCRTHTTPClient(),
)
bedrock_client: Final = BedrockRuntimeClient(config=config)
bedrock_client: Final = AsyncBedrockRuntimeClient(config=config)
async def open_bidirectional_stream() -> BedrockBidirectionalStream:
return await bedrock_client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=model)
)
try:
await self._run_session(websocket, open_bidirectional_stream, model, logging_obj, pending_session_update)
finally:
await _close_bedrock_client(bedrock_client)
async def _run_session(
self,
websocket: RealtimeClientWebSocket,
open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]],
model: str,
logging_obj: LiteLLMLogging,
pending_session_update: str | None,
) -> None:
transformation_config: Final = BedrockRealtimeConfig()
bedrock_stream: Final = await open_bidirectional_stream()

135
litellm/model_prices_and_context_window_backup.json
View file

@ -7605,7 +7605,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7733,7 +7733,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7887,7 +7887,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
@ -7956,7 +7956,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8856,7 +8856,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/us/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -8955,7 +8955,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/eu/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -9054,7 +9054,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
@ -10987,14 +10987,14 @@
"supports_vision": true
},
"azure_ai/FW-Kimi-K3": {
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.5e-05,
"reasoning_effort_levels": [
"low",
"high",
@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -45245,7 +45245,7 @@
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-20b": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@ -45482,6 +45482,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45503,7 +45504,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://api.together.xyz/v1/models",
"source": "https://api.together.ai/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
@ -45528,6 +45529,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 1.3e-07,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45552,7 +45554,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 3.9e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@ -45567,7 +45569,7 @@
"supports_vision": true
},
"together_ai/intfloat/multilingual-e5-large-instruct": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 2e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 514,
@ -45680,7 +45682,7 @@
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
@ -50573,7 +50575,7 @@
"wandb/openai/gpt-oss-120b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.7e-07,
@ -50584,7 +50586,7 @@
"wandb/openai/gpt-oss-20b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.3e-07,
@ -50593,6 +50595,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -50603,6 +50606,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"deprecation_date": "2026-08-04",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50612,6 +50616,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"deprecation_date": "2026-08-25",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50622,6 +50627,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"deprecation_date": "2026-08-04",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
@ -50632,6 +50638,7 @@
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"deprecation_date": "2026-03-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -50656,6 +50663,7 @@
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"deprecation_date": "2026-08-25",
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
@ -50670,7 +50678,7 @@
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"max_output_tokens": 128000,
"input_cost_per_token": 2.2e-07,
"output_cost_per_token": 2.2e-07,
@ -50690,6 +50698,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 161000,
"max_input_tokens": 161000,
@ -50700,6 +50709,7 @@
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"deprecation_date": "2026-03-04",
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
@ -50719,6 +50729,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"deprecation_date": "2026-04-21",
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
@ -50728,6 +50739,7 @@
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"deprecation_date": "2026-08-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -56692,7 +56704,8 @@
"supports_function_calling": true,
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true
"gemini_audio_only_live": true,
"supports_response_schema": false
},
"gemini-3.8-live-extended-thinking": {
"input_cost_per_audio_token": 3e-06,
@ -56726,7 +56739,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -60970,10 +60984,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -60983,7 +60998,7 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
@ -60996,10 +61011,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Pro": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.15e-06,
"output_cost_per_token": 2.55e-06,
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61009,7 +61025,7 @@
"wandb/google/gemma-4-31B-it": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3.4e-07,
"litellm_provider": "wandb",
@ -61018,8 +61034,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/ibm-granite/granite-4.1-8b": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61028,8 +61045,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61038,8 +61056,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-3.1-70B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"input_cost_per_token": 8e-07,
"output_cost_per_token": 8e-07,
"litellm_provider": "wandb",
@ -61050,7 +61069,7 @@
"wandb/MiniMaxAI/MiniMax-M3": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 9.6e-07,
"cache_read_input_token_cost": 5e-08,
@ -61063,7 +61082,7 @@
"wandb/moonshotai/Kimi-K2.7-Code": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 7.1e-07,
"output_cost_per_token": 3.5e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61076,7 +61095,7 @@
"wandb/moonshotai/Kimi-K2.6": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3.41e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61089,10 +61108,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 262000,
"input_cost_per_token": 7e-08,
"output_cost_per_token": 2e-07,
"cache_read_input_token_cost": 4e-08,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61102,10 +61121,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 7.5e-07,
"output_cost_per_token": 2.75e-06,
"cache_read_input_token_cost": 1.5e-07,
"max_input_tokens": 262000,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 1e-07,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61113,8 +61132,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/OpenPipe/Qwen3-14B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_input_tokens": 32800,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
@ -61125,7 +61145,7 @@
"wandb/Qwen/Qwen3.8-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61138,7 +61158,7 @@
"wandb/Qwen/Qwen3.6-35B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61149,10 +61169,11 @@
"wandb/Qwen/Qwen3.6-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3.6e-06,
"cache_read_input_token_cost": 1.2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61160,9 +61181,10 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3.5-35B-A3B": {
"deprecation_date": "2026-10-05",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61171,8 +61193,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"deprecation_date": "2026-10-05",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "wandb",
@ -61187,6 +61210,7 @@
"input_cost_per_token": 1.31e-06,
"output_cost_per_token": 3.96e-06,
"cache_read_input_token_cost": 4.4e-08,
"max_input_tokens": 1049000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
@ -61197,13 +61221,14 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 131000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-5.2": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 1049000,
"input_cost_per_token": 7.6e-07,
"output_cost_per_token": 2.42e-06,
"cache_read_input_token_cost": 1.4e-07,
@ -62866,7 +62891,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -69158,5 +69183,19 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_vision": true
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "wandb",
"max_input_tokens": 1049000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://wandb.ai/site/pricing/tokens/",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
}
}

8
litellm/proxy/common_request_processing.py
View file

@ -1937,6 +1937,14 @@ class ProxyBaseLLMRequestProcessing:
) -> tuple[dict, LiteLLMLoggingObj]:
start_time: Final = datetime.now() # start before calling guardrail hooks
requested_model: Final = self.data.get("model")
if requested_model is not None and not isinstance(requested_model, str):
raise ProxyException(
message="'model' must be a string.",
type=ProxyErrorTypes.bad_request_error,
param="model",
code=status.HTTP_400_BAD_REQUEST,
)
self.data = await add_litellm_data_to_request(
data=self.data,
request=request,

180
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -26,7 +26,6 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy._types import (
DB_RETRY_SAFE_ERROR_TYPES,
LiteLLM_BudgetTableFull,
LiteLLM_EndUserTable,
Litellm_EntityType,
LiteLLM_TeamTable,
LiteLLM_UserTable,
@ -193,13 +192,6 @@ def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]:
return (end_user_cache_key(row.user_id),)
def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float:
if not caps:
return 0.0
effective_budget_id: Final[str | None] = row.budget_id or litellm.max_end_user_budget_id
return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None)
def _budget_link_where(
budget_ids: Sequence[str],
extra: Mapping[str, object] = MappingProxyType({}),
@ -207,6 +199,19 @@ def _budget_link_where(
return {"budget_id": {"in": list(budget_ids)}, **extra}
def _enduser_invalidation_where(budget_ids: Sequence[str]) -> dict[str, object]:
"""Customers whose cached spend a committed reset of these tiers invalidated.
Mirrors ``_queue_enduser_resets`` without its ``spend > 0`` filter, which
post-commit would match nobody.
"""
linked: Final = _budget_link_where(budget_ids)
default_budget_id: Final = litellm.max_end_user_budget_id
if default_budget_id is None or default_budget_id not in budget_ids:
return linked
return {"OR": [linked, {"budget_id": None}]} # mutable-ok: prisma where filter must be a dict
def _queue_budget_linked_resets(
writes: LinkedSpendResetWrites,
cascade: "_BudgetCascade",
@ -265,16 +270,29 @@ class _BudgetCascade:
budgets: tuple[LiteLLM_BudgetTableFull, ...] = ()
budget_ids: tuple[str, ...] = ()
budget_resets: tuple[tuple[str, datetime], ...] = ()
endusers: tuple[_EndUserRow, ...] = ()
counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)
class _EndUserWalk:
"""Where the customer walk stands. ``cursor`` is None once it is done, and
``truncated`` says a failed page read cut it short of the tail."""
cursor: str | None = ""
invalidated: int = 0
truncated: bool = False
_ENDUSER_WALK_DONE: Final = _EndUserWalk(cursor=None)
@dataclass(frozen=True, slots=True)
class _BudgetCascadeCommitted:
cascade: _BudgetCascade
advanced: int
endusers: _EndUserWalk
@dataclass(frozen=True, slots=True)
@ -285,6 +303,8 @@ class _BudgetCascadeFailed:
_EMPTY_CASCADE: Final = _BudgetCascade()
_InvalidatedCache = Literal["spend counter", "user_api_key_cache"]
@dataclass(frozen=True, slots=True)
class _ChunkOutcome:
@ -416,10 +436,12 @@ _WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = (
)
def _budget_cascade_event_metadata(cascade: _BudgetCascade) -> dict[str, object]:
def _budget_cascade_event_metadata(
cascade: _BudgetCascade, endusers: _EndUserWalk = _ENDUSER_WALK_DONE
) -> dict[str, object]:
return {
"num_budgets_found": len(cascade.budgets),
"num_endusers_found": len(cascade.endusers),
"num_endusers_found": endusers.invalidated,
}
@ -593,6 +615,38 @@ class ResetBudgetJob:
e,
)
@staticmethod
async def _invalidate_caches(counter_keys: Sequence[str], cache_keys: Sequence[str]) -> None:
"""Batch twin of ``_invalidate_spend_counter`` and
``_invalidate_user_api_key_cache_entry``, after the commit like both:
one round trip per chunk where a tier's dependents are unbounded."""
await ResetBudgetJob._invalidate_cache("spend counter", counter_keys)
await ResetBudgetJob._invalidate_cache("user_api_key_cache", cache_keys)
@staticmethod
async def _invalidate_cache(cache: _InvalidatedCache, keys: Sequence[str]) -> None:
"""One cache's share of a batch, awaited separately so either failing
still leaves the other invalidated."""
if not keys:
return
try:
from litellm.proxy.proxy_server import spend_counter_cache, user_api_key_cache
match cache:
case "spend counter":
await spend_counter_cache.async_delete_cache_keys(keys)
case "user_api_key_cache":
await user_api_key_cache.async_delete_cache_keys(keys)
case _:
assert_never(cache)
except Exception as e:
verbose_proxy_logger.warning(
"Failed to invalidate %d %s entries: %s. Budgets may be over-enforced until they expire.",
len(keys),
cache,
e,
)
async def _fetch_linked_rows(
self,
table: SpendLinkedTable[_RowT],
@ -612,18 +666,57 @@ class ResetBudgetJob:
verbose_proxy_logger.warning("Failed to fetch %s for counter invalidation: %s", log_subject, e)
return ()
async def _collect_endusers_to_reset(self, budget_ids: Sequence[str]) -> tuple[_EndUserRow, ...]:
linked: Final[Sequence[_EndUserRow] | None] = await self._with_db_retry(
lambda: self.prisma_client.get_data(
table_name="enduser",
query_type="find_all",
budget_id_list=list(budget_ids),
),
reason="reset_budget_read_endusers_failure",
async def _invalidate_enduser_caches(self, budget_ids: Sequence[str]) -> _EndUserWalk:
"""Drop the cached spend of every customer the committed tier reset zeroed.
Paged like ``_reset_windows_for``, and capless for its reason too: the
customers on one tier are unbounded, and a cap cannot keep its position
across pod elections, so it would restart at the first customer forever.
"""
if not budget_ids:
return _ENDUSER_WALK_DONE
where: Final = _enduser_invalidation_where(budget_ids)
walk = _EndUserWalk()
while walk.cursor is not None:
walk = await self._invalidate_enduser_page(where=where, cursor=walk.cursor, reached=walk.invalidated)
return walk
async def _invalidate_enduser_page(self, where: Mapping[str, object], cursor: str, reached: int) -> _EndUserWalk:
"""Invalidate one page of customers and say where the walk goes next."""
try:
rows: Final = await self._fetch_enduser_page(where=where, cursor=cursor)
except Exception as e:
verbose_proxy_logger.warning(
"Failed to fetch end users for cache invalidation after %s customers (cursor %r): %s. "
"The customers past that page keep their cached spend until it expires.",
reached,
cursor,
e,
)
return _EndUserWalk(cursor=None, invalidated=reached, truncated=True)
if not rows:
return _EndUserWalk(cursor=None, invalidated=reached)
await self._invalidate_caches(
counter_keys=tuple(_enduser_counter_key(row) for row in rows),
cache_keys=tuple(key for row in rows for key in _enduser_cache_keys(row)),
)
walked: Final = reached + len(rows)
if len(rows) < RESET_BUDGET_JOB_BATCH_SIZE:
return _EndUserWalk(cursor=None, invalidated=walked)
return _EndUserWalk(cursor=rows[-1].user_id, invalidated=walked)
async def _fetch_enduser_page(self, where: Mapping[str, object], cursor: str) -> tuple[_EndUserRow, ...]:
"""One keyset page of customers, ordered by primary key so the cursor never repeats a row."""
return tuple(
await self._with_db_retry(
lambda: EndUserRepository(self.prisma_client).table.find_many(
where={**where, "user_id": {"gt": cursor}}, # mutable-ok: prisma where filter must be a dict
order={"user_id": "asc"}, # mutable-ok: prisma order filter must be a dict
take=RESET_BUDGET_JOB_BATCH_SIZE,
),
reason="reset_budget_read_endusers_failure",
)
)
if litellm.max_end_user_budget_id is None or litellm.max_end_user_budget_id not in budget_ids:
return tuple(linked or ())
return (*(linked or ()), *await self._get_endusers_with_no_budget_id())
async def _collect_budget_cascade(self, budgets_to_reset: Sequence[LiteLLM_BudgetTableFull]) -> _BudgetCascade:
"""Resolve every row the expiring budget tiers gate, before any write.
@ -670,7 +763,6 @@ class ResetBudgetJob:
if _rollover_enabled()
else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
)
endusers: Final[tuple[_EndUserRow, ...]] = await self._collect_endusers_to_reset(budget_ids)
return _BudgetCascade(
budgets=tuple(budgets_to_reset),
budget_ids=budget_ids,
@ -682,7 +774,6 @@ class ResetBudgetJob:
for b in budgets_to_reset
if b.budget_id is not None and b.budget_duration is not None
),
endusers=endusers,
counter_resets=(
*(
(_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps))
@ -695,7 +786,6 @@ class ResetBudgetJob:
(_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps))
for row in model_access_groups
),
*((_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) for row in endusers),
),
rollover_caps=rollover_caps,
cache_keys=(
@ -704,7 +794,6 @@ class ResetBudgetJob:
*(key for row in orgs for key in _org_cache_keys(row)),
*(key for row in tags for key in _tag_cache_keys(row)),
*(key for row in model_access_groups for key in _model_access_group_cache_keys(row)),
*(key for row in endusers for key in _enduser_cache_keys(row)),
),
)
@ -736,10 +825,10 @@ class ResetBudgetJob:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
for counter_key, _ in cascade.counter_resets:
await self._invalidate_spend_counter(counter_key)
for cache_key in cascade.cache_keys:
await self._invalidate_user_api_key_cache_entry(cache_key)
await self._invalidate_caches(
counter_keys=tuple(counter_key for counter_key, _ in cascade.counter_resets),
cache_keys=cascade.cache_keys,
)
async def _reset_expired_budget_cascade(self) -> _BudgetCascadeCommitted | _BudgetCascadeFailed:
now: Final = datetime.now(timezone.utc)
@ -769,6 +858,7 @@ class ResetBudgetJob:
(reset_at for _, reset_at in cascade.budget_resets),
cutoff=datetime.now(timezone.utc),
),
endusers=await self._invalidate_enduser_caches(cascade.budget_ids),
)
async def reset_budget_for_litellm_budget_table(self) -> None:
@ -788,7 +878,7 @@ class ResetBudgetJob:
end_time: Final = time.time()
match outcome:
case _BudgetCascadeCommitted(cascade=cascade, advanced=advanced):
case _BudgetCascadeCommitted() as committed:
asyncio.create_task(
self.proxy_logging_obj.service_logging_obj.async_service_success_hook(
service=ServiceTypes.RESET_BUDGET_JOB,
@ -797,13 +887,14 @@ class ResetBudgetJob:
start_time=start_time,
end_time=end_time,
event_metadata={
**_budget_cascade_event_metadata(cascade),
"num_endusers_updated": len(cascade.endusers),
**_budget_cascade_event_metadata(committed.cascade, committed.endusers),
"num_endusers_updated": committed.endusers.invalidated,
"num_endusers_failed": 0,
"enduser_invalidation_truncated": committed.endusers.truncated,
},
)
)
return _ChunkOutcome(fetched=len(cascade.budgets), advanced=advanced)
return _ChunkOutcome(fetched=len(committed.cascade.budgets), advanced=committed.advanced)
case _BudgetCascadeFailed(cascade=cascade, error=error):
verbose_proxy_logger.exception(
"Failed to reset the budget table cascade (team member, enduser, org, tag and model access "
@ -827,27 +918,6 @@ class ResetBudgetJob:
case _:
assert_never(outcome)
async def _get_endusers_with_no_budget_id(
self,
) -> list[LiteLLM_EndUserTable]:
"""
Fetch end users that have no explicit budget_id set (NULL) and have
accumulated spend > 0. These are implicitly-created end users that
rely on the default budget (litellm.max_end_user_budget_id) applied
in-memory during auth checks.
"""
table: Final = EndUserRepository(self.prisma_client).table
rows: Final = await self._with_db_retry(
lambda: table.find_many(
where={
"budget_id": None,
"spend": {"gt": 0},
},
),
reason="reset_budget_read_endusers_without_budget_id_failure",
)
return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows]
async def _write_key_reset_updates(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
"""
Write per-row {spend, budget_reset_at} updates for keys.

19
litellm/proxy/common_utils/user_api_key_cache.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import re
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast, overload
@ -221,6 +222,24 @@ class UserApiKeyCache(DualCache):
return
await super().async_delete_cache(key)
async def async_delete_cache_keys(self, keys: Sequence[str]) -> None:
"""Batch twin of ``async_delete_cache``, partitioned like
``async_set_cache_pipeline``.
Both partitions are cleared even when one raises, because a caller
batching these has already committed the rows they cache.
"""
key_object_keys: Final = tuple(key for key in keys if is_user_key_cache_key(key))
other_keys: Final = tuple(key for key in keys if not is_user_key_cache_key(key))
outcomes: Final = await asyncio.gather(
self.key_object_cache.async_delete_cache_keys(key_object_keys),
super().async_delete_cache_keys(other_keys),
return_exceptions=True,
)
failed: Final = tuple(outcome for outcome in outcomes if isinstance(outcome, BaseException))
if failed:
raise failed[0]
def flush_cache(self) -> None:
super().flush_cache()
self.key_object_cache.in_memory_cache.flush_cache()

1
litellm/proxy/schema.prisma
View file

@ -678,6 +678,7 @@ model LiteLLM_SpendLogs {
@@index([end_user])
@@index([session_id])
@@index([litellm_call_id])
@@index([api_key, startTime])
}
model LiteLLM_BudgetWindowSpend {

13
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -485,10 +485,13 @@ def get_logging_payload(
or None
)
custom_llm_provider: Final = logged_provider or _model_group_provider(_model_group, llm_router)
raw_model: Final = cast(str, kwargs.get("model") or "")
resolved_model: Final = (
standard_logging_payload.get("model") if standard_logging_payload is not None else None
) or reconstruct_model_name(raw_model, logged_provider, metadata or {})
requested_model: Final = cast(object, kwargs.get("model"))
raw_model: Final = requested_model if isinstance(requested_model, str) else ""
model_is_malformed: Final = requested_model is not None and not isinstance(requested_model, str)
logged_model: Final = standard_logging_payload.get("model") if standard_logging_payload is not None else None
resolved_model: Final = (logged_model if isinstance(logged_model, str) else None) or reconstruct_model_name(
raw_model, logged_provider, metadata or {}
)
failed_with_prompt_shaped_model: Final = (
_get_status_for_spend_log(metadata=metadata) == "failure"
and not _model_group
@ -496,7 +499,7 @@ def get_logging_payload(
)
model_name: Final = (
UNKNOWN_MODEL_SPEND_LOG_MODEL
if rejected_as_unknown_model or failed_with_prompt_shaped_model
if rejected_as_unknown_model or failed_with_prompt_shaped_model or model_is_malformed
else resolved_model
)
litellm_call_id: Final = cast(

9
litellm/utils.py
View file

@ -2916,6 +2916,15 @@ def supports_none_reasoning_effort(model: str, custom_llm_provider: str | None =
return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_none_reasoning_effort")
def supports_mid_conversation_system(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model accepts a system role message after the leading system block and return a boolean value.
"""
return _supports_factory(
model=model, custom_llm_provider=custom_llm_provider, key="supports_mid_conversation_system"
)
def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model supports native structured outputs and return a boolean value.

135
model_prices_and_context_window.json
View file

@ -7605,7 +7605,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7733,7 +7733,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7887,7 +7887,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
@ -7956,7 +7956,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8856,7 +8856,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/us/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -8955,7 +8955,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/eu/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -9054,7 +9054,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
@ -10987,14 +10987,14 @@
"supports_vision": true
},
"azure_ai/FW-Kimi-K3": {
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.5e-05,
"reasoning_effort_levels": [
"low",
"high",
@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -45245,7 +45245,7 @@
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-20b": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@ -45482,6 +45482,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45503,7 +45504,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://api.together.xyz/v1/models",
"source": "https://api.together.ai/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
@ -45528,6 +45529,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 1.3e-07,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45552,7 +45554,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 3.9e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@ -45567,7 +45569,7 @@
"supports_vision": true
},
"together_ai/intfloat/multilingual-e5-large-instruct": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 2e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 514,
@ -45680,7 +45682,7 @@
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
@ -50573,7 +50575,7 @@
"wandb/openai/gpt-oss-120b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.7e-07,
@ -50584,7 +50586,7 @@
"wandb/openai/gpt-oss-20b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.3e-07,
@ -50593,6 +50595,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -50603,6 +50606,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"deprecation_date": "2026-08-04",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50612,6 +50616,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"deprecation_date": "2026-08-25",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50622,6 +50627,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"deprecation_date": "2026-08-04",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
@ -50632,6 +50638,7 @@
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"deprecation_date": "2026-03-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -50656,6 +50663,7 @@
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"deprecation_date": "2026-08-25",
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
@ -50670,7 +50678,7 @@
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"max_output_tokens": 128000,
"input_cost_per_token": 2.2e-07,
"output_cost_per_token": 2.2e-07,
@ -50690,6 +50698,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 161000,
"max_input_tokens": 161000,
@ -50700,6 +50709,7 @@
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"deprecation_date": "2026-03-04",
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
@ -50719,6 +50729,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"deprecation_date": "2026-04-21",
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
@ -50728,6 +50739,7 @@
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"deprecation_date": "2026-08-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -56692,7 +56704,8 @@
"supports_function_calling": true,
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true
"gemini_audio_only_live": true,
"supports_response_schema": false
},
"gemini-3.8-live-extended-thinking": {
"input_cost_per_audio_token": 3e-06,
@ -56726,7 +56739,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -60970,10 +60984,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -60983,7 +60998,7 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
@ -60996,10 +61011,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Pro": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.15e-06,
"output_cost_per_token": 2.55e-06,
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61009,7 +61025,7 @@
"wandb/google/gemma-4-31B-it": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3.4e-07,
"litellm_provider": "wandb",
@ -61018,8 +61034,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/ibm-granite/granite-4.1-8b": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61028,8 +61045,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61038,8 +61056,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-3.1-70B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"input_cost_per_token": 8e-07,
"output_cost_per_token": 8e-07,
"litellm_provider": "wandb",
@ -61050,7 +61069,7 @@
"wandb/MiniMaxAI/MiniMax-M3": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 9.6e-07,
"cache_read_input_token_cost": 5e-08,
@ -61063,7 +61082,7 @@
"wandb/moonshotai/Kimi-K2.7-Code": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 7.1e-07,
"output_cost_per_token": 3.5e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61076,7 +61095,7 @@
"wandb/moonshotai/Kimi-K2.6": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3.41e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61089,10 +61108,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 262000,
"input_cost_per_token": 7e-08,
"output_cost_per_token": 2e-07,
"cache_read_input_token_cost": 4e-08,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61102,10 +61121,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 7.5e-07,
"output_cost_per_token": 2.75e-06,
"cache_read_input_token_cost": 1.5e-07,
"max_input_tokens": 262000,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 1e-07,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61113,8 +61132,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/OpenPipe/Qwen3-14B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_input_tokens": 32800,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
@ -61125,7 +61145,7 @@
"wandb/Qwen/Qwen3.8-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61138,7 +61158,7 @@
"wandb/Qwen/Qwen3.6-35B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61149,10 +61169,11 @@
"wandb/Qwen/Qwen3.6-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3.6e-06,
"cache_read_input_token_cost": 1.2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61160,9 +61181,10 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3.5-35B-A3B": {
"deprecation_date": "2026-10-05",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61171,8 +61193,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"deprecation_date": "2026-10-05",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "wandb",
@ -61187,6 +61210,7 @@
"input_cost_per_token": 1.31e-06,
"output_cost_per_token": 3.96e-06,
"cache_read_input_token_cost": 4.4e-08,
"max_input_tokens": 1049000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
@ -61197,13 +61221,14 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 131000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-5.2": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 1049000,
"input_cost_per_token": 7.6e-07,
"output_cost_per_token": 2.42e-06,
"cache_read_input_token_cost": 1.4e-07,
@ -62866,7 +62891,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -69158,5 +69183,19 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_vision": true
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "wandb",
"max_input_tokens": 1049000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://wandb.ai/site/pricing/tokens/",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
}
}

7
pyproject.toml
View file

@ -67,7 +67,7 @@ proxy = [
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=1.28.1,<2.0",
"litellm-proxy-extras==0.4.98",
"litellm-proxy-extras==0.4.99",
"litellm-enterprise==0.1.68",
"RestrictedPython>=8.5,<9.0",
"rich>=13.9.4,<14.0",
@ -143,8 +143,9 @@ bedrock-realtime = [
# InvokeModelWithBidirectionalStream API, which boto3 cannot do. This
# experimental AWS SDK (with its smithy-* deps, pulled transitively)
# provides the bidirectional stream; imported lazily in the realtime
# handler so litellm core stays usable without it.
"aws-sdk-bedrock-runtime>=0.7.0,<0.8.0; python_version >= '3.12'",
# handler so litellm core stays usable without it. The awscrt extra is
# required: the SDK's default aiohttp transport has no duplex streaming.
"aws-sdk-bedrock-runtime[awscrt]>=0.10.0,<0.12.0; python_version >= '3.12'",
]
proxy-runtime = [
# Historically bundled in the proxy Docker images via requirements.txt.

1
schema.prisma
View file

@ -678,6 +678,7 @@ model LiteLLM_SpendLogs {
@@index([end_user])
@@index([session_id])
@@index([litellm_call_id])
@@index([api_key, startTime])
}
model LiteLLM_BudgetWindowSpend {

3
tests/integration/_support/client.py
View file

@ -10,9 +10,10 @@ from hashlib import sha256
from typing import Final, TypeVar
import httpx
from integration._support.database import read_rows
from pydantic import JsonValue, TypeAdapter
from tests.integration._support.database import read_rows
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
T = TypeVar("T")

2
tests/integration/_support/generation.py
View file

@ -6,7 +6,7 @@ from contextlib import contextmanager
import httpx
from hypothesis import Phase, settings
from integration._support.client import Gateway
from tests.integration._support.client import Gateway
LIFECYCLE_SETTINGS: Final = settings(
max_examples=20,

6
tests/integration/authorization/test_warmed_policy.py
View file

@ -10,9 +10,9 @@ from pydantic import JsonValue
from hypothesis import strategies as st
from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
from tests.integration._support.client import Gateway, eventually, object_value
from tests.integration._support.database import read_rows
from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
def assert_serving(gateway: Gateway, model: str, key: str, status: int, error_type: str = "auth_error") -> None:

4
tests/integration/configuration/test_effective_settings.py
View file

@ -4,8 +4,8 @@ from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, object_value, string_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, object_value, string_value
from tests.integration._support.database import read_rows
def model_identity(gateway: Gateway, alias: str) -> str:

6
tests/integration/conftest.py
View file

@ -12,9 +12,9 @@ import pytest
import httpx
from redis import Redis
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.manifest import OWNED_DIRECTORIES, contracts
from integration._support.generation import LIFECYCLE_SETTINGS
from tests.integration._support.client import Gateway, eventually, gateway_from_environment
from tests.integration._support.manifest import OWNED_DIRECTORIES, contracts
from tests.integration._support.generation import LIFECYCLE_SETTINGS
COLLECTED: Final = pytest.StashKey[tuple[str, ...]]()
REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]()

4
tests/integration/management/test_key_updates.py
View file

@ -3,8 +3,8 @@ from hashlib import sha256
import pytest
from integration._support.client import Gateway, object_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, object_value
from tests.integration._support.database import read_rows
@pytest.mark.covers("mgmt.key.update.preserves_independent_fields")

7
tests/integration/management/test_partial_update_sequences.py
View file

@ -5,11 +5,12 @@ from typing import Final
import pytest
from hypothesis import strategies as st
from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test
from integration._support.client import Gateway, object_value
from integration._support.database import read_rows
from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
from pydantic import JsonValue
from tests.integration._support.client import Gateway, object_value
from tests.integration._support.database import read_rows
from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
def _key_rows(digest: str) -> list[dict[str, JsonValue]]:
return read_rows(

4
tests/integration/pricing/test_configured_prices.py
View file

@ -6,8 +6,8 @@ import uuid
import pytest
import yaml
from integration._support.client import Gateway, eventually, object_value, string_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, eventually, object_value, string_value
from tests.integration._support.database import read_rows
@pytest.mark.covers("quota_management.spend_tracking.custom_price.matches_input_rates")

2
tests/integration/providers/test_request_boundary.py
View file

@ -3,7 +3,7 @@ from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, JSON_OBJECT, object_value
from tests.integration._support.client import Gateway, JSON_OBJECT, object_value
@pytest.mark.covers("other.provider_wire.internal_parameters_filtered")

22
tests/litellm_utils_tests/test_proxy_budget_reset.py
View file

@ -102,21 +102,24 @@ def _wire_batcher_for_test(prisma_client, fail_commit=False):
return batch_calls
def _wire_cascade_reads_for_test(prisma_client):
def _wire_cascade_reads_for_test(prisma_client, endusers=()):
"""
The budget tier's cascade reads the rows it is about to zero, so their
spend counters can be invalidated after the commit. Give each of those
tables an awaitable find_many so the reads resolve instead of falling into
the job's warn-and-continue path.
End users are read by the post-commit invalidation walk rather than by
``get_data``, so callers that care about customers pass them here.
"""
for table in (
"litellm_teammembership",
"litellm_verificationtoken",
"litellm_organizationtable",
"litellm_tagtable",
"litellm_endusertable",
):
getattr(prisma_client.db, table).find_many = AsyncMock(return_value=[])
prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=list(endusers))
@pytest.mark.asyncio
@ -556,7 +559,7 @@ async def test_reset_budget_continues_other_categories_on_failure():
**{u["user_id"]: u["spend"] for u in [user2]},
**{t["team_id"]: t["spend"] for t in [team1, team2]},
}
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=[enduser1])
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -607,7 +610,10 @@ async def test_reset_budget_continues_other_categories_on_failure():
called_tables = {
call.kwargs.get("table_name") for call in prisma_client.get_data.await_args_list
}
assert called_tables == {"key", "user", "team", "budget", "enduser"}
assert called_tables == {"key", "user", "team", "budget"}
# Customers are not part of that set: the cascade zeroes them by budget link
# and reads them only afterwards, to invalidate their cached spend.
prisma_client.db.litellm_endusertable.find_many.assert_awaited()
# Every category writes through the batch path now, so update_data is unused.
prisma_client.update_data.assert_not_awaited()
@ -1029,7 +1035,7 @@ async def test_service_logger_endusers_success():
prisma_client.get_data = AsyncMock(side_effect=fake_get_data)
prisma_client.update_data = AsyncMock()
batch_calls = _wire_batcher_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=endusers)
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -1094,7 +1100,7 @@ async def test_service_logger_endusers_failure():
prisma_client.get_data = AsyncMock(side_effect=fake_get_data)
prisma_client.update_data = AsyncMock()
_wire_batcher_for_test(prisma_client, fail_commit=True)
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=endusers)
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -1121,7 +1127,9 @@ async def test_service_logger_endusers_failure():
) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args
event_metadata = kwargs.get("event_metadata", {})
assert event_metadata.get("num_budgets_found") == len(budgets)
assert event_metadata.get("num_endusers_found") == len(endusers)
# Customers are read by the post-commit invalidation walk, which a failed
# commit never reaches, so a failure reports none touched.
assert event_metadata.get("num_endusers_found") == 0
assert "endusers_found" not in event_metadata
assert "budgets_found" not in event_metadata
proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()

8
tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py
View file

@ -20,7 +20,9 @@ import pytest
IMAGE: Final = os.getenv("LITELLM_IMAGE")
NON_ROOT_UID: Final = "12345:0"
IMPORT_PROBE: Final = "import aws_sdk_bedrock_runtime, smithy_aws_core; print('bedrock-realtime ok')"
IMPORT_PROBE: Final = (
"import aws_sdk_bedrock_runtime, smithy_aws_core, smithy_http.aio.crt; print('bedrock-realtime ok')"
)
pytestmark = [
pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"),
@ -52,7 +54,7 @@ def test_image_imports_bedrock_realtime_sdk():
)
assert probe.returncode == 0 and "bedrock-realtime ok" in probe.stdout, (
f"{IMAGE} cannot import aws_sdk_bedrock_runtime as uid {NON_ROOT_UID}, so Bedrock Nova Sonic "
"/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'. Is `--extra bedrock-realtime` "
f"{IMAGE} cannot import aws_sdk_bedrock_runtime with its awscrt transport as uid {NON_ROOT_UID}, so "
"Bedrock Nova Sonic /v1/realtime sessions fail at SDK import. Is `--extra bedrock-realtime` "
f"passed to every `uv sync` in its Dockerfile?\nstdout:\n{probe.stdout}\nstderr:\n{probe.stderr}"
)

32
tests/test_litellm/caching/test_dual_cache.py
View file

@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.constants import DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache, _redis_circuit_breaker_guard, _redis_circuit_breaker_guard_sync
@ -759,3 +760,34 @@ async def test_redis_timeouts_falling_back_to_memory_log_once_per_interval(caplo
" (199 more Redis timeouts since the previous Redis timeout line were logged at DEBUG)",
)
]
@pytest.mark.asyncio
async def test_async_delete_cache_keys_drops_memory_and_chunks_redis():
"""Batch delete clears both layers, and chunks Redis so one caller's large
key list cannot become a single oversized DELETE command."""
redis_cache = MagicMock(spec=RedisCache)
redis_cache.delete_cache_keys = AsyncMock()
dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
keys = [f"key-{i}" for i in range(DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE + 7)]
for key in keys:
dual_cache.in_memory_cache.set_cache(key=key, value=1)
await dual_cache.async_delete_cache_keys(keys)
assert all(dual_cache.in_memory_cache.get_cache(key=key) is None for key in keys)
sent = [call.args[0] for call in redis_cache.delete_cache_keys.await_args_list]
assert [len(chunk) for chunk in sent] == [DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE, 7]
assert [key for chunk in sent for key in chunk] == keys
@pytest.mark.asyncio
async def test_async_delete_cache_keys_on_empty_list_touches_no_backend():
"""An empty page must not reach Redis: DELETE with no arguments is an error."""
redis_cache = MagicMock(spec=RedisCache)
redis_cache.delete_cache_keys = AsyncMock()
dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
await dual_cache.async_delete_cache_keys([])
redis_cache.delete_cache_keys.assert_not_awaited()

162
tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
View file

@ -23,6 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
create_tool_name_mapping,
truncate_tool_name,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
CONVERTED_SYSTEM_NOTE,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.anthropic import (
AnthopicMessagesAssistantMessageParam,
@ -563,10 +566,19 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
@pytest.mark.parametrize(
("system_content", "expected_content"),
[
("Use the corrected result.", "Use the corrected result."),
(
"Use the corrected result.",
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[{"type": "text", "text": "Use the corrected result."}],
[{"type": "text", "text": "Use the corrected result."}],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[
@ -576,7 +588,11 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
},
{"type": "text", "text": "Use the corrected result."},
],
[{"type": "text", "text": "Use the corrected result."}],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[
@ -584,13 +600,14 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
{"type": "text", "text": "Second correction."},
],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "First correction."},
{"type": "text", "text": "Second correction."},
],
),
],
)
def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correction(
def test_translate_anthropic_messages_to_openai_converts_midturn_system_correction(
system_content: object,
expected_content: object,
):
@ -646,7 +663,7 @@ def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correct
"tool_call_id": "toolu_01234",
"content": "Rainy, 55°F",
},
{"role": "system", "content": expected_content},
{"role": "user", "content": expected_content},
{"role": "user", "content": "Continue."},
]
@ -752,8 +769,8 @@ def test_translate_anthropic_messages_to_openai_drops_empty_midturn_system(
def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
"""
Request level: the trusted top-level prompt is hoisted to index 0 exactly once and the
in-sequence correction keeps its own position and `role: "system"` -- no duplication of
either, and no reordering of the surrounding turns.
in-sequence correction keeps its own position as a user turn prefixed with the operator
note -- no duplication of either, and no reordering of the surrounding turns.
"""
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={
@ -773,11 +790,140 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
{"role": "system", "content": "Trusted top-level prompt."},
{"role": "user", "content": "First question."},
{"role": "assistant", "content": "First answer.", "thinking_blocks": None},
{"role": "system", "content": "Use the corrected result."},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
},
{"role": "user", "content": "Continue."},
]
_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST: Final = {
"max_tokens": 128,
"system": [{"type": "text", "text": "You are Claude Code."}],
"messages": [
{"role": "user", "content": "say hi"},
{
"role": "system",
"content": [{"type": "text", "text": "<system-reminder>Keep answers to one sentence.</system-reminder>"}],
},
{"role": "assistant", "content": "Hi."},
{"role": "user", "content": "say bye"},
],
}
@pytest.mark.parametrize("custom_llm_provider", [None, "hosted_vllm"])
def test_translate_anthropic_to_openai_converts_claude_code_midturn_system_turn(custom_llm_provider: str | None):
"""
Claude Code appends a system-role harness reminder after the user turn. On a chat-completions
target that does not declare ``supports_mid_conversation_system`` (a self-hosted model the cost
map knows nothing about) the outbound request must have exactly one system message, at index 0,
and the converted turn must carry the operator note first.
"""
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
custom_llm_provider=custom_llm_provider,
)
roles = [m["role"] for m in openai_request["messages"]]
assert roles == ["system", "user", "user", "assistant", "user"]
converted = openai_request["messages"][2]
assert converted["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
assert converted["content"][1]["text"] == "<system-reminder>Keep answers to one sentence.</system-reminder>"
def test_translate_anthropic_to_openai_keeps_midturn_system_when_target_declares_support(monkeypatch):
"""
A chat-completions target flagged ``supports_mid_conversation_system`` in the cost map accepts
the role anywhere, so the harness reminder is forwarded in place with its role and content
untouched, the same rule the native Anthropic Messages path applies.
"""
model: Final = "system-role-anywhere-chat-model"
monkeypatch.setitem(
litellm.model_cost,
model,
{"litellm_provider": "openai", "mode": "chat", "supports_mid_conversation_system": True},
)
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={"model": model, **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
custom_llm_provider="openai",
)
assert openai_request["messages"] == [
{"role": "system", "content": [{"type": "text", "text": "You are Claude Code."}]},
{"role": "user", "content": "say hi"},
{
"role": "system",
"content": [{"type": "text", "text": "<system-reminder>Keep answers to one sentence.</system-reminder>"}],
},
{"role": "assistant", "content": "Hi.", "thinking_blocks": None},
{"role": "user", "content": "say bye"},
]
def test_translate_anthropic_to_openai_moves_midturn_system_after_tool_result():
"""
A system entry wedged between an assistant tool_use turn and its tool_result turn is
emitted after the role: "tool" message, so the tool call stays paired with its result.
"""
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=[
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "toolu_01234",
"name": "get_weather",
"input": {"location": "Boston"},
}
],
},
{"role": "system", "content": "Use the corrected result."},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_01234",
"content": "Rainy, 55°F",
}
],
},
],
model="claude-3-5-sonnet-20240620",
)
assert [m["role"] for m in result] == ["assistant", "tool", "user"]
assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
def test_translate_anthropic_messages_to_openai_converts_string_midturn_system():
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Keep it short."},
],
model="claude-3-5-sonnet-20240620",
)
assert result == [
{"role": "user", "content": "hi"},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
},
]
def _claude_code_user_id(session_id: str) -> str:
return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id})

89
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py Normal file
View file

@ -0,0 +1,89 @@
from collections import Counter
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
CONVERTED_SYSTEM_NOTE,
convert_mid_conversation_system_turns,
)
class RoleReadCountingMessage(dict):
def __init__(self, role: str, content: object, reads: Counter):
super().__init__(role=role, content=content)
self.reads = reads
def get(self, key, default=None):
self.reads[key] += 1
return super().get(key, default)
def test_convert_mid_conversation_system_turns_converts_system_to_user_in_place():
result = convert_mid_conversation_system_turns(
[
{"role": "user", "content": "hi"},
{"role": "system", "content": [{"type": "text", "text": "Keep it short."}]},
{"role": "assistant", "content": "Hi."},
]
)
assert result == (
{"role": "user", "content": "hi"},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
},
{"role": "assistant", "content": "Hi."},
)
def test_convert_mid_conversation_system_turns_wraps_string_content():
result = convert_mid_conversation_system_turns(
[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Keep it short."},
]
)
assert result[1] == {
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
}
def test_convert_mid_conversation_system_turns_moves_system_after_tool_result():
assistant_tool_use = {
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}],
}
wedged_system = {"role": "system", "content": "Use the corrected result."}
tool_result = {
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}],
}
result = convert_mid_conversation_system_turns([assistant_tool_use, wedged_system, tool_result])
assert result[0] is assistant_tool_use
assert result[1] is tool_result
assert result[2]["role"] == "user"
assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
def test_convert_mid_conversation_system_turns_reads_each_role_a_bounded_number_of_times():
reads = Counter()
system_run = [RoleReadCountingMessage("system", f"reminder {i}", reads) for i in range(2_000)]
tool_result = RoleReadCountingMessage(
"user", [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}], reads
)
messages = [RoleReadCountingMessage("user", "hi", reads), *system_run, tool_result]
result = convert_mid_conversation_system_turns(messages)
assert reads["role"] <= 3 * len(messages)
assert result[1] is tool_result
assert [m["content"][1]["text"] for m in result[2:]] == [m["content"] for m in system_run]

17
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -23,6 +23,9 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
as_system_content_blocks,
)
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
AmazonAnthropicClaudeMessagesStreamDecoder,
@ -2533,20 +2536,16 @@ def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_
def test_as_system_content_blocks_handles_each_shape():
"""``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
"""``as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
a string -> a single text block, a list -> a shallow copy, and any other value
(e.g. a bare content-block dict) -> wrapped in a single-element list."""
block = {"type": "text", "text": "x"}
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(None) == []
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks("hello") == [
{"type": "text", "text": "hello"}
]
assert as_system_content_blocks(None) == []
assert as_system_content_blocks("hello") == [{"type": "text", "text": "hello"}]
blocks = [block]
out = AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(blocks)
out = as_system_content_blocks(blocks)
assert out == blocks and out is not blocks
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(block) == [
block
]
assert as_system_content_blocks(block) == [block]
@pytest.mark.parametrize(

255
tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
View file

@ -8,7 +8,11 @@ from unittest.mock import MagicMock
import pytest
import litellm
from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY
from litellm.constants import (
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
@ -207,7 +211,19 @@ class ScriptedBedrockStream:
return (None, self._receiver)
class FakeAWSCredentialsIdentity:
def __init__(self, access_key_id, secret_access_key, session_token=None):
self.access_key_id = access_key_id
self.secret_access_key = secret_access_key
self.session_token = session_token
class FakeStaticCredentialsResolver:
def __init__(self, identity=None):
self.identity = identity
class FakeAWSCRTHTTPClient:
pass
@ -227,48 +243,32 @@ class StubCredentialsBedrockRealtime(BedrockRealtime):
return SimpleNamespace(get_frozen_credentials=lambda: self.frozen_credentials)
@pytest.fixture
def stub_aws_sdk_client(monkeypatch):
captured = {}
class FakeOperationInput:
def __init__(self, model_id):
self.model_id = model_id
class CapturingConfig:
def __init__(self, **kwargs):
captured["config_kwargs"] = kwargs
self.kwargs = kwargs
class FakeOperationInput:
def __init__(self, model_id):
self.model_id = model_id
class FakeBedrockRuntimeClient:
def __init__(self, config):
captured["client_config"] = config
async def invoke_model_with_bidirectional_stream(self, operation_input):
captured["operation_input"] = operation_input
if captured.get("streams"):
stream = captured["streams"].pop(0)
if isinstance(stream, Exception):
raise stream
return stream
return ScriptedBedrockStream(captured.get("scripted_payloads", []))
def _install_fake_sdk_modules(monkeypatch, client_module, config_module):
"""Wire fake aws_sdk_bedrock_runtime / smithy packages into sys.modules for the handler's lazy imports."""
package = types.ModuleType("aws_sdk_bedrock_runtime")
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.BedrockRuntimeClient = FakeBedrockRuntimeClient
client_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.Config = CapturingConfig
models_module = types.ModuleType("aws_sdk_bedrock_runtime.models")
models_module.BidirectionalInputPayloadPart = FakePayloadPart
models_module.InvokeModelWithBidirectionalStreamInputChunk = FakeInputChunk
models_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput
package.client = client_module
package.config = config_module
package.models = models_module
smithy_package = types.ModuleType("smithy_aws_core")
identity_module = types.ModuleType("smithy_aws_core.identity")
identity_module.AWSCredentialsIdentity = FakeAWSCredentialsIdentity
identity_module.StaticCredentialsResolver = FakeStaticCredentialsResolver
smithy_package.identity = identity_module
smithy_http_package = types.ModuleType("smithy_http")
smithy_http_aio = types.ModuleType("smithy_http.aio")
crt_module = types.ModuleType("smithy_http.aio.crt")
crt_module.AWSCRTHTTPClient = FakeAWSCRTHTTPClient
smithy_http_aio.crt = crt_module
smithy_http_package.aio = smithy_http_aio
stubbed_modules = {
"aws_sdk_bedrock_runtime": package,
@ -277,10 +277,56 @@ def stub_aws_sdk_client(monkeypatch):
"aws_sdk_bedrock_runtime.models": models_module,
"smithy_aws_core": smithy_package,
"smithy_aws_core.identity": identity_module,
"smithy_http": smithy_http_package,
"smithy_http.aio": smithy_http_aio,
"smithy_http.aio.crt": crt_module,
}
for module_name, module in stubbed_modules.items():
monkeypatch.setitem(sys.modules, module_name, module)
@pytest.fixture
def stub_aws_sdk_client(monkeypatch):
"""Fake of the aws-sdk-bedrock-runtime 0.10/0.11 surface: async config resolve, async client with close()"""
captured = {}
class FakeAsyncBedrockRuntimeConfig:
def __init__(self, kwargs):
self.kwargs = kwargs
@classmethod
async def resolve(cls, **kwargs):
captured["config_kwargs"] = kwargs
return cls(kwargs)
class FakeAsyncBedrockRuntimeClient:
def __init__(self, config):
captured["client_config"] = config
captured["client_closed"] = False
async def invoke_model_with_bidirectional_stream(self, operation_input):
captured["operation_input"] = operation_input
if captured.get("streams"):
stream = captured["streams"].pop(0)
if isinstance(stream, Exception):
raise stream
captured["open_stream"] = stream
return stream
stream = ScriptedBedrockStream(captured.get("scripted_payloads", []))
captured["open_stream"] = stream
return stream
async def close(self):
open_stream = captured.get("open_stream")
captured["input_closed_before_client_close"] = open_stream is None or open_stream.input_stream.closed
captured["client_closed"] = True
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.AsyncBedrockRuntimeClient = FakeAsyncBedrockRuntimeClient
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.AsyncBedrockRuntimeConfig = FakeAsyncBedrockRuntimeConfig
_install_fake_sdk_modules(monkeypatch, client_module, config_module)
for env_var in (
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
@ -764,15 +810,33 @@ class TestBedrockRealtimeAwsAuth:
)
config_kwargs = stub_aws_sdk_client["config_kwargs"]
assert config_kwargs["aws_access_key_id"] == "litellm-params-access-key"
assert config_kwargs["aws_secret_access_key"] == "litellm-params-secret-key"
assert config_kwargs["aws_session_token"] == "litellm-params-session-token"
assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver)
resolver = config_kwargs["aws_credentials_identity_resolver"]
assert isinstance(resolver, FakeStaticCredentialsResolver)
assert resolver.identity.access_key_id == "litellm-params-access-key"
assert resolver.identity.secret_access_key == "litellm-params-secret-key"
assert resolver.identity.session_token == "litellm-params-session-token"
assert config_kwargs["region"] == "us-east-1"
assert config_kwargs["endpoint_uri"] == "https://bedrock-runtime.us-east-1.amazonaws.com"
assert isinstance(config_kwargs["transport"], FakeAWSCRTHTTPClient)
assert stub_aws_sdk_client["client_config"].kwargs is config_kwargs
assert stub_aws_sdk_client["operation_input"].model_id == "amazon.nova-sonic-v1:0"
assert websocket.closed
@pytest.mark.asyncio
async def test_api_base_overrides_default_endpoint(self, stub_aws_sdk_client):
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=RealtimeClientWS(),
logging_obj=FakeLogging(),
aws_region_name="us-east-1",
aws_access_key_id="k",
aws_secret_access_key="s",
api_base="https://vpce-bedrock.example.internal",
aws_bedrock_runtime_endpoint="https://ignored.example.internal",
)
assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://vpce-bedrock.example.internal"
@pytest.mark.asyncio
async def test_role_assumption_params_forwarded_to_get_credentials(self, stub_aws_sdk_client):
handler = StubCredentialsBedrockRealtime(
@ -805,11 +869,11 @@ class TestBedrockRealtimeAwsAuth:
"aws_sts_endpoint": None,
"aws_external_id": "realtime-external-id",
}
config_kwargs = stub_aws_sdk_client["config_kwargs"]
assert config_kwargs["aws_access_key_id"] == "assumed-access-key"
assert config_kwargs["aws_secret_access_key"] == "assumed-secret-key"
assert config_kwargs["aws_session_token"] == "assumed-session-token"
assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver)
resolver = stub_aws_sdk_client["config_kwargs"]["aws_credentials_identity_resolver"]
assert isinstance(resolver, FakeStaticCredentialsResolver)
assert resolver.identity.access_key_id == "assumed-access-key"
assert resolver.identity.secret_access_key == "assumed-secret-key"
assert resolver.identity.session_token == "assumed-session-token"
@pytest.mark.asyncio
async def test_unresolvable_credentials_raise_clear_auth_error(self, stub_aws_sdk_client):
@ -826,5 +890,118 @@ class TestBedrockRealtimeAwsAuth:
assert "config_kwargs" not in stub_aws_sdk_client
class TestBedrockRealtimeSdkLifecycle:
"""aws-sdk-bedrock-runtime 0.10/0.11: async config, async client, CRT transport, close() (LIT-7938 regression)"""
AWS_ARGS = {
"model": "amazon.nova-sonic-v1:0",
"aws_region_name": "us-east-1",
"aws_access_key_id": "k",
"aws_secret_access_key": "s",
}
@pytest.mark.asyncio
async def test_client_closed_after_input_stream_on_normal_completion(self, stub_aws_sdk_client):
await BedrockRealtime().async_realtime(websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS)
assert stub_aws_sdk_client["client_closed"]
assert stub_aws_sdk_client["input_closed_before_client_close"]
@pytest.mark.asyncio
async def test_client_closed_when_stream_open_fails(self, stub_aws_sdk_client):
stub_aws_sdk_client["streams"] = [ServiceUnavailableException("bedrock unavailable")]
with pytest.raises(ServiceUnavailableException):
await BedrockRealtime().async_realtime(
websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS
)
assert stub_aws_sdk_client["client_closed"]
@pytest.mark.asyncio
async def test_client_closed_when_provider_stream_fails_mid_session(self, stub_aws_sdk_client):
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver)]
with pytest.raises(BedrockError):
await BedrockRealtime().async_realtime(
websocket=ConnectedClientWS([]), logging_obj=FakeLogging(), **self.AWS_ARGS
)
assert stub_aws_sdk_client["client_closed"]
assert stub_aws_sdk_client["input_closed_before_client_close"]
@pytest.mark.asyncio
async def test_client_without_close_completes_session(self, monkeypatch):
class ClientWithoutClose:
def __init__(self, config):
pass
async def invoke_model_with_bidirectional_stream(self, operation_input):
return ScriptedBedrockStream([])
class ConfigWithoutCapture:
@classmethod
async def resolve(cls, **kwargs):
return cls()
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.AsyncBedrockRuntimeClient = ClientWithoutClose
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.AsyncBedrockRuntimeConfig = ConfigWithoutCapture
_install_fake_sdk_modules(monkeypatch, client_module, config_module)
websocket = RealtimeClientWS()
await BedrockRealtime().async_realtime(websocket=websocket, logging_obj=FakeLogging(), **self.AWS_ARGS)
assert websocket.closed
class TestBedrockRealtimeSdkImportErrors:
"""Init errors must tell 'SDK not installed' apart from 'SDK installed but unsupported version' (LIT-7938)"""
@pytest.mark.asyncio
async def test_absent_sdk_names_install_extra(self, monkeypatch):
monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime", None)
handler = BedrockRealtime(sdk_version_lookup=lambda: None)
with pytest.raises(ImportError) as exc_info:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging()
)
message = str(exc_info.value)
assert message.startswith("Missing aws_sdk_bedrock_runtime")
assert "litellm[bedrock-realtime]" in message
assert "is installed but" not in message
close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode()
assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason
assert "pip install 'litellm[bedrock-realtime]'" in close_reason
@pytest.mark.asyncio
async def test_incompatible_sdk_names_installed_version_and_supported_range(self, monkeypatch):
legacy_client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
legacy_client_module.BedrockRuntimeClient = object
legacy_config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
legacy_config_module.Config = object
_install_fake_sdk_modules(monkeypatch, legacy_client_module, legacy_config_module)
handler = BedrockRealtime(sdk_version_lookup=lambda: "0.7.0")
with pytest.raises(ImportError) as exc_info:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging()
)
message = str(exc_info.value)
assert "aws-sdk-bedrock-runtime 0.7.0 is installed but" in message
assert ">=0.10.0,<0.12.0" in message
assert not message.startswith("Missing aws_sdk_bedrock_runtime")
assert isinstance(exc_info.value.__cause__, ImportError)
assert str(exc_info.value.__cause__) not in message
assert "cannot import name" not in message
close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode()
assert "0.7.0 is installed" in close_reason
assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason
if __name__ == "__main__":
pytest.main([__file__, "-v"])

174
tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
View file

@ -4,7 +4,7 @@ import sys
import types
from datetime import datetime, timedelta, timezone
from datetime import time as dt_time
from typing import Any, Dict, Final, List
from typing import Any, Dict, Final, List, Optional
from unittest.mock import AsyncMock, MagicMock
import httpx
@ -16,6 +16,7 @@ from litellm.proxy._types import LiteLLM_VerificationToken
from litellm.proxy.common_utils import reset_budget_job as reset_budget_job_module
from litellm.constants import (
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
RESET_BUDGET_JOB_BATCH_SIZE,
RESET_BUDGET_JOB_LOCK_TTL_SECONDS,
RESET_BUDGET_JOB_NAME,
)
@ -31,13 +32,36 @@ class MockTable:
self.find_many_calls: List[Dict[str, Any]] = []
self.update_many_calls: List[Dict[str, Any]] = []
self._find_many_results: List[Any] = []
self._find_many_error: Optional[tuple[int, Exception]] = None
def set_find_many_results(self, results: List[Any]):
self._find_many_results = results
async def find_many(self, where: Dict[str, Any]) -> List[Any]:
self.find_many_calls.append({"where": where})
return self._find_many_results
def set_find_many_error(self, after_reads: int, error: Exception):
"""Fail every read past the first ``after_reads``, the way a connection
dropping partway through a paged walk does."""
self._find_many_error = (after_reads, error)
async def find_many(
self,
where: Dict[str, Any],
order: Optional[Dict[str, str]] = None,
take: Optional[int] = None,
) -> List[Any]:
"""Replays canned rows, honouring the keyset cursor + ``take`` a paged
caller relies on: without that a paged walk never advances and the
test would hang instead of failing."""
if self._find_many_error is not None and len(self.find_many_calls) >= self._find_many_error[0]:
raise self._find_many_error[1]
paging = {k: v for k, v in (("order", order), ("take", take)) if v is not None}
self.find_many_calls.append({"where": where, **paging})
rows = list(self._find_many_results)
for field, condition in where.items():
if isinstance(condition, dict) and "gt" in condition and field != "spend":
rows = [row for row in rows if getattr(row, field, "") > condition["gt"]]
for field, direction in (order or {}).items():
rows.sort(key=lambda row: getattr(row, field, ""), reverse=direction == "desc")
return rows[:take] if take is not None else rows
async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]:
self.update_many_calls.append({"where": where, "data": data})
@ -801,10 +825,16 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock
},
]
# Verify find_many was called to fetch NULL-budget-id end users
# The post-commit invalidation walk covers both branches, so implicitly
# created customers on the default tier get their cached spend dropped too,
# and it is paged rather than reading the whole customer population.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 1
assert find_many_calls[0]["where"] == {"budget_id": None, "spend": {"gt": 0}}
assert find_many_calls[0]["where"]["OR"] == [
{"budget_id": {"in": [default_budget_id]}},
{"budget_id": None},
]
assert find_many_calls[0]["take"] == RESET_BUDGET_JOB_BATCH_SIZE
litellm.max_end_user_budget_id = None
@ -835,9 +865,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_configured(
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
# Should NOT have queried for NULL-budget-id end users
# The invalidation walk must not reach for NULL-budget-id customers: they
# ride a default tier that is not expiring, so their spend stays put.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 0
assert [call["where"] for call in find_many_calls] == [
{"budget_id": {"in": ["some-budget"]}, "user_id": {"gt": ""}}
]
litellm.max_end_user_budget_id = None
@ -872,9 +905,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_in_reset_li
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
# Should NOT have queried for NULL-budget-id end users
# The invalidation walk must not reach for NULL-budget-id customers: they
# ride a default tier that is not expiring, so their spend stays put.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 0
assert [call["where"] for call in find_many_calls] == [
{"budget_id": {"in": ["other-budget"]}, "user_id": {"gt": ""}}
]
litellm.max_end_user_budget_id = None
@ -1252,6 +1288,21 @@ def _make_counter_invalidation_job(monkeypatch):
user_api_key_cache = MagicMock()
user_api_key_cache.async_delete_cache = AsyncMock()
# Batch deletes fan out to the same per-key calls the real DualCache makes,
# so an assertion reads "this key was invalidated" whether the caller went
# one key at a time or a page at a time.
async def _delete_counter_keys(keys):
for key in keys:
spend_counter_cache.in_memory_cache.delete_cache(key=key)
await spend_counter_cache.redis_cache.async_delete_cache(key=key)
async def _delete_management_keys(keys):
for key in keys:
await user_api_key_cache.async_delete_cache(key=key)
spend_counter_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_counter_keys)
user_api_key_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_management_keys)
fake_module = types.ModuleType("litellm.proxy.proxy_server")
fake_module.spend_counter_cache = spend_counter_cache
fake_module.user_api_key_cache = user_api_key_cache
@ -1586,7 +1637,7 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j
"user_id": "customer-42",
},
)
mock_prisma_client.data["enduser"] = [test_enduser]
mock_prisma_client.db.litellm_endusertable.set_find_many_results([test_enduser])
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
@ -1596,6 +1647,107 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j
assert "end_user_id:customer-42" in deleted
def test_enduser_invalidation_is_paged_and_batched(reset_budget_job, mock_prisma_client, monkeypatch):
"""The post-commit invalidation walk stays bounded in memory and in round trips.
Reading every customer on an expiring tier into one result set puts a
customer-count-sized list in the proxy's heap on every tick, which is an OOM
on a large enough deployment rather than a slow tick. Awaiting one cache call
per customer makes the last customer wait out every customer ahead of it.
Both regress silently, so pin the page size, the strictly advancing cursor,
and one batched call per page.
"""
counter_cache: Final = _make_counter_invalidation_job(monkeypatch)
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
population: Final = RESET_BUDGET_JOB_BATCH_SIZE * 2 + 3
mock_prisma_client.db.litellm_endusertable.set_find_many_results(
[
type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"})
for i in range(population)
]
)
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
reads: Final = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert [read["take"] for read in reads] == [RESET_BUDGET_JOB_BATCH_SIZE] * 3
assert [read["where"]["user_id"]["gt"] for read in reads] == [
"",
f"cust-{RESET_BUDGET_JOB_BATCH_SIZE - 1:06d}",
f"cust-{RESET_BUDGET_JOB_BATCH_SIZE * 2 - 1:06d}",
]
assert counter_cache.async_delete_cache_keys.await_count == 3
assert counter_cache.user_api_key_cache.async_delete_cache_keys.await_count == 3
counter_cache.async_delete_cache.assert_not_called()
invalidated: Final = {
key for call in counter_cache.async_delete_cache_keys.await_args_list for key in call.args[0]
}
assert invalidated == {f"spend:end_user:cust-{i:06d}" for i in range(population)}
evicted: Final = {
key for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list for key in call.args[0]
}
assert evicted == {f"end_user_id:cust-{i:06d}" for i in range(population)}
def test_enduser_invalidation_reports_a_page_read_failure_instead_of_a_clean_finish(
mock_prisma_client, monkeypatch
):
"""A page that fails to read is not the end of the customer list.
The tier's window is already advanced by the time this walk runs, so no later
tick comes back for the customers past the page that failed: their cached
spend goes on rejecting requests until it expires. Returning the same empty
page normal end-of-data returns hid that behind a report of a clean pass.
"""
_make_counter_invalidation_job(monkeypatch)
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
endusers: Final = mock_prisma_client.db.litellm_endusertable
endusers.set_find_many_results(
[
type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"})
for i in range(RESET_BUDGET_JOB_BATCH_SIZE + 3)
]
)
endusers.set_find_many_error(1, RuntimeError("connection reset while paging customers"))
logging_obj: Final = RecordingProxyLogging()
job: Final = ResetBudgetJob(proxy_logging_obj=logging_obj, prisma_client=mock_prisma_client)
_run_and_drain_hooks(job.reset_budget_for_litellm_budget_table)
metadata: Final = logging_obj.service_logging_obj.success_calls[0]["event_metadata"]
assert metadata["enduser_invalidation_truncated"] is True
assert metadata["num_endusers_updated"] == RESET_BUDGET_JOB_BATCH_SIZE
def test_a_failed_counter_batch_still_evicts_the_management_cache(
reset_budget_job, mock_prisma_client, monkeypatch
):
"""The spend counters and the management cache are invalidated independently.
Sharing one handler meant a Redis failure on the counters returned before the
management cache was touched at all. The commit has already zeroed those rows
by then, so the cached objects keep authorizing against their pre-reset spend
until they expire.
"""
counter_cache: Final = _make_counter_invalidation_job(monkeypatch)
counter_cache.async_delete_cache_keys = AsyncMock(side_effect=RuntimeError("redis unavailable"))
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
mock_prisma_client.db.litellm_endusertable.set_find_many_results(
[type("EndUser", (), {"user_id": "customer-42", "spend": 5.0, "budget_id": "budget-1"})]
)
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
evicted: Final = {
key
for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list
for key in call.args[0]
}
assert "end_user_id:customer-42" in evicted
def test_budget_table_reset_commits_even_when_cache_eviction_fails(reset_budget_job, mock_prisma_client, monkeypatch):
"""Eviction runs after the commit, so a broken cache cannot undo the write."""

53
tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py
View file

@ -82,6 +82,19 @@ class FakeRedisCache(RedisCache):
async def async_delete_cache(self, key: str): # type: ignore[override]
self._store.pop(key, None)
async def delete_cache_keys(self, keys): # type: ignore[override]
for key in keys:
self._store.pop(key, None)
class PartitionFailingRedisCache(FakeRedisCache):
"""Fails the batch delete for the key-object partition and no other."""
async def delete_cache_keys(self, keys): # type: ignore[override]
if any(is_user_key_cache_key(key) for key in keys):
raise ConnectionError("redis unavailable")
await super().delete_cache_keys(keys)
def _make_key_obj(token: str = "tok") -> UserAPIKeyAuth:
# Minimal object (UserAPIKeyAuth inherits token from base view).
@ -331,6 +344,46 @@ class TestUserKeyObjectPartition:
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None
assert await redis.async_get_cache(HASHED_TOKEN) is None
@pytest.mark.asyncio
async def test_batch_delete_routes_each_key_to_its_partition(self):
"""A batch delete has to clear the same partition the single delete does.
``DualCache``'s batch delete only knows about the main in-memory cache, so
inheriting it unchanged leaves a key object sitting in ``key_object_cache``
with its pre-reset spend, and the next request is authorized against that
stale copy until the local entry expires.
"""
redis = FakeRedisCache()
cache = UserApiKeyCache(redis_cache=redis)
await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth)
await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"})
await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")])
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None
assert await cache.async_get_cache(end_user_cache_key("u1")) is None
assert await redis.async_get_cache(HASHED_TOKEN) is None
assert await redis.async_get_cache(end_user_cache_key("u1")) is None
@pytest.mark.asyncio
async def test_batch_delete_clears_the_other_partition_when_one_fails(self):
"""One partition failing must not cost the other its deletions.
A caller batching these has already committed the rows they cache, so a
partition that is skipped keeps authorizing against pre-reset spend until
the entry expires. The failure is still raised for the caller to report.
"""
redis = PartitionFailingRedisCache()
cache = UserApiKeyCache(redis_cache=redis)
await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth)
await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"})
with pytest.raises(ConnectionError):
await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")])
assert await cache.async_get_cache(end_user_cache_key("u1")) is None
assert await redis.async_get_cache(end_user_cache_key("u1")) is None
@pytest.mark.asyncio
async def test_pipeline_write_routes_each_entry_to_its_partition(self):
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(max_size_in_memory=2))

21
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -1049,6 +1049,27 @@ def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_
assert payload["model"] == expected_model
@pytest.mark.parametrize("requested_model", [{"bad": "value"}, ["gpt-5.2"], 1])
def test_get_logging_payload_replaces_a_non_string_model_with_the_placeholder(
requested_model: dict[str, str] | list[str] | int,
):
kwargs: Final = {
"model": requested_model,
"messages": [{"role": "user", "content": "hi"}],
"call_type": "acompletion",
"litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=ValueError("model must be a string"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == UNKNOWN_MODEL_SPEND_LOG_MODEL
@pytest.mark.parametrize(
("metadata", "response_obj"),
[

29
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -327,6 +327,35 @@ class TestProxyBaseLLMRequestProcessing:
pytest.fail("litellm_call_id is not a valid UUID")
assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"]
@pytest.mark.asyncio
@pytest.mark.parametrize("requested_model", [{"bad": "value"}, ["gpt-5.2"], 1])
async def test_common_processing_pre_call_logic_rejects_a_non_string_model_with_400(
self, monkeypatch, requested_model: dict[str, str] | list[str] | int
):
processing_obj = ProxyBaseLLMRequestProcessing(
data={"model": requested_model, "messages": [{"role": "user", "content": "hi"}]}
)
mock_request = MagicMock(spec=Request)
mock_request.headers = {}
add_litellm_data_to_request = AsyncMock()
monkeypatch.setattr(
litellm.proxy.common_request_processing, "add_litellm_data_to_request", add_litellm_data_to_request
)
with pytest.raises(ProxyException) as exc_info:
await processing_obj.common_processing_pre_call_logic(
request=mock_request,
general_settings={},
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
proxy_logging_obj=MagicMock(spec=ProxyLogging),
proxy_config=MagicMock(spec=ProxyConfig),
route_type="acompletion",
)
assert exc_info.value.code == str(status.HTTP_400_BAD_REQUEST)
assert exc_info.value.param == "model"
add_litellm_data_to_request.assert_not_awaited()
@pytest.mark.asyncio
async def test_common_processing_pre_call_logic_refreshes_proxy_server_request_body_after_guardrails(
self, monkeypatch

23
tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py
View file

@ -4,15 +4,23 @@ Static checks that every proxy Docker image installs the `bedrock-realtime` extr
Bedrock Nova Sonic speech-to-speech (`/v1/realtime`) needs `aws-sdk-bedrock-runtime`,
which only ships in the `bedrock-realtime` extra. An image whose `uv sync` stages
omit the extra fails every Nova Sonic realtime session with
"Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime".
"Missing aws_sdk_bedrock_runtime: pip install 'litellm[bedrock-realtime]' ...".
"""
import os
import re
import sys
from typing import Final
import pytest
from litellm.constants import BEDROCK_REALTIME_SDK_DISTRIBUTION, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE
if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib
REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "..", "..")
PROXY_DOCKERFILES: Final = (
@ -54,3 +62,16 @@ def test_every_uv_sync_installs_bedrock_realtime_extra(relative_path: str):
"`--extra bedrock-realtime`, so aws-sdk-bedrock-runtime is absent and Bedrock Nova Sonic "
"/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'"
)
def test_bedrock_realtime_extra_pins_the_range_named_in_the_runtime_error():
with open(os.path.join(REPO_ROOT, "pyproject.toml"), "rb") as f:
extra_specs: Final = tomllib.load(f)["project"]["optional-dependencies"]["bedrock-realtime"]
sdk_specs: Final = tuple(spec for spec in extra_specs if spec.startswith(BEDROCK_REALTIME_SDK_DISTRIBUTION))
assert len(sdk_specs) == 1, f"expected exactly one {BEDROCK_REALTIME_SDK_DISTRIBUTION} spec, got {extra_specs}"
requirement: Final = sdk_specs[0].split(";")[0].strip()
assert requirement == f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}", (
f"pyproject pins {requirement!r} but the handler's install hint names "
f"{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE!r} with the awscrt extra; keep them in sync"
)

6
tests/test_litellm/test_together_ai_model_metadata.py
View file

@ -95,7 +95,7 @@ def _successor(info: dict[str, object]) -> str | None:
return successor if isinstance(successor, str) else None
def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
def test_together_successor_metadata_points_at_known_models(cost_map: CostMap):
successors = {
model: successor
for model, info in cost_map.items()
@ -103,9 +103,7 @@ def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
}
assert len(successors) >= 10
for model, successor in successors.items():
target = cost_map.get(successor)
assert target is not None, f"{model} names successor {successor} that is not in the map"
assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
assert successor in cost_map, f"{model} names successor {successor} that is not in the map"
def test_together_backup_cost_map_in_sync(cost_map: CostMap):

99
ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx
View file

@ -1,7 +1,7 @@
import React from "react";
import { describe, it, expect, vi, beforeEach } from "vitest";
import userEvent from "@testing-library/user-event";
import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils";
import { renderWithProviders, screen, waitFor, within } from "../../../../tests/test-utils";
import {
GuardrailInformation,
makeBedrockResponse,
@ -24,6 +24,42 @@ const skippedPreCall: Partial<GuardrailInformation> = {
duration: null,
};
const untimedPreCall: Partial<GuardrailInformation> = {
guardrail_name: "conduct",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: null,
end_time: null,
duration: null,
};
const timedPreCall: Partial<GuardrailInformation> = {
guardrail_name: "timed-pre-rail",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: 1_700_000_000,
end_time: 1_700_000_000.1,
duration: 0.1,
};
const latePreCall: Partial<GuardrailInformation> = {
guardrail_name: "late-pre-rail",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: 1_700_000_500,
end_time: 1_700_000_500.1,
duration: 0.1,
};
const untimedPostCall: Partial<GuardrailInformation> = {
guardrail_name: "untimed-post-rail",
guardrail_status: "success",
guardrail_mode: "post_call",
start_time: null,
end_time: null,
duration: null,
};
const ranPostCall: Partial<GuardrailInformation> = {
guardrail_name: "ran-rail",
guardrail_status: "success",
@ -98,6 +134,67 @@ describe("GuardrailViewer", () => {
expect(screen.getByText("—")).toBeInTheDocument();
});
it("keeps a guardrail that ran without any timing on the lifecycle", () => {
renderWithProviders(<GuardrailViewer data={makeGuardrailInformation(untimedPreCall)} />);
expect(screen.getByText("Request received")).toBeInTheDocument();
expect(screen.getByText(/Pre-call guardrail: conduct/)).toBeInTheDocument();
expect(screen.getByText("LLM call")).toBeInTheDocument();
expect(screen.getByText("Response returned")).toBeInTheDocument();
expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument();
});
it("keeps an untimed guardrail ahead of a timed one recorded after it in the same phase", () => {
const untimed = makeGuardrailInformation(untimedPreCall);
const timedPre = makeGuardrailInformation(timedPreCall);
renderWithProviders(<GuardrailViewer data={[untimed, timedPre]} />);
const rows = screen.getAllByTestId("lifecycle-row");
const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null);
const untimedIndex = rowIndex(/Pre-call guardrail: conduct/);
const timedIndex = rowIndex(/Pre-call guardrail: timed-pre-rail/);
expect(untimedIndex).toBeGreaterThanOrEqual(0);
expect(timedIndex).toBeGreaterThanOrEqual(0);
expect(untimedIndex).toBeLessThan(timedIndex);
});
it("orders each phase on its own clock when a later pre-call outlives an earlier post-call", () => {
const latePre = makeGuardrailInformation(latePreCall);
const untimedPost = makeGuardrailInformation(untimedPostCall);
const earlyPost = makeGuardrailInformation(ranPostCall);
renderWithProviders(<GuardrailViewer data={[latePre, untimedPost, earlyPost]} />);
const rows = screen.getAllByTestId("lifecycle-row");
const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null);
const untimedIndex = rowIndex(/Post-call guardrail: untimed-post-rail/);
const earlyIndex = rowIndex(/Post-call guardrail: ran-rail/);
expect(untimedIndex).toBeGreaterThanOrEqual(0);
expect(earlyIndex).toBeGreaterThanOrEqual(0);
expect(untimedIndex).toBeLessThan(earlyIndex);
});
it("anchors offsets on the timed entries and gives the untimed one no fabricated offset", () => {
const untimed = makeGuardrailInformation(untimedPreCall);
const ran = makeGuardrailInformation(ranPostCall);
renderWithProviders(<GuardrailViewer data={[untimed, ran]} />);
const lifecycleRow = (label: string | RegExp): HTMLElement => {
const row = screen.getAllByTestId("lifecycle-row").find((r) => within(r).queryByText(label) !== null);
if (row === undefined) throw new Error(`no lifecycle row labelled ${label}`);
return row;
};
expect(within(lifecycleRow("Request received")).getByText("T+0ms")).toBeInTheDocument();
expect(within(lifecycleRow(/Post-call guardrail: ran-rail/)).getByText("T+250ms")).toBeInTheDocument();
expect(within(lifecycleRow("Response returned")).getByText("T+251ms")).toBeInTheDocument();
const untimedRow = within(lifecycleRow(/Pre-call guardrail: conduct/));
expect(untimedRow.getByText("—")).toBeInTheDocument();
expect(untimedRow.queryByText(/^T\+/)).not.toBeInTheDocument();
});
it("calculates and displays masked entity totals", async () => {
const user = userEvent.setup();
const data = makeGuardrailInformation({

60
ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx
View file

@ -361,7 +361,7 @@ const GenericGuardrailResponse = ({ response }: { response: any }) => {
interface TimelineEntry {
type: "request" | "guardrail" | "llm" | "response";
label: string;
offsetMs: number;
offsetMs: number | null;
outcome?: EntryOutcome;
}
@ -370,73 +370,85 @@ type TimedGuardrailInformation = GuardrailInformation & { start_time: number; en
const isTimed = (e: GuardrailInformation): e is TimedGuardrailInformation =>
typeof e.start_time === "number" && typeof e.end_time === "number";
const belongsOnLifecycle = (e: GuardrailInformation): boolean => isTimed(e) || getEntryOutcome(e) !== "not_run";
// Sorts a phase's timed entries by start time while leaving its untimed entries in the
// slots they were recorded in. Applied per phase, never globally: an entry can land in
// more than one phase bucket, so a global pass can reorder one phase by another's clock.
const orderWithinPhase = (group: GuardrailInformation[]): GuardrailInformation[] => {
const byStart = group.filter(isTimed).sort((a, b) => a.start_time - b.start_time);
const timedSlots = new Map(group.flatMap((e, i) => (isTimed(e) ? [i] : [])).map((slot, k) => [slot, byStart[k]]));
return group.map((e, i) => timedSlots.get(i) ?? e);
};
const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
const sorted = useMemo(() => entries.filter(isTimed).sort((a, b) => a.start_time - b.start_time), [entries]);
const sorted = useMemo(() => entries.filter(belongsOnLifecycle), [entries]);
const timeline = useMemo(() => {
if (sorted.length === 0) return [];
const baseTime = sorted[0].start_time;
const timed = sorted.filter(isTimed);
const baseTime = timed.length > 0 ? Math.min(...timed.map((e) => e.start_time)) : null;
const offsetOf = (e: GuardrailInformation): number | null =>
baseTime === null || !isTimed(e) ? null : Math.round((e.end_time - baseTime) * 1000);
const items: TimelineEntry[] = [];
// Request received
items.push({ type: "request", label: "Request received", offsetMs: 0 });
items.push({ type: "request", label: "Request received", offsetMs: baseTime === null ? null : 0 });
// Pre-call guardrails — use modeMatches so array modes (e.g. ["pre_call", "post_call"])
// place the entry in every matching bucket.
const preCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call"));
const postCalls = sorted.filter(
(e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only"),
const preCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call")));
const postCalls = orderWithinPhase(
sorted.filter((e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only")),
);
const duringCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call"));
const duringCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call")));
for (const e of preCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `Pre-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// LLM call — infer from gap between pre-call end and post-call start
const lastPreEnd = preCalls.length > 0 ? Math.max(...preCalls.map((e) => e.end_time)) : baseTime;
const firstPostStart = postCalls.length > 0 ? Math.min(...postCalls.map((e) => e.start_time)) : undefined;
const llmEndTime = firstPostStart ?? lastPreEnd + 1;
const llmOffsetMs = Math.round((llmEndTime - baseTime) * 1000);
const timedPre = preCalls.filter(isTimed);
const timedPost = postCalls.filter(isTimed);
const lastPreEnd = timedPre.length > 0 ? Math.max(...timedPre.map((e) => e.end_time)) : baseTime;
const firstPostStart = timedPost.length > 0 ? Math.min(...timedPost.map((e) => e.start_time)) : undefined;
const llmEndTime = firstPostStart ?? (lastPreEnd === null ? null : lastPreEnd + 1);
items.push({
type: "llm",
label: "LLM call",
offsetMs: llmOffsetMs,
offsetMs: llmEndTime === null || baseTime === null ? null : Math.round((llmEndTime - baseTime) * 1000),
});
// During-call guardrails (rare)
for (const e of duringCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `During-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// Post-call guardrails
for (const e of postCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `Post-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// Response returned
const maxEnd = Math.max(...sorted.map((e) => e.end_time));
const responseOffsetMs = Math.round((maxEnd - baseTime) * 1000) + 1;
const maxEnd = timed.length > 0 ? Math.max(...timed.map((e) => e.end_time)) : null;
const responseOffsetMs = maxEnd === null || baseTime === null ? null : Math.round((maxEnd - baseTime) * 1000) + 1;
items.push({ type: "response", label: "Response returned", offsetMs: responseOffsetMs });
return items;
@ -447,7 +459,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
<h4 className="text-xs font-semibold text-muted-foreground uppercase tracking-wider mb-4">Request Lifecycle</h4>
<div className="relative">
{timeline.map((item, idx) => (
<div key={idx} className="flex items-start gap-3 relative">
<div key={idx} data-testid="lifecycle-row" className="flex items-start gap-3 relative">
{/* Vertical line */}
<div className="flex flex-col items-center">
<div className="shrink-0">
@ -475,7 +487,9 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
{OUTCOME_LABEL[item.outcome]}
</span>
)}
<span className="text-xs text-muted-foreground font-mono ml-auto shrink-0">T+{item.offsetMs}ms</span>
<span className="text-xs text-muted-foreground font-mono ml-auto shrink-0">
{item.offsetMs === null ? "—" : `T+${item.offsetMs}ms`}
</span>
</div>
</div>
</div>

49
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-09-12T22:48:38.53978Z"
exclude-newer = "2026-09-14T20:32:38.482736111Z"
exclude-newer-span = "P3D"
[manifest]
@ -535,16 +535,21 @@ wheels = [
[[package]]
name = "aws-sdk-bedrock-runtime"
version = "0.7.0"
version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", extra = ["aiohttp"], marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/67/8a/ed3fd98775273b0b7f6006b4970aa876d506668b7fe29145f54fcb941c3b/aws_sdk_bedrock_runtime-0.7.0.tar.gz", hash = "sha256:0cb172cbc03ff060e5c1d6f9cfa9a8ac5e71d9e0d58d3117006ebf614cbb4677", size = 170304, upload-time = "2026-06-23T04:04:52.382Z" }
sdist = { url = "https://files.pythonhosted.org/packages/8e/b3/9c225cbfe9f17ea2e3d75a0fdd0b325ef79839b9c09a376bda63a7bf3bb3/aws_sdk_bedrock_runtime-0.11.0.tar.gz", hash = "sha256:f2c45d34625bf6a7b56375e29a53a16b376880bda771e4bbf7d84491622eb193", size = 173854, upload-time = "2026-08-24T21:17:16.304Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9e/e1/f86d50f0ad9c8200645f315c524d285e86b30b94bb65118e1108597714e6/aws_sdk_bedrock_runtime-0.7.0-py3-none-any.whl", hash = "sha256:de67ede6f441bbb77ef61c237945d559513843fc827abe1af12535c2519650c5", size = 94948, upload-time = "2026-06-23T04:04:51.281Z" },
{ url = "https://files.pythonhosted.org/packages/29/0c/9512304ed017ce49992df6661eac2b914550247e13bccb55be6ca594170d/aws_sdk_bedrock_runtime-0.11.0-py3-none-any.whl", hash = "sha256:ef01c26ddfd83a5d3e438ab72ebb3c13b41fc0ef11d81095b22c8016f97e9795", size = 97112, upload-time = "2026-08-24T21:17:17.396Z" },
]
[package.optional-dependencies]
awscrt = [
{ name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
]
[[package]]
@ -4483,7 +4488,7 @@ dependencies = [
[package.optional-dependencies]
bedrock-realtime = [
{ name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12'" },
{ name = "aws-sdk-bedrock-runtime", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
]
caching = [
{ name = "diskcache" },
@ -4693,7 +4698,7 @@ requires-dist = [
{ name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" },
{ name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" },
{ name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.0.19,<1.0" },
{ name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.7.0,<0.8.0" },
{ name = "aws-sdk-bedrock-runtime", extras = ["awscrt"], marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.10.0,<0.12.0" },
{ name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = ">=1.0.0,<2.0" },
{ name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = ">=1.25.2,<2.0" },
{ name = "azure-identity", marker = "extra == 'proxy'", specifier = ">=1.25.2,<2.0" },
@ -4884,7 +4889,7 @@ source = { editable = "enterprise" }
[[package]]
name = "litellm-proxy-extras"
version = "0.4.98"
version = "0.4.99"
source = { editable = "litellm-proxy-extras" }
[[package]]
@ -9126,16 +9131,16 @@ wheels = [
[[package]]
name = "smithy-aws-core"
version = "0.7.0"
version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fc/a8/37bfde59519f45d2047d0033b791aca6574d867aaf57bb56a6de42ab5c26/smithy_aws_core-0.7.0.tar.gz", hash = "sha256:34e82d09fc808acd5ffc80f03828d0609c6a211f49f0884dc6ee7ca095a1b6af", size = 15670, upload-time = "2026-06-23T04:04:50.365Z" }
sdist = { url = "https://files.pythonhosted.org/packages/7d/d3/501c0023548173416109ac42298ca33b708469dc922005770811a597949f/smithy_aws_core-0.11.0.tar.gz", hash = "sha256:29ee89976a520a87e3db557e03e115fdc21a0a60b81161e95174395a1b064da1", size = 38791, upload-time = "2026-08-24T21:16:59.631Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fd/54/2d06dd9a3972a380d71bb8c3312e317aa8f1ea68dd28cffc06955ccf0220/smithy_aws_core-0.7.0-py3-none-any.whl", hash = "sha256:6c60c8fbb9431c60e80ea7f2d37e7ae48409cc1541f587fe073f202eca067e92", size = 24894, upload-time = "2026-06-23T04:04:49.349Z" },
{ url = "https://files.pythonhosted.org/packages/e4/f6/fefda9aab809fa1a62bf7073bd6d8ab427bd9989f39b13c0d6e29d4d1045/smithy_aws_core-0.11.0-py3-none-any.whl", hash = "sha256:77cf130c22deac14a8cbeb8ccc4bcfe5a91798f4b38cb53a987080ec58c89f23", size = 58855, upload-time = "2026-08-24T21:16:58.657Z" },
]
[package.optional-dependencies]
@ -9160,41 +9165,45 @@ wheels = [
[[package]]
name = "smithy-core"
version = "0.6.0"
version = "0.8.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/e9/45/688d52c61cd4d843bb230694259e91d4c7d6954eeecbadf452a168001d45/smithy_core-0.6.0.tar.gz", hash = "sha256:ba2e5d860d716aff75004a23f53e09dfaca3e2b94f8a00c1f76dcb355b769ce0", size = 52095, upload-time = "2026-06-23T04:04:44.687Z" }
sdist = { url = "https://files.pythonhosted.org/packages/7c/c6/93e9eea3c6163228dfe972c3e989e0553047858805ab7aa4a59f074ba129/smithy_core-0.8.1.tar.gz", hash = "sha256:3d2f8fca5960d74bd7ef380f70901c7bcdebe53f929d2d3d2fa6cb790b3f5214", size = 54259, upload-time = "2026-08-20T17:55:30.354Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/b6/06795faa9844b9667ae492e6293370393e19e7f0c2df8da1b4bf7e5f6ed9/smithy_core-0.6.0-py3-none-any.whl", hash = "sha256:51e347ed309d60ab9d36b783dbf88de614c460d51bec79d39cd403956b00f063", size = 66879, upload-time = "2026-06-23T04:04:43.596Z" },
{ url = "https://files.pythonhosted.org/packages/0c/23/c6430bbf406477fc7d16254b9908a723b299a4a21a94c99db9d12c84a8bf/smithy_core-0.8.1-py3-none-any.whl", hash = "sha256:44bd9bdf702f76919af58e44a6a1bb3dc136a745b2f955281743022ce767e347", size = 68805, upload-time = "2026-08-20T17:55:29.366Z" },
]
[[package]]
name = "smithy-http"
version = "0.4.2"
version = "0.5.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/58/5a772d212e066d6fc1398946c4aae19bcdaa75209879d776f641b6a06b5b/smithy_http-0.4.2.tar.gz", hash = "sha256:50d11b6a55e42448450a01e3d0f605ccee65a72abf52d02eed82862a15be5937", size = 29616, upload-time = "2026-06-23T04:04:45.687Z" }
sdist = { url = "https://files.pythonhosted.org/packages/98/78/b5f3113d6c8f0bc1f9777a7f5ca84b892d29efac05850e14f7d4f7e645b5/smithy_http-0.5.0.tar.gz", hash = "sha256:bb4a19672f7c7eeb872a308f777eb505281a5bafb1ee3d1ea9c760c06c352510", size = 31122, upload-time = "2026-08-24T21:16:56.488Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/57/3e/7b2464d40893bec0b5d1f479d25116d4aa09f9f66536b4c4b3126202215d/smithy_http-0.4.2-py3-none-any.whl", hash = "sha256:a158f107e9fab925289d20772c2e38b0bba94e55c05d0edc9290310f22a60454", size = 41025, upload-time = "2026-06-23T04:04:46.764Z" },
{ url = "https://files.pythonhosted.org/packages/27/27/e414082643028846b73afa52a1a8f934548196ee12b187a06803f02a3e66/smithy_http-0.5.0-py3-none-any.whl", hash = "sha256:af273d5f42e7733ce7a6e9bd6fdd6a59ef1b61f6cd1f4a89dd53dfce99da7bef", size = 42198, upload-time = "2026-08-24T21:16:57.52Z" },
]
[package.optional-dependencies]
aiohttp = [
{ name = "aiohttp", marker = "python_full_version >= '3.12'" },
{ name = "yarl", marker = "python_full_version >= '3.12'" },
]
awscrt = [
{ name = "awscrt", marker = "python_full_version >= '3.12'" },
]
[[package]]
name = "smithy-json"
version = "0.2.3"
version = "0.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ijson", marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/6c/418b5687d8933b7a135d5e1a98c61fe814b98f72517dbae0e666860cb876/smithy_json-0.2.3.tar.gz", hash = "sha256:686e9b55a36dacb08e472732b358573ef78009055e05e9fce2e806d61490b2b3", size = 7805, upload-time = "2026-06-23T04:04:47.71Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c7/ac/04164eefb3da7479f52f6535b4b39cc8384c292cb2bb74279f2acc4f4b4d/smithy_json-0.3.0.tar.gz", hash = "sha256:c81c7034587e01bc64767cbbecb05a7d65ca9070612fd94e8a03e80540290a22", size = 7956, upload-time = "2026-08-20T17:55:32.177Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3c/14/eabb26b355415bcd9feef27fb5b18f1dad3fabd4208cfcbaf152025fa9ae/smithy_json-0.2.3-py3-none-any.whl", hash = "sha256:594e1bbe3d480963237f8fd0fc648dbd4e988b4503fea90157b5f07706796327", size = 10252, upload-time = "2026-06-23T04:04:48.46Z" },
{ url = "https://files.pythonhosted.org/packages/9d/cf/0104c40a0e18fa307ea3da4310eba949f474a5bc1df3cc2b5851a72e8486/smithy_json-0.3.0-py3-none-any.whl", hash = "sha256:ffb73d2e60cf5e616e5d0a1019e7b9f518edba076cb423f10981457725dcddc4", size = 10252, upload-time = "2026-08-20T17:55:31.204Z" },
]
[[package]]