scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

merge: resolve conflict with main, move temp budget patch fields to shared member_budget_patch

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-17 23:04:25 +00:00
commit 9d5f65c26c
63 changed files with 3006 additions and 489 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cookbook/litellm_proxy_server/cli_token_usage.py
View file

@ -3,7 +3,7 @@
Example: Using CLI token with LiteLLM SDK
This example shows how to use the CLI authentication token
in your Python scripts after running `litellm-proxy login`.
in your Python scripts after running `lite login`.
"""
from textwrap import indent
@ -22,7 +22,7 @@ def main():
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
print("❌ No CLI token found. Please run 'lite login' first.")
return
print("✅ Found CLI token.")
@ -58,6 +58,6 @@ if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("1. Run 'lite login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none")

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.98"
version = "0.4.99"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.98"
version = "0.4.99"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

12
litellm/caching/dual_cache.py
View file

@ -521,6 +521,18 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
await self.redis_cache.async_delete_cache(key)
async def async_delete_cache_keys(self, keys: Sequence[str]) -> None:
"""Batch twin of ``async_delete_cache``, chunked because Redis takes the
whole list as one DELETE command."""
if not keys:
return
for key in keys:
self.in_memory_cache.delete_cache(key)
if self.redis_cache is None:
return
for start in range(0, len(keys), DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE):
await self.redis_cache.delete_cache_keys(keys[start : start + DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE])
async def async_get_ttl(self, key: str) -> int | None:
"""
Get the remaining TTL of a key in in-memory cache or redis

2
litellm/constants.py
View file

@ -320,6 +320,8 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE: Final = ">=0.10.0,<0.12.0"
CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved"
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"

3
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -508,7 +508,8 @@ class AnthropicMessagesHandler(BaseTranslation):
chat_completion_compatible_request,
_tool_name_mapping,
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()),
preserve_midturn_system=True,
)
return chat_completion_compatible_request

36
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -118,6 +118,10 @@ from litellm.llms.anthropic.common_utils import (
from litellm.llms.anthropic.experimental_pass_through.context_management import (
PolyfillResult,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
convert_mid_conversation_system_turns,
is_system_role_message,
)
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
openai_chat_refusal_text,
refusal_stop_details,
@ -176,6 +180,7 @@ from litellm.types.llms.openai import (
ToolMessageContentPart,
)
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
from litellm.utils import supports_mid_conversation_system
from .streaming_iterator import AnthropicStreamWrapper
@ -186,6 +191,12 @@ if TYPE_CHECKING:
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
def target_supports_mid_conversation_system(model: str | None, custom_llm_provider: str | None) -> bool:
if not model:
return False
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
class AnthropicAdapter:
def __init__(self) -> None:
pass
@ -418,10 +429,28 @@ class LiteLLMAnthropicMessagesAdapter:
self,
messages: list[AllAnthropicPassThroughMessageValues],
model: str | None = None,
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
) -> list:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
for m in replayable_messages:
leading_count: Final = next(
(i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
len(replayable_messages),
)
trailing_messages: Final = replayable_messages[leading_count:]
keeps_midturn_system: Final = (
preserve_midturn_system
or not any(is_system_role_message(m) for m in trailing_messages)
or target_supports_mid_conversation_system(model, custom_llm_provider)
)
ordered_messages: Final = (
replayable_messages
if keeps_midturn_system
else (*replayable_messages[:leading_count], *convert_mid_conversation_system_turns(trailing_messages))
)
for m in ordered_messages:
user_message: ChatCompletionUserMessage | None = None
tool_message_list: list[ChatCompletionToolMessage] = []
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []
@ -494,7 +523,7 @@ class LiteLLMAnthropicMessagesAdapter:
if isinstance(m.get("content"), str):
assistant_message_str = str(m.get("content", ""))
elif isinstance(m.get("content"), list):
for content in m.get("content", []):
for content in cast(list, m.get("content", [])): # cast-ok: untrusted client payload
if isinstance(content, str):
assistant_message_str = str(content)
elif isinstance(content, dict):
@ -1154,6 +1183,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1175,6 +1205,8 @@ class LiteLLMAnthropicMessagesAdapter:
new_messages = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
preserve_midturn_system=preserve_midturn_system,
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)

6
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -765,7 +765,8 @@ def _count_effective_tokens(
messages=cast(
"list[AllAnthropicPassThroughMessageValues]",
messages_without_compaction,
)
),
preserve_midturn_system=True,
)
except Exception as e:
verbose_logger.debug(
@ -920,7 +921,8 @@ def _build_summary_messages(
messages=cast(
"list[AllAnthropicPassThroughMessageValues]",
stripped,
)
),
preserve_midturn_system=True,
)
except Exception as e:
verbose_logger.warning(

77
litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py Normal file
View file

@ -0,0 +1,77 @@
from collections.abc import Mapping, Sequence
from itertools import groupby
from typing import Final
CONVERTED_SYSTEM_NOTE: Final = (
"Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
)
def as_system_content_blocks(value: object) -> list[object]:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
def is_system_role_message(message: object) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def system_role_message_as_user(message: Mapping[str, object]) -> Mapping[str, object]:
return {
"role": "user",
"content": as_system_content_blocks(CONVERTED_SYSTEM_NOTE) + as_system_content_blocks(message.get("content")),
}
def opens_with_tool_results(message: object) -> bool:
if not isinstance(message, dict) or message.get("role") != "user":
return False
content: Final = message.get("content")
return (
isinstance(content, list)
and len(content) > 0
and isinstance(content[0], dict)
and content[0].get("type") == "tool_result"
)
def system_run_placed_after_tool_results(
system_run: Sequence[Mapping[str, object]], follower_run: Sequence[Mapping[str, object]]
) -> tuple[Mapping[str, object], ...]:
if follower_run and opens_with_tool_results(follower_run[0]):
return (follower_run[0], *system_run, *follower_run[1:])
return (*system_run, *follower_run)
def system_turns_after_tool_results(
messages: Sequence[Mapping[str, object]],
) -> tuple[Mapping[str, object], ...]:
runs: Final = tuple(tuple(run) for _, run in groupby(messages, key=is_system_role_message))
if not runs:
return ()
first_system_run: Final = 0 if is_system_role_message(runs[0][0]) else 1
paired_runs: Final = tuple(
(runs[i], runs[i + 1] if i + 1 < len(runs) else ()) for i in range(first_system_run, len(runs), 2)
)
return (
*(runs[0] if first_system_run else ()),
*(
m
for system_run, follower_run in paired_runs
for m in system_run_placed_after_tool_results(system_run, follower_run)
),
)
def convert_mid_conversation_system_turns(
messages: Sequence[Mapping[str, object]],
) -> tuple[Mapping[str, object], ...]:
return tuple(
system_role_message_as_user(m) if is_system_role_message(m) else m
for m in system_turns_after_tool_results(messages)
)

81
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -27,6 +27,11 @@ from ...common_utils import (
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
from .mid_conversation_system import (
as_system_content_blocks,
convert_mid_conversation_system_turns,
is_system_role_message,
)
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -151,73 +156,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
else:
return system_param
@staticmethod
def _as_system_content_blocks(value: object) -> list:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
@staticmethod
def _is_system_role_message(message: object) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
_CONVERTED_SYSTEM_NOTE: Final = (
"Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
)
def _system_role_message_as_user(self, message: Mapping) -> Mapping:
return {
"role": "user",
"content": self._as_system_content_blocks(self._CONVERTED_SYSTEM_NOTE)
+ self._as_system_content_blocks(message.get("content")),
}
@staticmethod
def _opens_with_tool_results(message: object) -> bool:
if not isinstance(message, dict) or message.get("role") != "user":
return False
content: Final = message.get("content")
return (
isinstance(content, list)
and len(content) > 0
and isinstance(content[0], dict)
and content[0].get("type") == "tool_result"
)
def _system_run_before(self, messages: Sequence, index: int) -> Sequence:
start: Final = next(
(j + 1 for j in range(index - 1, -1, -1) if not self._is_system_role_message(messages[j])),
0,
)
return messages[start:index]
def _system_run_end(self, messages: Sequence, index: int) -> int:
return next(
(j for j in range(index, len(messages)) if not self._is_system_role_message(messages[j])),
len(messages),
)
def _reordered_around_tool_results(self, messages: Sequence, index: int) -> tuple:
message: Final = messages[index]
if self._opens_with_tool_results(message):
return (message, *self._system_run_before(messages, index))
if not self._is_system_role_message(message):
return (message,)
run_end: Final = self._system_run_end(messages, index)
follower: Final = messages[run_end] if run_end < len(messages) else None
return () if self._opens_with_tool_results(follower) else (message,)
def _system_turns_after_tool_results(self, messages: Sequence) -> tuple:
return tuple(
message
for index in range(len(messages))
for message in self._reordered_around_tool_results(messages, index)
)
def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None:
"""Normalize ``role: "system"`` entries in ``messages`` per the Anthropic
``/v1/messages`` contract, which the first-party API, Bedrock Invoke,
@ -254,7 +192,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if not isinstance(messages, list):
return
leading_count: Final = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
(i for i, m in enumerate(messages) if not is_system_role_message(m)),
len(messages),
)
hoisted: Final = messages[:leading_count]
@ -265,10 +203,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self.custom_llm_provider,
key="supports_mid_conversation_system",
)
else [
self._system_role_message_as_user(m) if self._is_system_role_message(m) else m
for m in self._system_turns_after_tool_results(messages[leading_count:])
]
else list(convert_mid_conversation_system_turns(messages[leading_count:]))
)
if hoisted or remaining != messages:
anthropic_messages_request["messages"] = remaining
@ -278,7 +213,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
anthropic_messages_request.get("system"),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
for block in as_system_content_blocks(source)
]
filtered_system: Final = self._filter_billing_headers_from_system(system_content)
if filtered_system:

10
litellm/llms/bedrock/common_utils.py
View file

@ -34,6 +34,7 @@ if TYPE_CHECKING:
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
def error_response_text(response: httpx.Response) -> str:
@ -878,9 +879,10 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool:
"""
Whether Converse ``cachePoint`` blocks may be sent to this model.
Bedrock rejects requests carrying cachePoint blocks for models without prompt
caching support ("You invoked an unsupported model or your request did not allow
prompt caching"), so a model whose cost-map entry does not declare
OpenAI-family models only support implicit caching and never accept explicit
``cachePoint`` blocks. Bedrock rejects requests carrying cachePoint blocks for
models without prompt caching support ("You invoked an unsupported model or your
request did not allow prompt caching"), so a model whose cost-map entry does not declare
``supports_prompt_caching`` must not receive them. A model absent from the map
(an application inference profile ARN, a model newer than the map) keeps emitting
so existing caching setups never silently degrade. ``litellm.utils.supports_prompt_caching``
@ -888,6 +890,8 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool:
"""
if model is None:
return True
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
entries: Final = tuple(
entry
for candidate in (model, get_bedrock_base_model(model))

86
litellm/llms/bedrock/realtime/handler.py
View file

@ -6,11 +6,12 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
import asyncio
import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Mapping, MutableMapping
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, NoReturn, Protocol
from typing import Final, NoReturn, Protocol, runtime_checkable
from pydantic import JsonValue, TypeAdapter
@ -19,6 +20,8 @@ from litellm._logging import _redact_string, verbose_proxy_logger
from litellm.constants import (
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_DISTRIBUTION,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
@ -121,6 +124,38 @@ class BedrockBidirectionalStream(Protocol):
async def await_output(self) -> tuple[object, BedrockOutputStream]: ...
@runtime_checkable
class ClosableBedrockRuntimeClient(Protocol):
async def close(self) -> None: ...
def _installed_sdk_version() -> str | None:
try:
return importlib.metadata.version(BEDROCK_REALTIME_SDK_DISTRIBUTION)
except importlib.metadata.PackageNotFoundError:
return None
def _sdk_import_error(installed_version: str | None, cause: ImportError) -> ImportError:
install_hint: Final = "pip install 'litellm[bedrock-realtime]'"
requirement: Final = f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}"
verbose_proxy_logger.error("Bedrock Realtime: SDK import failed (installed=%s): %s", installed_version, cause)
if installed_version is None:
return ImportError(f"Missing aws_sdk_bedrock_runtime: {install_hint} ({requirement})")
return ImportError(
f"{BEDROCK_REALTIME_SDK_DISTRIBUTION} {installed_version} is installed but Bedrock realtime needs "
f"[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}: {install_hint}"
)
async def _close_bedrock_client(bedrock_client: object) -> None:
if not isinstance(bedrock_client, ClosableBedrockRuntimeClient):
return
with contextlib.suppress(Exception):
await bedrock_client.close()
verbose_proxy_logger.debug("Bedrock Realtime: closed SDK client")
@dataclass(frozen=True, slots=True)
class _BridgeOutcome:
logged_events: tuple[OpenAIRealtimeEvents, ...]
@ -199,8 +234,9 @@ async def _ack_session_update(
class BedrockRealtime(BaseAWSLLM):
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
def __init__(self):
def __init__(self, sdk_version_lookup: Callable[[], str | None] = _installed_sdk_version):
super().__init__()
self._sdk_version_lookup: Final = sdk_version_lookup
async def async_realtime(
self,
@ -234,14 +270,13 @@ class BedrockRealtime(BaseAWSLLM):
Various AWS authentication parameters
"""
try:
from aws_sdk_bedrock_runtime.client import (
BedrockRuntimeClient,
InvokeModelWithBidirectionalStreamOperationInput,
)
from aws_sdk_bedrock_runtime.config import Config
from smithy_aws_core.identity import StaticCredentialsResolver
except ImportError:
raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime")
from aws_sdk_bedrock_runtime.client import AsyncBedrockRuntimeClient
from aws_sdk_bedrock_runtime.config import AsyncBedrockRuntimeConfig
from aws_sdk_bedrock_runtime.models import InvokeModelWithBidirectionalStreamOperationInput
from smithy_aws_core.identity import AWSCredentialsIdentity, StaticCredentialsResolver
from smithy_http.aio.crt import AWSCRTHTTPClient
except ImportError as e:
raise _sdk_import_error(self._sdk_version_lookup(), e) from e
pending_session_update: Final = _pending_session_update(websocket.scope)
@ -285,22 +320,37 @@ class BedrockRealtime(BaseAWSLLM):
)
frozen_credentials: Final = await run_aws_signing(credentials.get_frozen_credentials)
# Initialize Bedrock client with aws_sdk_bedrock_runtime
config: Final = Config(
credentials_identity: Final = AWSCredentialsIdentity(
access_key_id=frozen_credentials.access_key,
secret_access_key=frozen_credentials.secret_key,
session_token=frozen_credentials.token,
)
config: Final = await AsyncBedrockRuntimeConfig.resolve(
endpoint_uri=endpoint_uri,
region=aws_region_name,
aws_access_key_id=frozen_credentials.access_key,
aws_secret_access_key=frozen_credentials.secret_key,
aws_session_token=frozen_credentials.token,
aws_credentials_identity_resolver=StaticCredentialsResolver(),
aws_credentials_identity_resolver=StaticCredentialsResolver(identity=credentials_identity),
transport=AWSCRTHTTPClient(),
)
bedrock_client: Final = BedrockRuntimeClient(config=config)
bedrock_client: Final = AsyncBedrockRuntimeClient(config=config)
async def open_bidirectional_stream() -> BedrockBidirectionalStream:
return await bedrock_client.invoke_model_with_bidirectional_stream(
InvokeModelWithBidirectionalStreamOperationInput(model_id=model)
)
try:
await self._run_session(websocket, open_bidirectional_stream, model, logging_obj, pending_session_update)
finally:
await _close_bedrock_client(bedrock_client)
async def _run_session(
self,
websocket: RealtimeClientWebSocket,
open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]],
model: str,
logging_obj: LiteLLMLogging,
pending_session_update: str | None,
) -> None:
transformation_config: Final = BedrockRealtimeConfig()
bedrock_stream: Final = await open_bidirectional_stream()

135
litellm/model_prices_and_context_window_backup.json
View file

@ -7605,7 +7605,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7733,7 +7733,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7887,7 +7887,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
@ -7956,7 +7956,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8856,7 +8856,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/us/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -8955,7 +8955,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/eu/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -9054,7 +9054,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
@ -10987,14 +10987,14 @@
"supports_vision": true
},
"azure_ai/FW-Kimi-K3": {
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.5e-05,
"reasoning_effort_levels": [
"low",
"high",
@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -45245,7 +45245,7 @@
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-20b": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@ -45482,6 +45482,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45503,7 +45504,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://api.together.xyz/v1/models",
"source": "https://api.together.ai/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
@ -45528,6 +45529,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 1.3e-07,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45552,7 +45554,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 3.9e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@ -45567,7 +45569,7 @@
"supports_vision": true
},
"together_ai/intfloat/multilingual-e5-large-instruct": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 2e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 514,
@ -45680,7 +45682,7 @@
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
@ -50573,7 +50575,7 @@
"wandb/openai/gpt-oss-120b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.7e-07,
@ -50584,7 +50586,7 @@
"wandb/openai/gpt-oss-20b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.3e-07,
@ -50593,6 +50595,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -50603,6 +50606,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"deprecation_date": "2026-08-04",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50612,6 +50616,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"deprecation_date": "2026-08-25",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50622,6 +50627,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"deprecation_date": "2026-08-04",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
@ -50632,6 +50638,7 @@
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"deprecation_date": "2026-03-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -50656,6 +50663,7 @@
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"deprecation_date": "2026-08-25",
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
@ -50670,7 +50678,7 @@
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"max_output_tokens": 128000,
"input_cost_per_token": 2.2e-07,
"output_cost_per_token": 2.2e-07,
@ -50690,6 +50698,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 161000,
"max_input_tokens": 161000,
@ -50700,6 +50709,7 @@
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"deprecation_date": "2026-03-04",
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
@ -50719,6 +50729,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"deprecation_date": "2026-04-21",
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
@ -50728,6 +50739,7 @@
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"deprecation_date": "2026-08-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -56692,7 +56704,8 @@
"supports_function_calling": true,
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true
"gemini_audio_only_live": true,
"supports_response_schema": false
},
"gemini-3.8-live-extended-thinking": {
"input_cost_per_audio_token": 3e-06,
@ -56726,7 +56739,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -60970,10 +60984,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -60983,7 +60998,7 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
@ -60996,10 +61011,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Pro": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.15e-06,
"output_cost_per_token": 2.55e-06,
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61009,7 +61025,7 @@
"wandb/google/gemma-4-31B-it": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3.4e-07,
"litellm_provider": "wandb",
@ -61018,8 +61034,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/ibm-granite/granite-4.1-8b": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61028,8 +61045,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61038,8 +61056,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-3.1-70B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"input_cost_per_token": 8e-07,
"output_cost_per_token": 8e-07,
"litellm_provider": "wandb",
@ -61050,7 +61069,7 @@
"wandb/MiniMaxAI/MiniMax-M3": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 9.6e-07,
"cache_read_input_token_cost": 5e-08,
@ -61063,7 +61082,7 @@
"wandb/moonshotai/Kimi-K2.7-Code": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 7.1e-07,
"output_cost_per_token": 3.5e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61076,7 +61095,7 @@
"wandb/moonshotai/Kimi-K2.6": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3.41e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61089,10 +61108,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 262000,
"input_cost_per_token": 7e-08,
"output_cost_per_token": 2e-07,
"cache_read_input_token_cost": 4e-08,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61102,10 +61121,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 7.5e-07,
"output_cost_per_token": 2.75e-06,
"cache_read_input_token_cost": 1.5e-07,
"max_input_tokens": 262000,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 1e-07,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61113,8 +61132,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/OpenPipe/Qwen3-14B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_input_tokens": 32800,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
@ -61125,7 +61145,7 @@
"wandb/Qwen/Qwen3.8-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61138,7 +61158,7 @@
"wandb/Qwen/Qwen3.6-35B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61149,10 +61169,11 @@
"wandb/Qwen/Qwen3.6-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3.6e-06,
"cache_read_input_token_cost": 1.2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61160,9 +61181,10 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3.5-35B-A3B": {
"deprecation_date": "2026-10-05",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61171,8 +61193,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"deprecation_date": "2026-10-05",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "wandb",
@ -61187,6 +61210,7 @@
"input_cost_per_token": 1.31e-06,
"output_cost_per_token": 3.96e-06,
"cache_read_input_token_cost": 4.4e-08,
"max_input_tokens": 1049000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
@ -61197,13 +61221,14 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 131000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-5.2": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 1049000,
"input_cost_per_token": 7.6e-07,
"output_cost_per_token": 2.42e-06,
"cache_read_input_token_cost": 1.4e-07,
@ -62866,7 +62891,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -69158,5 +69183,19 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_vision": true
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "wandb",
"max_input_tokens": 1049000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://wandb.ai/site/pricing/tokens/",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
}
}

6
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -38680,8 +38680,7 @@
"required": false,
"schema": {
"default": 10,
"maximum": 100,
"minimum": 1,
"minimum": 0,
"title": "Count",
"type": "integer"
}
@ -39385,8 +39384,7 @@
"required": false,
"schema": {
"default": 10,
"maximum": 100,
"minimum": 1,
"minimum": 0,
"title": "Count",
"type": "integer"
}

1
litellm/proxy/_types.py
View file

@ -850,6 +850,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/member_add",
"/team/member_delete",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/management/v1/teams/{team_id}/members/bulk_update",
"/team/member_update",
"/team/{team_id}/member/{user_id}/reset_spend",
"/team/permissions_list",

2
litellm/proxy/auth/route_checks.py
View file

@ -31,6 +31,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
# team
"/team/new",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/management/v1/teams/{team_id}/members/bulk_update",
"/team/update",
"/team/delete",
"/team/block",
@ -767,6 +768,7 @@ class RouteChecks:
"/user/bulk_update",
"/team/new",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/management/v1/teams/{team_id}/members/bulk_update",
"/team/update",
"/team/delete",
"/model/new",

4
litellm/proxy/client/cli/__init__.py
View file

@ -1,5 +1,5 @@
"""CLI package for LiteLLM Proxy Client."""
from .main import cli
from .main import cli, litellm_proxy_cli
__all__ = ["cli"]
__all__ = ["cli", "litellm_proxy_cli"]

4
litellm/proxy/client/cli/commands/encryption.py
View file

@ -36,8 +36,8 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool):
resumable; safe to re-run after an interruption.
Examples:
litellm-proxy encryption migrate --check # attestation scan, no writes
litellm-proxy encryption migrate # perform the migration
lite encryption migrate --check # attestation scan, no writes
lite encryption migrate # perform the migration
"""
client: Final = HTTPClient(ctx.obj["base_url"], ctx.obj["api_key"])

11
litellm/proxy/client/cli/main.py
View file

@ -168,5 +168,16 @@ cli.add_command(configure_group)
cli.add_command(unconfigure_group)
LITELLM_PROXY_DEPRECATION_NOTICE: Final = (
"The `litellm-proxy` command is deprecated and will be removed in a future release; "
"run `lite` instead, it takes the same commands and options."
)
def litellm_proxy_cli() -> None:
click.secho(LITELLM_PROXY_DEPRECATION_NOTICE, err=True, fg="yellow")
cli()
if __name__ == "__main__":
cli()

180
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -26,7 +26,6 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy._types import (
DB_RETRY_SAFE_ERROR_TYPES,
LiteLLM_BudgetTableFull,
LiteLLM_EndUserTable,
Litellm_EntityType,
LiteLLM_TeamTable,
LiteLLM_UserTable,
@ -193,13 +192,6 @@ def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]:
return (end_user_cache_key(row.user_id),)
def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float:
if not caps:
return 0.0
effective_budget_id: Final[str | None] = row.budget_id or litellm.max_end_user_budget_id
return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None)
def _budget_link_where(
budget_ids: Sequence[str],
extra: Mapping[str, object] = MappingProxyType({}),
@ -207,6 +199,19 @@ def _budget_link_where(
return {"budget_id": {"in": list(budget_ids)}, **extra}
def _enduser_invalidation_where(budget_ids: Sequence[str]) -> dict[str, object]:
"""Customers whose cached spend a committed reset of these tiers invalidated.
Mirrors ``_queue_enduser_resets`` without its ``spend > 0`` filter, which
post-commit would match nobody.
"""
linked: Final = _budget_link_where(budget_ids)
default_budget_id: Final = litellm.max_end_user_budget_id
if default_budget_id is None or default_budget_id not in budget_ids:
return linked
return {"OR": [linked, {"budget_id": None}]} # mutable-ok: prisma where filter must be a dict
def _queue_budget_linked_resets(
writes: LinkedSpendResetWrites,
cascade: "_BudgetCascade",
@ -265,16 +270,29 @@ class _BudgetCascade:
budgets: tuple[LiteLLM_BudgetTableFull, ...] = ()
budget_ids: tuple[str, ...] = ()
budget_resets: tuple[tuple[str, datetime], ...] = ()
endusers: tuple[_EndUserRow, ...] = ()
counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)
class _EndUserWalk:
"""Where the customer walk stands. ``cursor`` is None once it is done, and
``truncated`` says a failed page read cut it short of the tail."""
cursor: str | None = ""
invalidated: int = 0
truncated: bool = False
_ENDUSER_WALK_DONE: Final = _EndUserWalk(cursor=None)
@dataclass(frozen=True, slots=True)
class _BudgetCascadeCommitted:
cascade: _BudgetCascade
advanced: int
endusers: _EndUserWalk
@dataclass(frozen=True, slots=True)
@ -285,6 +303,8 @@ class _BudgetCascadeFailed:
_EMPTY_CASCADE: Final = _BudgetCascade()
_InvalidatedCache = Literal["spend counter", "user_api_key_cache"]
@dataclass(frozen=True, slots=True)
class _ChunkOutcome:
@ -416,10 +436,12 @@ _WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = (
)
def _budget_cascade_event_metadata(cascade: _BudgetCascade) -> dict[str, object]:
def _budget_cascade_event_metadata(
cascade: _BudgetCascade, endusers: _EndUserWalk = _ENDUSER_WALK_DONE
) -> dict[str, object]:
return {
"num_budgets_found": len(cascade.budgets),
"num_endusers_found": len(cascade.endusers),
"num_endusers_found": endusers.invalidated,
}
@ -593,6 +615,38 @@ class ResetBudgetJob:
e,
)
@staticmethod
async def _invalidate_caches(counter_keys: Sequence[str], cache_keys: Sequence[str]) -> None:
"""Batch twin of ``_invalidate_spend_counter`` and
``_invalidate_user_api_key_cache_entry``, after the commit like both:
one round trip per chunk where a tier's dependents are unbounded."""
await ResetBudgetJob._invalidate_cache("spend counter", counter_keys)
await ResetBudgetJob._invalidate_cache("user_api_key_cache", cache_keys)
@staticmethod
async def _invalidate_cache(cache: _InvalidatedCache, keys: Sequence[str]) -> None:
"""One cache's share of a batch, awaited separately so either failing
still leaves the other invalidated."""
if not keys:
return
try:
from litellm.proxy.proxy_server import spend_counter_cache, user_api_key_cache
match cache:
case "spend counter":
await spend_counter_cache.async_delete_cache_keys(keys)
case "user_api_key_cache":
await user_api_key_cache.async_delete_cache_keys(keys)
case _:
assert_never(cache)
except Exception as e:
verbose_proxy_logger.warning(
"Failed to invalidate %d %s entries: %s. Budgets may be over-enforced until they expire.",
len(keys),
cache,
e,
)
async def _fetch_linked_rows(
self,
table: SpendLinkedTable[_RowT],
@ -612,18 +666,57 @@ class ResetBudgetJob:
verbose_proxy_logger.warning("Failed to fetch %s for counter invalidation: %s", log_subject, e)
return ()
async def _collect_endusers_to_reset(self, budget_ids: Sequence[str]) -> tuple[_EndUserRow, ...]:
linked: Final[Sequence[_EndUserRow] | None] = await self._with_db_retry(
lambda: self.prisma_client.get_data(
table_name="enduser",
query_type="find_all",
budget_id_list=list(budget_ids),
),
reason="reset_budget_read_endusers_failure",
async def _invalidate_enduser_caches(self, budget_ids: Sequence[str]) -> _EndUserWalk:
"""Drop the cached spend of every customer the committed tier reset zeroed.
Paged like ``_reset_windows_for``, and capless for its reason too: the
customers on one tier are unbounded, and a cap cannot keep its position
across pod elections, so it would restart at the first customer forever.
"""
if not budget_ids:
return _ENDUSER_WALK_DONE
where: Final = _enduser_invalidation_where(budget_ids)
walk = _EndUserWalk()
while walk.cursor is not None:
walk = await self._invalidate_enduser_page(where=where, cursor=walk.cursor, reached=walk.invalidated)
return walk
async def _invalidate_enduser_page(self, where: Mapping[str, object], cursor: str, reached: int) -> _EndUserWalk:
"""Invalidate one page of customers and say where the walk goes next."""
try:
rows: Final = await self._fetch_enduser_page(where=where, cursor=cursor)
except Exception as e:
verbose_proxy_logger.warning(
"Failed to fetch end users for cache invalidation after %s customers (cursor %r): %s. "
"The customers past that page keep their cached spend until it expires.",
reached,
cursor,
e,
)
return _EndUserWalk(cursor=None, invalidated=reached, truncated=True)
if not rows:
return _EndUserWalk(cursor=None, invalidated=reached)
await self._invalidate_caches(
counter_keys=tuple(_enduser_counter_key(row) for row in rows),
cache_keys=tuple(key for row in rows for key in _enduser_cache_keys(row)),
)
walked: Final = reached + len(rows)
if len(rows) < RESET_BUDGET_JOB_BATCH_SIZE:
return _EndUserWalk(cursor=None, invalidated=walked)
return _EndUserWalk(cursor=rows[-1].user_id, invalidated=walked)
async def _fetch_enduser_page(self, where: Mapping[str, object], cursor: str) -> tuple[_EndUserRow, ...]:
"""One keyset page of customers, ordered by primary key so the cursor never repeats a row."""
return tuple(
await self._with_db_retry(
lambda: EndUserRepository(self.prisma_client).table.find_many(
where={**where, "user_id": {"gt": cursor}}, # mutable-ok: prisma where filter must be a dict
order={"user_id": "asc"}, # mutable-ok: prisma order filter must be a dict
take=RESET_BUDGET_JOB_BATCH_SIZE,
),
reason="reset_budget_read_endusers_failure",
)
)
if litellm.max_end_user_budget_id is None or litellm.max_end_user_budget_id not in budget_ids:
return tuple(linked or ())
return (*(linked or ()), *await self._get_endusers_with_no_budget_id())
async def _collect_budget_cascade(self, budgets_to_reset: Sequence[LiteLLM_BudgetTableFull]) -> _BudgetCascade:
"""Resolve every row the expiring budget tiers gate, before any write.
@ -670,7 +763,6 @@ class ResetBudgetJob:
if _rollover_enabled()
else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
)
endusers: Final[tuple[_EndUserRow, ...]] = await self._collect_endusers_to_reset(budget_ids)
return _BudgetCascade(
budgets=tuple(budgets_to_reset),
budget_ids=budget_ids,
@ -682,7 +774,6 @@ class ResetBudgetJob:
for b in budgets_to_reset
if b.budget_id is not None and b.budget_duration is not None
),
endusers=endusers,
counter_resets=(
*(
(_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps))
@ -695,7 +786,6 @@ class ResetBudgetJob:
(_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps))
for row in model_access_groups
),
*((_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) for row in endusers),
),
rollover_caps=rollover_caps,
cache_keys=(
@ -704,7 +794,6 @@ class ResetBudgetJob:
*(key for row in orgs for key in _org_cache_keys(row)),
*(key for row in tags for key in _tag_cache_keys(row)),
*(key for row in model_access_groups for key in _model_access_group_cache_keys(row)),
*(key for row in endusers for key in _enduser_cache_keys(row)),
),
)
@ -736,10 +825,10 @@ class ResetBudgetJob:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
for counter_key, _ in cascade.counter_resets:
await self._invalidate_spend_counter(counter_key)
for cache_key in cascade.cache_keys:
await self._invalidate_user_api_key_cache_entry(cache_key)
await self._invalidate_caches(
counter_keys=tuple(counter_key for counter_key, _ in cascade.counter_resets),
cache_keys=cascade.cache_keys,
)
async def _reset_expired_budget_cascade(self) -> _BudgetCascadeCommitted | _BudgetCascadeFailed:
now: Final = datetime.now(timezone.utc)
@ -769,6 +858,7 @@ class ResetBudgetJob:
(reset_at for _, reset_at in cascade.budget_resets),
cutoff=datetime.now(timezone.utc),
),
endusers=await self._invalidate_enduser_caches(cascade.budget_ids),
)
async def reset_budget_for_litellm_budget_table(self) -> None:
@ -788,7 +878,7 @@ class ResetBudgetJob:
end_time: Final = time.time()
match outcome:
case _BudgetCascadeCommitted(cascade=cascade, advanced=advanced):
case _BudgetCascadeCommitted() as committed:
asyncio.create_task(
self.proxy_logging_obj.service_logging_obj.async_service_success_hook(
service=ServiceTypes.RESET_BUDGET_JOB,
@ -797,13 +887,14 @@ class ResetBudgetJob:
start_time=start_time,
end_time=end_time,
event_metadata={
**_budget_cascade_event_metadata(cascade),
"num_endusers_updated": len(cascade.endusers),
**_budget_cascade_event_metadata(committed.cascade, committed.endusers),
"num_endusers_updated": committed.endusers.invalidated,
"num_endusers_failed": 0,
"enduser_invalidation_truncated": committed.endusers.truncated,
},
)
)
return _ChunkOutcome(fetched=len(cascade.budgets), advanced=advanced)
return _ChunkOutcome(fetched=len(committed.cascade.budgets), advanced=committed.advanced)
case _BudgetCascadeFailed(cascade=cascade, error=error):
verbose_proxy_logger.exception(
"Failed to reset the budget table cascade (team member, enduser, org, tag and model access "
@ -827,27 +918,6 @@ class ResetBudgetJob:
case _:
assert_never(outcome)
async def _get_endusers_with_no_budget_id(
self,
) -> list[LiteLLM_EndUserTable]:
"""
Fetch end users that have no explicit budget_id set (NULL) and have
accumulated spend > 0. These are implicitly-created end users that
rely on the default budget (litellm.max_end_user_budget_id) applied
in-memory during auth checks.
"""
table: Final = EndUserRepository(self.prisma_client).table
rows: Final = await self._with_db_retry(
lambda: table.find_many(
where={
"budget_id": None,
"spend": {"gt": 0},
},
),
reason="reset_budget_read_endusers_without_budget_id_failure",
)
return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows]
async def _write_key_reset_updates(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
"""
Write per-row {spend, budget_reset_at} updates for keys.

24
litellm/proxy/common_utils/timezone_utils.py
View file

@ -78,3 +78,27 @@ def get_budget_reset_time(budget_duration: str) -> datetime:
`BudgetResetSettings` by injection (creation/update endpoints, startup backfill).
"""
return compute_budget_reset_at(budget_duration, get_budget_reset_settings())
def _is_persistable_budget_duration(budget_duration: str) -> bool:
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
try:
if duration_in_seconds(budget_duration) <= 0:
return False
get_budget_reset_time(budget_duration=budget_duration)
except (ValueError, OverflowError):
return False
return True
def budget_duration_error(budget_duration: str | None) -> str | None:
"""Why `budget_duration` cannot be persisted, or None when it is usable.
A non-positive duration resolves to a reset time of "now", which leaves the row
permanently due: the reset job re-reads it every tick and, once enough of them
exist, they fill each batch and starve every other tenant's reset.
"""
if budget_duration is None or _is_persistable_budget_duration(budget_duration):
return None
return f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'."

19
litellm/proxy/common_utils/user_api_key_cache.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import re
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast, overload
@ -221,6 +222,24 @@ class UserApiKeyCache(DualCache):
return
await super().async_delete_cache(key)
async def async_delete_cache_keys(self, keys: Sequence[str]) -> None:
"""Batch twin of ``async_delete_cache``, partitioned like
``async_set_cache_pipeline``.
Both partitions are cleared even when one raises, because a caller
batching these has already committed the rows they cache.
"""
key_object_keys: Final = tuple(key for key in keys if is_user_key_cache_key(key))
other_keys: Final = tuple(key for key in keys if not is_user_key_cache_key(key))
outcomes: Final = await asyncio.gather(
self.key_object_cache.async_delete_cache_keys(key_object_keys),
super().async_delete_cache_keys(other_keys),
return_exceptions=True,
)
failed: Final = tuple(outcome for outcome in outcomes if isinstance(outcome, BaseException))
if failed:
raise failed[0]
def flush_cache(self) -> None:
super().flush_cache()
self.key_object_cache.in_memory_cache.flush_cache()

18
litellm/proxy/image_endpoints/endpoints.py
View file

@ -36,6 +36,10 @@ router: Final = APIRouter()
IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams))
IMAGE_ARRAY_FIELD: Final = "image[]"
MASK_ARRAY_FIELD: Final = "mask[]"
BRACKETED_FILE_FIELDS: Final = frozenset({IMAGE_ARRAY_FIELD, MASK_ARRAY_FIELD})
async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO:
"""
@ -244,9 +248,9 @@ async def image_edit_api(
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
image: list[UploadFile] | None = File(None),
image_array: list[UploadFile] | None = File(None, alias="image[]"),
image_array: list[UploadFile] | None = File(None, alias=IMAGE_ARRAY_FIELD),
mask: list[UploadFile] | None = File(None),
mask_array: list[UploadFile] | None = File(None, alias="mask[]"),
mask_array: list[UploadFile] | None = File(None, alias=MASK_ARRAY_FIELD),
model: str | None = None,
):
"""
@ -294,12 +298,14 @@ async def image_edit_api(
#########################################################
# Read request body and convert UploadFiles to BytesIO
#########################################################
data: Final = dict(
coerce_numeric_form_fields(
data: Final = {
key: value
for key, value in coerce_numeric_form_fields(
parsed_body=await _read_request_body(request=request),
numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS,
)
)
).items()
if key not in BRACKETED_FILE_FIELDS
}
image_files: Final = await batch_to_bytesio(image)
mask_files: Final = await batch_to_bytesio(mask)
if image_files:

61
litellm/proxy/management_endpoints/common_utils.py
View file

@ -1,5 +1,6 @@
import math
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Union
from fastapi import HTTPException, status
@ -33,23 +34,11 @@ def validate_budget_duration(budget_duration: str | None, status_code: int = 400
enough of them exist, they fill each batch and starve every other tenant's
reset.
"""
if budget_duration is None:
return
from litellm.proxy.common_utils.timezone_utils import budget_duration_error
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
try:
if duration_in_seconds(budget_duration) <= 0:
raise ValueError("budget_duration must be positive")
get_budget_reset_time(budget_duration=budget_duration)
except (ValueError, OverflowError):
raise HTTPException(
status_code=status_code,
detail={
"error": f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'."
},
)
error: Final = budget_duration_error(budget_duration)
if error is not None:
raise HTTPException(status_code=status_code, detail={"error": error})
from litellm._logging import verbose_proxy_logger
@ -492,6 +481,35 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = (
)
MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType(
{
"max_budget_in_team": "max_budget",
"tpm_limit": "tpm_limit",
"rpm_limit": "rpm_limit",
"budget_duration": "budget_duration",
"allowed_models": "allowed_models",
"temp_budget_increase": "temp_budget_increase",
"temp_budget_expiry": "temp_budget_expiry",
}
)
def _prisma_value(value: object) -> object:
return list(value) if isinstance(value, tuple) else value
def member_budget_patch(source: BaseModel) -> dict[str, Any]:
"""Map the per-member limit fields a request actually set to their budget-table
columns (merge-patch: a sent value updates, an explicit null clears, an absent
field is left untouched)."""
provided: Final = source.model_dump(exclude_unset=True)
return {
column: _prisma_value(provided[request_field])
for request_field, column in MEMBER_BUDGET_PATCH_FIELDS.items()
if request_field in provided
}
def _is_set_budget_value(value: object) -> bool:
if value is None:
return False
@ -515,6 +533,7 @@ async def _upsert_budget_and_membership(
user_api_key_dict: UserAPIKeyAuth,
budget_patch: dict[str, Any],
team_default_budget_id: str | None = None,
shared_budget_ids: frozenset[str] | None = None,
):
"""
Apply a merge-patch of per-member budget fields to a team membership.
@ -529,6 +548,10 @@ async def _upsert_budget_and_membership(
(from team metadata.team_member_budget_id). When the membership still
points at it, we clone-on-write so editing one member's budget does not
mutate the shared default that every other member points at.
``shared_budget_ids`` extends that protection to any other row more than one
membership points at, which a caller patching several members at once has
already counted; a row listed there is cloned rather than written in place.
"""
if not budget_patch:
return
@ -540,10 +563,8 @@ async def _upsert_budget_and_membership(
get_budget_reset_time(budget_duration=duration) if duration is not None else None
)
is_shared_default: Final = (
existing_budget_id is not None
and team_default_budget_id is not None
and existing_budget_id == team_default_budget_id
is_shared_default: Final = existing_budget_id is not None and (
existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset())
)
async def _disconnect():

82
litellm/proxy/management_endpoints/management_v1/teams.py
View file

@ -1,4 +1,4 @@
"""`POST /management/v1/teams/{team_id}/members/bulk_delete`."""
"""`POST /management/v1/teams/{team_id}/members/bulk_delete` and `.../members/bulk_update`."""
from typing import Annotated, Final
@ -9,12 +9,15 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.management_helpers.bulk_team_member_budgets import bulk_update_team_member_budgets
from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members
from litellm.proxy.management_helpers.utils import (
management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped
)
from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
from litellm.types.proxy.management_endpoints.team_endpoints import (
BulkTeamMemberBudgetUpdateRequest,
BulkTeamMemberBudgetUpdateResponse,
BulkTeamMemberDeleteRequest,
BulkTeamMemberDeleteResponse,
)
@ -92,3 +95,80 @@ async def bulk_delete_team_members_action(
detail="Failed to remove team members.",
)
)
@router.post(
"/teams/{team_id}/members/bulk_update",
tags=["team management"], # mutable-ok: FastAPI types `tags` as list[str], not Sequence
dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)),
response_model=BulkTeamMemberBudgetUpdateResponse,
)
@management_endpoint_wrapper
async def bulk_update_team_member_budgets_action(
team_id: str,
data: BulkTeamMemberBudgetUpdateRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> BulkTeamMemberBudgetUpdateResponse:
"""
Set per-member limits for up to 500 members of one team in one call. Same
authorization and member addressing as `/team/member_update`: proxy admins, the team's
admins, and admins of the team's organization, with each member named by exactly one of
`user_id` or `user_email`. Unknown body fields are a 422 and an unknown team is a 404.
Each row is a merge patch of that member's limits: a field left out is untouched, a
field sent as null is cleared, and clearing the last limit drops the member back to the
team default. A budget row shared by several memberships, the team default included, is
copied for the member being patched rather than written in place, so one member's new
cap never lands on anybody else.
`data` holds one result per requested member, in request order, carrying the limits in
force after the write. A row is `success: false` with an `error` when it names nobody on
the team or repeats an earlier row. Roles are not part of this route; `/team/member_update`
still owns them.
Example curl:
```
curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{"members": [{"user_id": "user-1", "max_budget_in_team": 10}, {"user_email": "user-2@example.com", "max_budget_in_team": 10, "budget_duration": "30d"}]}'
```
"""
try:
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
if prisma_client is None:
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}database-not-connected",
title="Database not connected",
status=503,
detail=CommonProxyErrors.db_not_connected_error.value,
)
)
results: Final = await bulk_update_team_member_budgets(
team_id=team_id,
data=data,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
)
return BulkTeamMemberBudgetUpdateResponse(data=results)
except ManagementProblem:
raise
except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape
verbose_proxy_logger.exception(
"litellm.proxy.management_endpoints.management_v1.teams.bulk_update_team_member_budgets_action(): "
"Exception occured - %s",
e,
)
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}internal-server-error",
title="Internal server error",
status=500,
detail="Failed to update team member budgets.",
)
)

16
litellm/proxy/management_endpoints/scim/scim_v2.py
View file

@ -264,6 +264,8 @@ scim_router: Final = APIRouter(
dependencies=[Depends(_premium_user_check)],
)
SCIM_MAX_PAGE_SIZE: Final = 100
# Helper functions for common operations
async def _get_prisma_client_or_raise_exception():
@ -1572,12 +1574,13 @@ def _parse_scim_eq_filter(scim_filter: str) -> tuple[str, str] | None:
)
async def get_users(
startIndex: int = Query(1, ge=1),
count: int = Query(10, ge=1, le=100),
count: int = Query(10, ge=0),
filter: str | None = Query(None),
):
"""
Get a list of users according to SCIM v2 protocol
"""
page_size: Final = min(count, SCIM_MAX_PAGE_SIZE)
verbose_proxy_logger.debug(
"SCIM GET USERS request: startIndex=%s count=%s filter=%s",
startIndex,
@ -1607,7 +1610,7 @@ async def get_users(
users: Final[Sequence[LiteLLM_UserTable]] = await _table(UserRepository(prisma_client)).find_many(
where=where_conditions,
skip=(startIndex - 1),
take=count,
take=page_size,
order={"created_at": "desc"},
)
@ -1623,7 +1626,7 @@ async def get_users(
return SCIMListResponse(
totalResults=total_count,
startIndex=startIndex,
itemsPerPage=min(count, len(scim_users)),
itemsPerPage=len(scim_users),
Resources=scim_users,
)
@ -2399,12 +2402,13 @@ class _TeamWhereConditions(TypedDict, total=False):
)
async def get_groups(
startIndex: int = Query(1, ge=1),
count: int = Query(10, ge=1, le=100),
count: int = Query(10, ge=0),
filter: str | None = Query(None),
):
"""
Get a list of groups according to SCIM v2 protocol
"""
page_size: Final = min(count, SCIM_MAX_PAGE_SIZE)
verbose_proxy_logger.debug(
"SCIM GET GROUPS request: startIndex=%s count=%s filter=%s",
startIndex,
@ -2425,7 +2429,7 @@ async def get_groups(
teams: Final = await _table(TeamRepository(prisma_client)).find_many(
where=where_conditions,
skip=(startIndex - 1),
take=count,
take=page_size,
order={"created_at": "desc"},
)
@ -2462,7 +2466,7 @@ async def get_groups(
return SCIMListResponse(
totalResults=total_count,
startIndex=startIndex,
itemsPerPage=min(count, len(scim_groups)),
itemsPerPage=len(scim_groups),
Resources=scim_groups,
)

26
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -129,6 +129,7 @@ from litellm.proxy.management_endpoints.common_utils import (
_update_metadata_fields,
_upsert_budget_and_membership,
_user_has_admin_view,
member_budget_patch,
validate_budget_duration,
validate_team_model_max_budget,
)
@ -3686,29 +3687,6 @@ async def team_member_delete(
return existing_team_row
_MEMBER_BUDGET_PATCH_FIELDS: Final = {
"max_budget_in_team": "max_budget",
"tpm_limit": "tpm_limit",
"rpm_limit": "rpm_limit",
"budget_duration": "budget_duration",
"allowed_models": "allowed_models",
"temp_budget_increase": "temp_budget_increase",
"temp_budget_expiry": "temp_budget_expiry",
}
def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> dict[str, object]:
"""Map the budget fields the request actually set (merge-patch: a sent
value updates, an explicit null clears, an absent field is left untouched)
to their budget-table columns."""
provided: Final = data.model_dump(exclude_unset=True)
return {
column: provided[request_field]
for request_field, column in _MEMBER_BUDGET_PATCH_FIELDS.items()
if request_field in provided
}
@router.post(
"/team/member_update",
tags=["team management"],
@ -3814,7 +3792,7 @@ async def team_member_update(
team_default_budget_id = raw_default_budget_id
### upsert new budget
budget_patch: Final = _build_member_budget_patch(data)
budget_patch: Final = member_budget_patch(data)
async with prisma_client.tx() as tx:
await _upsert_budget_and_membership(
tx=tx,

4
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -354,7 +354,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict:
status_code=400,
detail=(
"Your litellm CLI is out of date and uses a login flow this proxy no longer supports. "
"Upgrade it with `pip install -U 'litellm[proxy]'` and run `litellm-proxy login` again."
"Upgrade it with `pip install -U 'litellm[proxy]'` and run `lite login` again."
),
)
if not _is_valid_cli_sso_login_id(login_id):
@ -375,7 +375,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict:
raise HTTPException(
status_code=400,
detail=(
"CLI login session not found or expired. Run `litellm-proxy login` again. "
"CLI login session not found or expired. Run `lite login` again. "
"If this happens immediately after starting a login, the proxy is likely running multiple "
"replicas without a shared cache; configure a Redis cache "
"so every replica can see the login session."

192
litellm/proxy/management_helpers/bulk_team_member_budgets.py Normal file
View file

@ -0,0 +1,192 @@
"""Batched per-member limit writes behind `POST /management/v1/teams/{team_id}/members/bulk_update`.
Every read runs on the writer inside the batch transaction, so the write plan can never be
built from a lagging read replica. Any budget row that more than one membership points at,
the team's shared default included, is cloned before it is written, so raising one member's
cap never moves another member's.
"""
from collections.abc import Sequence
from datetime import timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import invalidate_team_member_spend_state
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.proxy.management_endpoints.common_utils import (
_is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_update uses
_is_user_team_admin, # pyright: ignore[reportPrivateUsage] # same check /team/member_update uses
_upsert_budget_and_membership, # pyright: ignore[reportPrivateUsage] # the single-member write, shared so the two surfaces cannot drift
member_budget_patch,
)
from litellm.proxy.management_helpers.bulk_user_deletion import (
_duplicate_member_indexes, # pyright: ignore[reportPrivateUsage] # same duplicate rule as members/bulk_delete
_eq_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete
_forbidden, # pyright: ignore[reportPrivateUsage] # same problem shape as members/bulk_delete
_in_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete
_team_not_found, # pyright: ignore[reportPrivateUsage] # same problem shape as members/bulk_delete
_team_users_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete
)
from litellm.proxy.utils import PrismaClient
from litellm.repositories.team_repository import TeamRepository
from litellm.types.proxy.management_endpoints.team_endpoints import (
BulkTeamMemberBudgetUpdateRequest,
TeamMemberBudgetPatch,
TeamMemberBudgetUpdateResult,
)
if TYPE_CHECKING:
from prisma import Prisma
from prisma import models as prisma_models
from litellm.repositories.prisma_protocols import TableActions
_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60)
_NO_METADATA: Final = MappingProxyType({})
_WITH_BUDGET: Final = MappingProxyType({"litellm_budget_table": True})
def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]":
return tx.litellm_teammembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do
def _budget_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_BudgetTable]":
return tx.litellm_budgettable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do
def _roster_user_id(member: TeamMemberBudgetPatch, roster: Sequence[Member]) -> str | None:
"""The team member this row addresses, or None when it names nobody on the team."""
if member.user_id is not None:
return member.user_id if any(m.user_id == member.user_id for m in roster) else None
return next((m.user_id for m in roster if m.user_email is not None and m.user_email == member.user_email), None)
def _team_default_budget_id(team: LiteLLM_TeamTable) -> str | None:
raw: Final = (team.metadata or _NO_METADATA).get("team_member_budget_id")
return raw if isinstance(raw, str) else None
async def _shared_budget_ids(tx: "Prisma", budget_ids: frozenset[str]) -> frozenset[str]:
"""The rows in ``budget_ids`` more than one membership points at, counted across every
team so a row shared with another team is protected too."""
if not budget_ids:
return frozenset()
rows: Final = await _membership_tx_db(tx).find_many(where=_in_filter("budget_id", budget_ids))
return frozenset(budget_id for budget_id in budget_ids if sum(1 for row in rows if row.budget_id == budget_id) > 1)
def _result(
member: TeamMemberBudgetPatch,
user_id: str | None,
error: str | None,
budget_of: "MappingProxyType[str, prisma_models.LiteLLM_BudgetTable | None]",
team_default_max_budget: float | None,
) -> TeamMemberBudgetUpdateResult:
if error is not None or user_id is None:
return TeamMemberBudgetUpdateResult(
user_id=member.user_id,
user_email=member.user_email,
success=False,
error=error or "User not found in team",
)
budget: Final = budget_of.get(user_id)
own_max_budget: Final = budget.max_budget if budget is not None else None
inherits: Final = own_max_budget is None and team_default_max_budget is not None and team_default_max_budget > 0
return TeamMemberBudgetUpdateResult(
user_id=user_id,
user_email=member.user_email,
success=True,
budget_id=budget.budget_id if budget is not None else None,
max_budget=team_default_max_budget if inherits else own_max_budget,
max_budget_source=("team_default" if inherits else "member" if own_max_budget is not None else None),
tpm_limit=budget.tpm_limit if budget is not None else None,
rpm_limit=budget.rpm_limit if budget is not None else None,
budget_duration=budget.budget_duration if budget is not None else None,
allowed_models=tuple(budget.allowed_models) if budget is not None else None,
)
async def bulk_update_team_member_budgets(
team_id: str,
data: BulkTeamMemberBudgetUpdateRequest,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
user_api_key_cache: UserApiKeyCache,
) -> tuple[TeamMemberBudgetUpdateResult, ...]:
"""Apply one merge patch of per-member limits per requested member, in one transaction."""
team: Final = await TeamRepository(WriterPinnedClient(prisma_client.db)).find_by_id(team_id)
if team is None:
raise _team_not_found(team_id)
if (
user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team)
and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team)
):
raise _forbidden(
"Call not allowed. User not proxy admin OR team admin OR org admin for this team. "
f"route='/management/v1/teams/{team_id}/members/bulk_update'"
)
roster: Final = team.members_with_roles or ()
named: Final = tuple(_roster_user_id(member, roster) for member in data.members)
duplicates: Final = _duplicate_member_indexes(data.members) | frozenset(
index for index, user_id in enumerate(named) if user_id is not None and user_id in named[:index]
)
applied: Final = tuple(
(index, user_id) for index, user_id in enumerate(named) if user_id is not None and index not in duplicates
)
if not applied:
return tuple(
_result(
member, None, "Duplicate member in request" if index in duplicates else None, MappingProxyType({}), None
)
for index, member in enumerate(data.members)
)
user_ids: Final = sorted(user_id for _, user_id in applied)
default_budget_id: Final = _team_default_budget_id(team)
team_members_filter: Final = _team_users_filter(team_id, user_ids)
async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx:
memberships: Final = await _membership_tx_db(tx).find_many(where=team_members_filter)
budget_id_of: Final = MappingProxyType({m.user_id: m.budget_id for m in memberships})
shared: Final = await _shared_budget_ids(
tx, frozenset(budget_id for budget_id in budget_id_of.values() if budget_id is not None)
)
for index, user_id in applied:
await _upsert_budget_and_membership(
tx=tx,
team_id=team_id,
user_id=user_id,
existing_budget_id=budget_id_of.get(user_id),
user_api_key_dict=user_api_key_dict,
budget_patch=member_budget_patch(data.members[index]),
team_default_budget_id=default_budget_id,
shared_budget_ids=shared,
)
written: Final = await _membership_tx_db(tx).find_many(where=team_members_filter, include=_WITH_BUDGET)
team_default: Final = (
await _budget_tx_db(tx).find_unique(where=_eq_filter("budget_id", default_budget_id))
if default_budget_id is not None
else None
)
for user_id in user_ids:
await invalidate_team_member_spend_state(
user_id=user_id, team_id=team_id, user_api_key_cache=user_api_key_cache
)
budget_of: Final = MappingProxyType({m.user_id: m.litellm_budget_table for m in written})
return tuple(
_result(
member,
named[index],
"Duplicate member in request" if index in duplicates else None,
budget_of,
team_default.max_budget if team_default is not None else None,
)
for index, member in enumerate(data.members)
)

56
litellm/types/proxy/management_endpoints/team_endpoints.py
View file

@ -1,6 +1,6 @@
from typing import Any, Final, Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from litellm.proxy._types import (
KeyManagementRoutes,
@ -10,12 +10,15 @@ from litellm.proxy._types import (
Member,
MemberDeleteRequest,
)
from litellm.proxy.common_utils.timezone_utils import budget_duration_error
from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse
TeamIdSearchMatch = Literal["exact", "prefix"]
MAX_BULK_TEAM_MEMBER_DELETES: Final = 500
MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES: Final = 500
class GetTeamMemberPermissionsRequest(BaseModel):
"""Request to get the team member permissions for a team"""
@ -123,7 +126,7 @@ class BulkTeamMemberAddResponse(BaseModel):
class TeamMemberRef(MemberDeleteRequest):
"""One member to remove, named by exactly one of `user_id` or `user_email`."""
"""One member, named by exactly one of `user_id` or `user_email`."""
model_config = ConfigDict(extra="forbid")
@ -155,6 +158,55 @@ class BulkTeamMemberDeleteResponse(ResourceResponse[tuple[TeamMemberDeleteResult
"""`{data: [...]}` with one `TeamMemberDeleteResult` per requested member, in request order."""
class TeamMemberBudgetPatch(TeamMemberRef):
"""One member's per-member limits, merge-patch style: a field left out of the row is
untouched, a field sent as null is cleared, and clearing the last limit drops the
member back to the team default."""
max_budget_in_team: float | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
budget_duration: str | None = None
allowed_models: tuple[str, ...] | None = None
@field_validator("budget_duration")
@classmethod
def persistable_budget_duration(cls, value: str | None) -> str | None:
error: Final = budget_duration_error(value)
if error is not None:
raise ValueError(error)
return value
class BulkTeamMemberBudgetUpdateRequest(BaseModel):
"""Body of `POST /management/v1/teams/{team_id}/members/bulk_update`."""
model_config = ConfigDict(extra="forbid")
members: tuple[TeamMemberBudgetPatch, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES)
class TeamMemberBudgetUpdateResult(BaseModel):
"""Outcome for one requested member, in request order, carrying the limits in force
after the write rather than the ones that were asked for."""
user_id: str | None = None
user_email: str | None = None
success: bool
error: str | None = None
budget_id: str | None = None
max_budget: float | None = None
max_budget_source: Literal["member", "team_default"] | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
budget_duration: str | None = None
allowed_models: tuple[str, ...] | None = None
class BulkTeamMemberBudgetUpdateResponse(ResourceResponse[tuple[TeamMemberBudgetUpdateResult, ...]]):
"""`{data: [...]}` with one `TeamMemberBudgetUpdateResult` per requested member, in request order."""
class TeamMemberInfoResponse(LiteLLM_TeamMembership):
"""Response for GET /team/{team_id}/members/me — caller's own membership row."""

9
litellm/utils.py
View file

@ -2916,6 +2916,15 @@ def supports_none_reasoning_effort(model: str, custom_llm_provider: str | None =
return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_none_reasoning_effort")
def supports_mid_conversation_system(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model accepts a system role message after the leading system block and return a boolean value.
"""
return _supports_factory(
model=model, custom_llm_provider=custom_llm_provider, key="supports_mid_conversation_system"
)
def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model supports native structured outputs and return a boolean value.

135
model_prices_and_context_window.json
View file

@ -7605,7 +7605,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7733,7 +7733,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -7887,7 +7887,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
@ -7956,7 +7956,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8856,7 +8856,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/us/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -8955,7 +8955,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/eu/gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5.5e-07,
@ -9054,7 +9054,7 @@
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
},
"azure/gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
@ -10987,14 +10987,14 @@
"supports_vision": true
},
"azure_ai/FW-Kimi-K3": {
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.5e-05,
"reasoning_effort_levels": [
"low",
"high",
@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": false
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -45245,7 +45245,7 @@
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-20b": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 131072,
@ -45482,6 +45482,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45503,7 +45504,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://api.together.xyz/v1/models",
"source": "https://api.together.ai/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
@ -45528,6 +45529,7 @@
},
"together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 1.3e-07,
"deprecation_date": "2026-09-29",
"input_cost_per_token": 1.32e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
@ -45552,7 +45554,7 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 3.9e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
@ -45567,7 +45569,7 @@
"supports_vision": true
},
"together_ai/intfloat/multilingual-e5-large-instruct": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"input_cost_per_token": 2e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 514,
@ -45680,7 +45682,7 @@
"supports_tool_choice": true
},
"together_ai/thinkingmachines/Inkling-Small": {
"deprecation_date": "2026-09-15",
"deprecation_date": "2026-09-14",
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "together_ai",
@ -50573,7 +50575,7 @@
"wandb/openai/gpt-oss-120b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.7e-07,
@ -50584,7 +50586,7 @@
"wandb/openai/gpt-oss-20b": {
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"max_output_tokens": 131072,
"input_cost_per_token": 3e-08,
"output_cost_per_token": 1.3e-07,
@ -50593,6 +50595,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-4.5": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -50603,6 +50606,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"deprecation_date": "2026-08-04",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50612,6 +50616,7 @@
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"deprecation_date": "2026-08-25",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -50622,6 +50627,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"deprecation_date": "2026-08-04",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
@ -50632,6 +50638,7 @@
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"deprecation_date": "2026-03-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -50656,6 +50663,7 @@
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"deprecation_date": "2026-08-25",
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
@ -50670,7 +50678,7 @@
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"max_output_tokens": 128000,
"input_cost_per_token": 2.2e-07,
"output_cost_per_token": 2.2e-07,
@ -50690,6 +50698,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"deprecation_date": "2026-03-04",
"supports_reasoning": true,
"max_tokens": 161000,
"max_input_tokens": 161000,
@ -50700,6 +50709,7 @@
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"deprecation_date": "2026-03-04",
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
@ -50719,6 +50729,7 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"deprecation_date": "2026-04-21",
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
@ -50728,6 +50739,7 @@
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"deprecation_date": "2026-08-04",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
@ -56692,7 +56704,8 @@
"supports_function_calling": true,
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true
"gemini_audio_only_live": true,
"supports_response_schema": false
},
"gemini-3.8-live-extended-thinking": {
"input_cost_per_audio_token": 3e-06,
@ -56726,7 +56739,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -60970,10 +60984,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -60983,7 +60998,7 @@
"wandb/deepseek-ai/DeepSeek-V4-Flash-0731": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 2.8e-07,
"cache_read_input_token_cost": 7e-08,
@ -60996,10 +61011,11 @@
"wandb/deepseek-ai/DeepSeek-V4-Pro": {
"supports_reasoning": true,
"max_tokens": 1048576,
"max_input_tokens": 1048576,
"max_input_tokens": 1049000,
"input_cost_per_token": 1.15e-06,
"output_cost_per_token": 2.55e-06,
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61009,7 +61025,7 @@
"wandb/google/gemma-4-31B-it": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3.4e-07,
"litellm_provider": "wandb",
@ -61018,8 +61034,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/ibm-granite/granite-4.1-8b": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61028,8 +61045,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_input_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 1e-07,
"litellm_provider": "wandb",
@ -61038,8 +61056,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/meta-llama/Llama-3.1-70B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"input_cost_per_token": 8e-07,
"output_cost_per_token": 8e-07,
"litellm_provider": "wandb",
@ -61050,7 +61069,7 @@
"wandb/MiniMaxAI/MiniMax-M3": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 9.6e-07,
"cache_read_input_token_cost": 5e-08,
@ -61063,7 +61082,7 @@
"wandb/moonshotai/Kimi-K2.7-Code": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 7.1e-07,
"output_cost_per_token": 3.5e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61076,7 +61095,7 @@
"wandb/moonshotai/Kimi-K2.6": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3.41e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61089,10 +61108,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 262000,
"input_cost_per_token": 7e-08,
"output_cost_per_token": 2e-07,
"cache_read_input_token_cost": 4e-08,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61102,10 +61121,10 @@
"wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 7.5e-07,
"output_cost_per_token": 2.75e-06,
"cache_read_input_token_cost": 1.5e-07,
"max_input_tokens": 262000,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 1e-07,
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61113,8 +61132,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/OpenPipe/Qwen3-14B-Instruct": {
"deprecation_date": "2026-10-05",
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_input_tokens": 32800,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2.2e-07,
"litellm_provider": "wandb",
@ -61125,7 +61145,7 @@
"wandb/Qwen/Qwen3.8-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 1.5e-07,
@ -61138,7 +61158,7 @@
"wandb/Qwen/Qwen3.6-35B-A3B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61149,10 +61169,11 @@
"wandb/Qwen/Qwen3.6-27B": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3.6e-06,
"cache_read_input_token_cost": 1.2e-07,
"deprecation_date": "2026-10-05",
"supports_prompt_caching": true,
"litellm_provider": "wandb",
"mode": "chat",
@ -61160,9 +61181,10 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3.5-35B-A3B": {
"deprecation_date": "2026-10-05",
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 1.25e-06,
"litellm_provider": "wandb",
@ -61171,8 +61193,9 @@
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"deprecation_date": "2026-10-05",
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 262000,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 3e-07,
"litellm_provider": "wandb",
@ -61187,6 +61210,7 @@
"input_cost_per_token": 1.31e-06,
"output_cost_per_token": 3.96e-06,
"cache_read_input_token_cost": 4.4e-08,
"max_input_tokens": 1049000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
@ -61197,13 +61221,14 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 5e-08,
"max_input_tokens": 131000,
"supports_prompt_caching": true,
"source": "https://wandb.ai/site/pricing/tokens/"
},
"wandb/zai-org/GLM-5.2": {
"supports_reasoning": true,
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_input_tokens": 1049000,
"input_cost_per_token": 7.6e-07,
"output_cost_per_token": 2.42e-06,
"cache_read_input_token_cost": 1.4e-07,
@ -62866,7 +62891,7 @@
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"source": "https://docs.fireworks.ai/serverless/pricing",
"source": "https://api.fireworks.ai/v1/serverless/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -69158,5 +69183,19 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_vision": true
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "wandb",
"max_input_tokens": 1049000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://wandb.ai/site/pricing/tokens/",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
}
}

9
pyproject.toml
View file

@ -67,7 +67,7 @@ proxy = [
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=1.28.1,<2.0",
"litellm-proxy-extras==0.4.98",
"litellm-proxy-extras==0.4.99",
"litellm-enterprise==0.1.68",
"RestrictedPython>=8.5,<9.0",
"rich>=13.9.4,<14.0",
@ -143,8 +143,9 @@ bedrock-realtime = [
# InvokeModelWithBidirectionalStream API, which boto3 cannot do. This
# experimental AWS SDK (with its smithy-* deps, pulled transitively)
# provides the bidirectional stream; imported lazily in the realtime
# handler so litellm core stays usable without it.
"aws-sdk-bedrock-runtime>=0.7.0,<0.8.0; python_version >= '3.12'",
# handler so litellm core stays usable without it. The awscrt extra is
# required: the SDK's default aiohttp transport has no duplex streaming.
"aws-sdk-bedrock-runtime[awscrt]>=0.10.0,<0.12.0; python_version >= '3.12'",
]
proxy-runtime = [
# Historically bundled in the proxy Docker images via requirements.txt.
@ -173,7 +174,7 @@ proxy-runtime = [
[project.scripts]
litellm = "litellm:run_server"
lite = "litellm.proxy.client.cli:cli"
litellm-proxy = "litellm.proxy.client.cli:cli"
litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli"
[dependency-groups]
dev = [

3
tests/integration/_support/client.py
View file

@ -10,9 +10,10 @@ from hashlib import sha256
from typing import Final, TypeVar
import httpx
from integration._support.database import read_rows
from pydantic import JsonValue, TypeAdapter
from tests.integration._support.database import read_rows
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
T = TypeVar("T")

2
tests/integration/_support/generation.py
View file

@ -6,7 +6,7 @@ from contextlib import contextmanager
import httpx
from hypothesis import Phase, settings
from integration._support.client import Gateway
from tests.integration._support.client import Gateway
LIFECYCLE_SETTINGS: Final = settings(
max_examples=20,

6
tests/integration/authorization/test_warmed_policy.py
View file

@ -10,9 +10,9 @@ from pydantic import JsonValue
from hypothesis import strategies as st
from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
from tests.integration._support.client import Gateway, eventually, object_value
from tests.integration._support.database import read_rows
from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
def assert_serving(gateway: Gateway, model: str, key: str, status: int, error_type: str = "auth_error") -> None:

4
tests/integration/configuration/test_effective_settings.py
View file

@ -4,8 +4,8 @@ from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, object_value, string_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, object_value, string_value
from tests.integration._support.database import read_rows
def model_identity(gateway: Gateway, alias: str) -> str:

6
tests/integration/conftest.py
View file

@ -12,9 +12,9 @@ import pytest
import httpx
from redis import Redis
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.manifest import OWNED_DIRECTORIES, contracts
from integration._support.generation import LIFECYCLE_SETTINGS
from tests.integration._support.client import Gateway, eventually, gateway_from_environment
from tests.integration._support.manifest import OWNED_DIRECTORIES, contracts
from tests.integration._support.generation import LIFECYCLE_SETTINGS
COLLECTED: Final = pytest.StashKey[tuple[str, ...]]()
REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]()

4
tests/integration/management/test_key_updates.py
View file

@ -3,8 +3,8 @@ from hashlib import sha256
import pytest
from integration._support.client import Gateway, object_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, object_value
from tests.integration._support.database import read_rows
@pytest.mark.covers("mgmt.key.update.preserves_independent_fields")

7
tests/integration/management/test_partial_update_sequences.py
View file

@ -5,11 +5,12 @@ from typing import Final
import pytest
from hypothesis import strategies as st
from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test
from integration._support.client import Gateway, object_value
from integration._support.database import read_rows
from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
from pydantic import JsonValue
from tests.integration._support.client import Gateway, object_value
from tests.integration._support.database import read_rows
from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests
def _key_rows(digest: str) -> list[dict[str, JsonValue]]:
return read_rows(

4
tests/integration/pricing/test_configured_prices.py
View file

@ -6,8 +6,8 @@ import uuid
import pytest
import yaml
from integration._support.client import Gateway, eventually, object_value, string_value
from integration._support.database import read_rows
from tests.integration._support.client import Gateway, eventually, object_value, string_value
from tests.integration._support.database import read_rows
@pytest.mark.covers("quota_management.spend_tracking.custom_price.matches_input_rates")

2
tests/integration/providers/test_request_boundary.py
View file

@ -3,7 +3,7 @@ from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, JSON_OBJECT, object_value
from tests.integration._support.client import Gateway, JSON_OBJECT, object_value
@pytest.mark.covers("other.provider_wire.internal_parameters_filtered")

22
tests/litellm_utils_tests/test_proxy_budget_reset.py
View file

@ -102,21 +102,24 @@ def _wire_batcher_for_test(prisma_client, fail_commit=False):
return batch_calls
def _wire_cascade_reads_for_test(prisma_client):
def _wire_cascade_reads_for_test(prisma_client, endusers=()):
"""
The budget tier's cascade reads the rows it is about to zero, so their
spend counters can be invalidated after the commit. Give each of those
tables an awaitable find_many so the reads resolve instead of falling into
the job's warn-and-continue path.
End users are read by the post-commit invalidation walk rather than by
``get_data``, so callers that care about customers pass them here.
"""
for table in (
"litellm_teammembership",
"litellm_verificationtoken",
"litellm_organizationtable",
"litellm_tagtable",
"litellm_endusertable",
):
getattr(prisma_client.db, table).find_many = AsyncMock(return_value=[])
prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=list(endusers))
@pytest.mark.asyncio
@ -556,7 +559,7 @@ async def test_reset_budget_continues_other_categories_on_failure():
**{u["user_id"]: u["spend"] for u in [user2]},
**{t["team_id"]: t["spend"] for t in [team1, team2]},
}
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=[enduser1])
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -607,7 +610,10 @@ async def test_reset_budget_continues_other_categories_on_failure():
called_tables = {
call.kwargs.get("table_name") for call in prisma_client.get_data.await_args_list
}
assert called_tables == {"key", "user", "team", "budget", "enduser"}
assert called_tables == {"key", "user", "team", "budget"}
# Customers are not part of that set: the cascade zeroes them by budget link
# and reads them only afterwards, to invalidate their cached spend.
prisma_client.db.litellm_endusertable.find_many.assert_awaited()
# Every category writes through the batch path now, so update_data is unused.
prisma_client.update_data.assert_not_awaited()
@ -1029,7 +1035,7 @@ async def test_service_logger_endusers_success():
prisma_client.get_data = AsyncMock(side_effect=fake_get_data)
prisma_client.update_data = AsyncMock()
batch_calls = _wire_batcher_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=endusers)
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -1094,7 +1100,7 @@ async def test_service_logger_endusers_failure():
prisma_client.get_data = AsyncMock(side_effect=fake_get_data)
prisma_client.update_data = AsyncMock()
_wire_batcher_for_test(prisma_client, fail_commit=True)
_wire_cascade_reads_for_test(prisma_client)
_wire_cascade_reads_for_test(prisma_client, endusers=endusers)
proxy_logging_obj = MagicMock()
proxy_logging_obj.service_logging_obj = MagicMock()
@ -1121,7 +1127,9 @@ async def test_service_logger_endusers_failure():
) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args
event_metadata = kwargs.get("event_metadata", {})
assert event_metadata.get("num_budgets_found") == len(budgets)
assert event_metadata.get("num_endusers_found") == len(endusers)
# Customers are read by the post-commit invalidation walk, which a failed
# commit never reaches, so a failure reports none touched.
assert event_metadata.get("num_endusers_found") == 0
assert "endusers_found" not in event_metadata
assert "budgets_found" not in event_metadata
proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()

4
tests/otel_tests/test_e2e_budgeting.py
View file

@ -367,7 +367,7 @@ async def obtain_cli_sso_token_via_poll_flow(
models: list[str],
) -> str:
"""
Obtain a CLI SSO JWT through the same HTTP flow as `litellm-proxy login`:
Obtain a CLI SSO JWT through the same HTTP flow as `lite login`:
/sso/cli/start -> (SSO callback) -> /sso/cli/complete -> /sso/cli/poll.
When the proxy SSO session cache is not shared with the test runner (otel CI
@ -551,7 +551,7 @@ async def test_team_budget_enforcement():
@pytest.mark.asyncio
async def test_team_budget_enforcement_cli_sso_token():
"""
Team budget enforcement for CLI SSO session tokens (litellm-proxy login JWT).
Team budget enforcement for CLI SSO session tokens (lite login JWT).
1. Create team with a tiny max_budget and a user on that team
2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint)

8
tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py
View file

@ -20,7 +20,9 @@ import pytest
IMAGE: Final = os.getenv("LITELLM_IMAGE")
NON_ROOT_UID: Final = "12345:0"
IMPORT_PROBE: Final = "import aws_sdk_bedrock_runtime, smithy_aws_core; print('bedrock-realtime ok')"
IMPORT_PROBE: Final = (
"import aws_sdk_bedrock_runtime, smithy_aws_core, smithy_http.aio.crt; print('bedrock-realtime ok')"
)
pytestmark = [
pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"),
@ -52,7 +54,7 @@ def test_image_imports_bedrock_realtime_sdk():
)
assert probe.returncode == 0 and "bedrock-realtime ok" in probe.stdout, (
f"{IMAGE} cannot import aws_sdk_bedrock_runtime as uid {NON_ROOT_UID}, so Bedrock Nova Sonic "
"/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'. Is `--extra bedrock-realtime` "
f"{IMAGE} cannot import aws_sdk_bedrock_runtime with its awscrt transport as uid {NON_ROOT_UID}, so "
"Bedrock Nova Sonic /v1/realtime sessions fail at SDK import. Is `--extra bedrock-realtime` "
f"passed to every `uv sync` in its Dockerfile?\nstdout:\n{probe.stdout}\nstderr:\n{probe.stderr}"
)

32
tests/test_litellm/caching/test_dual_cache.py
View file

@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.constants import DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache, _redis_circuit_breaker_guard, _redis_circuit_breaker_guard_sync
@ -759,3 +760,34 @@ async def test_redis_timeouts_falling_back_to_memory_log_once_per_interval(caplo
" (199 more Redis timeouts since the previous Redis timeout line were logged at DEBUG)",
)
]
@pytest.mark.asyncio
async def test_async_delete_cache_keys_drops_memory_and_chunks_redis():
"""Batch delete clears both layers, and chunks Redis so one caller's large
key list cannot become a single oversized DELETE command."""
redis_cache = MagicMock(spec=RedisCache)
redis_cache.delete_cache_keys = AsyncMock()
dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
keys = [f"key-{i}" for i in range(DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE + 7)]
for key in keys:
dual_cache.in_memory_cache.set_cache(key=key, value=1)
await dual_cache.async_delete_cache_keys(keys)
assert all(dual_cache.in_memory_cache.get_cache(key=key) is None for key in keys)
sent = [call.args[0] for call in redis_cache.delete_cache_keys.await_args_list]
assert [len(chunk) for chunk in sent] == [DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE, 7]
assert [key for chunk in sent for key in chunk] == keys
@pytest.mark.asyncio
async def test_async_delete_cache_keys_on_empty_list_touches_no_backend():
"""An empty page must not reach Redis: DELETE with no arguments is an error."""
redis_cache = MagicMock(spec=RedisCache)
redis_cache.delete_cache_keys = AsyncMock()
dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
await dual_cache.async_delete_cache_keys([])
redis_cache.delete_cache_keys.assert_not_awaited()

162
tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
View file

@ -23,6 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
create_tool_name_mapping,
truncate_tool_name,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
CONVERTED_SYSTEM_NOTE,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.anthropic import (
AnthopicMessagesAssistantMessageParam,
@ -563,10 +566,19 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
@pytest.mark.parametrize(
("system_content", "expected_content"),
[
("Use the corrected result.", "Use the corrected result."),
(
"Use the corrected result.",
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[{"type": "text", "text": "Use the corrected result."}],
[{"type": "text", "text": "Use the corrected result."}],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[
@ -576,7 +588,11 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
},
{"type": "text", "text": "Use the corrected result."},
],
[{"type": "text", "text": "Use the corrected result."}],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
{"type": "text", "text": "Use the corrected result."},
],
),
(
[
@ -584,13 +600,14 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement():
{"type": "text", "text": "Second correction."},
],
[
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "First correction."},
{"type": "text", "text": "Second correction."},
],
),
],
)
def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correction(
def test_translate_anthropic_messages_to_openai_converts_midturn_system_correction(
system_content: object,
expected_content: object,
):
@ -646,7 +663,7 @@ def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correct
"tool_call_id": "toolu_01234",
"content": "Rainy, 55°F",
},
{"role": "system", "content": expected_content},
{"role": "user", "content": expected_content},
{"role": "user", "content": "Continue."},
]
@ -752,8 +769,8 @@ def test_translate_anthropic_messages_to_openai_drops_empty_midturn_system(
def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
"""
Request level: the trusted top-level prompt is hoisted to index 0 exactly once and the
in-sequence correction keeps its own position and `role: "system"` -- no duplication of
either, and no reordering of the surrounding turns.
in-sequence correction keeps its own position as a user turn prefixed with the operator
note -- no duplication of either, and no reordering of the surrounding turns.
"""
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={
@ -773,11 +790,140 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
{"role": "system", "content": "Trusted top-level prompt."},
{"role": "user", "content": "First question."},
{"role": "assistant", "content": "First answer.", "thinking_blocks": None},
{"role": "system", "content": "Use the corrected result."},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Use the corrected result."},
],
},
{"role": "user", "content": "Continue."},
]
_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST: Final = {
"max_tokens": 128,
"system": [{"type": "text", "text": "You are Claude Code."}],
"messages": [
{"role": "user", "content": "say hi"},
{
"role": "system",
"content": [{"type": "text", "text": "<system-reminder>Keep answers to one sentence.</system-reminder>"}],
},
{"role": "assistant", "content": "Hi."},
{"role": "user", "content": "say bye"},
],
}
@pytest.mark.parametrize("custom_llm_provider", [None, "hosted_vllm"])
def test_translate_anthropic_to_openai_converts_claude_code_midturn_system_turn(custom_llm_provider: str | None):
"""
Claude Code appends a system-role harness reminder after the user turn. On a chat-completions
target that does not declare ``supports_mid_conversation_system`` (a self-hosted model the cost
map knows nothing about) the outbound request must have exactly one system message, at index 0,
and the converted turn must carry the operator note first.
"""
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
custom_llm_provider=custom_llm_provider,
)
roles = [m["role"] for m in openai_request["messages"]]
assert roles == ["system", "user", "user", "assistant", "user"]
converted = openai_request["messages"][2]
assert converted["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
assert converted["content"][1]["text"] == "<system-reminder>Keep answers to one sentence.</system-reminder>"
def test_translate_anthropic_to_openai_keeps_midturn_system_when_target_declares_support(monkeypatch):
"""
A chat-completions target flagged ``supports_mid_conversation_system`` in the cost map accepts
the role anywhere, so the harness reminder is forwarded in place with its role and content
untouched, the same rule the native Anthropic Messages path applies.
"""
model: Final = "system-role-anywhere-chat-model"
monkeypatch.setitem(
litellm.model_cost,
model,
{"litellm_provider": "openai", "mode": "chat", "supports_mid_conversation_system": True},
)
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={"model": model, **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
custom_llm_provider="openai",
)
assert openai_request["messages"] == [
{"role": "system", "content": [{"type": "text", "text": "You are Claude Code."}]},
{"role": "user", "content": "say hi"},
{
"role": "system",
"content": [{"type": "text", "text": "<system-reminder>Keep answers to one sentence.</system-reminder>"}],
},
{"role": "assistant", "content": "Hi.", "thinking_blocks": None},
{"role": "user", "content": "say bye"},
]
def test_translate_anthropic_to_openai_moves_midturn_system_after_tool_result():
"""
A system entry wedged between an assistant tool_use turn and its tool_result turn is
emitted after the role: "tool" message, so the tool call stays paired with its result.
"""
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=[
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "toolu_01234",
"name": "get_weather",
"input": {"location": "Boston"},
}
],
},
{"role": "system", "content": "Use the corrected result."},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_01234",
"content": "Rainy, 55°F",
}
],
},
],
model="claude-3-5-sonnet-20240620",
)
assert [m["role"] for m in result] == ["assistant", "tool", "user"]
assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
def test_translate_anthropic_messages_to_openai_converts_string_midturn_system():
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Keep it short."},
],
model="claude-3-5-sonnet-20240620",
)
assert result == [
{"role": "user", "content": "hi"},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
},
]
def _claude_code_user_id(session_id: str) -> str:
return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id})

89
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py Normal file
View file

@ -0,0 +1,89 @@
from collections import Counter
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
CONVERTED_SYSTEM_NOTE,
convert_mid_conversation_system_turns,
)
class RoleReadCountingMessage(dict):
def __init__(self, role: str, content: object, reads: Counter):
super().__init__(role=role, content=content)
self.reads = reads
def get(self, key, default=None):
self.reads[key] += 1
return super().get(key, default)
def test_convert_mid_conversation_system_turns_converts_system_to_user_in_place():
result = convert_mid_conversation_system_turns(
[
{"role": "user", "content": "hi"},
{"role": "system", "content": [{"type": "text", "text": "Keep it short."}]},
{"role": "assistant", "content": "Hi."},
]
)
assert result == (
{"role": "user", "content": "hi"},
{
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
},
{"role": "assistant", "content": "Hi."},
)
def test_convert_mid_conversation_system_turns_wraps_string_content():
result = convert_mid_conversation_system_turns(
[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Keep it short."},
]
)
assert result[1] == {
"role": "user",
"content": [
{"type": "text", "text": CONVERTED_SYSTEM_NOTE},
{"type": "text", "text": "Keep it short."},
],
}
def test_convert_mid_conversation_system_turns_moves_system_after_tool_result():
assistant_tool_use = {
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}],
}
wedged_system = {"role": "system", "content": "Use the corrected result."}
tool_result = {
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}],
}
result = convert_mid_conversation_system_turns([assistant_tool_use, wedged_system, tool_result])
assert result[0] is assistant_tool_use
assert result[1] is tool_result
assert result[2]["role"] == "user"
assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE
def test_convert_mid_conversation_system_turns_reads_each_role_a_bounded_number_of_times():
reads = Counter()
system_run = [RoleReadCountingMessage("system", f"reminder {i}", reads) for i in range(2_000)]
tool_result = RoleReadCountingMessage(
"user", [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}], reads
)
messages = [RoleReadCountingMessage("user", "hi", reads), *system_run, tool_result]
result = convert_mid_conversation_system_turns(messages)
assert reads["role"] <= 3 * len(messages)
assert result[1] is tool_result
assert [m["content"][1]["text"] for m in result[2:]] == [m["content"] for m in system_run]

18
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -1189,17 +1189,24 @@ def test_get_supported_openai_params_bedrock_converse():
@pytest.mark.parametrize(
"tools, expected_marker",
"tools, model, expected_marker",
[
pytest.param(
[{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}],
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"dep-bedrock",
id="tools-present-so-the-cachepoint-is-placed",
),
pytest.param(None, None, id="no-tools-so-nothing-is-placed"),
pytest.param(None, "anthropic.claude-sonnet-4-5-20250929-v1:0", None, id="no-tools-so-nothing-is-placed"),
pytest.param(
[{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}],
"global.openai.gpt-6-astra",
None,
id="openai-family-implicit-caching-only",
),
],
)
def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, expected_marker):
def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, model, expected_marker):
"""Spend attribution credits the gateway for breakpoints it placed, and a tool_config
point becomes one here or nowhere.
@ -1213,7 +1220,7 @@ def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, expec
optional_params["tools"] = tools
data = AmazonConverseConfig()._transform_request_helper(
model="anthropic.claude-sonnet-4-5-20250929-v1:0",
model=model,
system_content_blocks=[],
optional_params=optional_params,
messages=[{"role": "user", "content": "hi"}],
@ -5591,6 +5598,9 @@ def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model():
True,
id="unmapped-arn-keeps-emitting",
),
pytest.param("global.openai.gpt-6-astra", False, id="openai-family-implicit-caching-only"),
pytest.param("openai.gpt-oss-120b-1:0", False, id="openai-gpt-oss"),
pytest.param("us.openai.gpt-99-unmapped", False, id="unmapped-openai-family-still-suppressed"),
],
)
def test_cache_points_emitted_only_for_models_that_support_prompt_caching(model, expects_cache_points, monkeypatch):

17
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -23,6 +23,9 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import (
as_system_content_blocks,
)
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
AmazonAnthropicClaudeMessagesStreamDecoder,
@ -2533,20 +2536,16 @@ def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_
def test_as_system_content_blocks_handles_each_shape():
"""``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
"""``as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
a string -> a single text block, a list -> a shallow copy, and any other value
(e.g. a bare content-block dict) -> wrapped in a single-element list."""
block = {"type": "text", "text": "x"}
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(None) == []
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks("hello") == [
{"type": "text", "text": "hello"}
]
assert as_system_content_blocks(None) == []
assert as_system_content_blocks("hello") == [{"type": "text", "text": "hello"}]
blocks = [block]
out = AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(blocks)
out = as_system_content_blocks(blocks)
assert out == blocks and out is not blocks
assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(block) == [
block
]
assert as_system_content_blocks(block) == [block]
@pytest.mark.parametrize(

255
tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
View file

@ -8,7 +8,11 @@ from unittest.mock import MagicMock
import pytest
import litellm
from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY
from litellm.constants import (
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
@ -207,7 +211,19 @@ class ScriptedBedrockStream:
return (None, self._receiver)
class FakeAWSCredentialsIdentity:
def __init__(self, access_key_id, secret_access_key, session_token=None):
self.access_key_id = access_key_id
self.secret_access_key = secret_access_key
self.session_token = session_token
class FakeStaticCredentialsResolver:
def __init__(self, identity=None):
self.identity = identity
class FakeAWSCRTHTTPClient:
pass
@ -227,48 +243,32 @@ class StubCredentialsBedrockRealtime(BedrockRealtime):
return SimpleNamespace(get_frozen_credentials=lambda: self.frozen_credentials)
@pytest.fixture
def stub_aws_sdk_client(monkeypatch):
captured = {}
class FakeOperationInput:
def __init__(self, model_id):
self.model_id = model_id
class CapturingConfig:
def __init__(self, **kwargs):
captured["config_kwargs"] = kwargs
self.kwargs = kwargs
class FakeOperationInput:
def __init__(self, model_id):
self.model_id = model_id
class FakeBedrockRuntimeClient:
def __init__(self, config):
captured["client_config"] = config
async def invoke_model_with_bidirectional_stream(self, operation_input):
captured["operation_input"] = operation_input
if captured.get("streams"):
stream = captured["streams"].pop(0)
if isinstance(stream, Exception):
raise stream
return stream
return ScriptedBedrockStream(captured.get("scripted_payloads", []))
def _install_fake_sdk_modules(monkeypatch, client_module, config_module):
"""Wire fake aws_sdk_bedrock_runtime / smithy packages into sys.modules for the handler's lazy imports."""
package = types.ModuleType("aws_sdk_bedrock_runtime")
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.BedrockRuntimeClient = FakeBedrockRuntimeClient
client_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.Config = CapturingConfig
models_module = types.ModuleType("aws_sdk_bedrock_runtime.models")
models_module.BidirectionalInputPayloadPart = FakePayloadPart
models_module.InvokeModelWithBidirectionalStreamInputChunk = FakeInputChunk
models_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput
package.client = client_module
package.config = config_module
package.models = models_module
smithy_package = types.ModuleType("smithy_aws_core")
identity_module = types.ModuleType("smithy_aws_core.identity")
identity_module.AWSCredentialsIdentity = FakeAWSCredentialsIdentity
identity_module.StaticCredentialsResolver = FakeStaticCredentialsResolver
smithy_package.identity = identity_module
smithy_http_package = types.ModuleType("smithy_http")
smithy_http_aio = types.ModuleType("smithy_http.aio")
crt_module = types.ModuleType("smithy_http.aio.crt")
crt_module.AWSCRTHTTPClient = FakeAWSCRTHTTPClient
smithy_http_aio.crt = crt_module
smithy_http_package.aio = smithy_http_aio
stubbed_modules = {
"aws_sdk_bedrock_runtime": package,
@ -277,10 +277,56 @@ def stub_aws_sdk_client(monkeypatch):
"aws_sdk_bedrock_runtime.models": models_module,
"smithy_aws_core": smithy_package,
"smithy_aws_core.identity": identity_module,
"smithy_http": smithy_http_package,
"smithy_http.aio": smithy_http_aio,
"smithy_http.aio.crt": crt_module,
}
for module_name, module in stubbed_modules.items():
monkeypatch.setitem(sys.modules, module_name, module)
@pytest.fixture
def stub_aws_sdk_client(monkeypatch):
"""Fake of the aws-sdk-bedrock-runtime 0.10/0.11 surface: async config resolve, async client with close()"""
captured = {}
class FakeAsyncBedrockRuntimeConfig:
def __init__(self, kwargs):
self.kwargs = kwargs
@classmethod
async def resolve(cls, **kwargs):
captured["config_kwargs"] = kwargs
return cls(kwargs)
class FakeAsyncBedrockRuntimeClient:
def __init__(self, config):
captured["client_config"] = config
captured["client_closed"] = False
async def invoke_model_with_bidirectional_stream(self, operation_input):
captured["operation_input"] = operation_input
if captured.get("streams"):
stream = captured["streams"].pop(0)
if isinstance(stream, Exception):
raise stream
captured["open_stream"] = stream
return stream
stream = ScriptedBedrockStream(captured.get("scripted_payloads", []))
captured["open_stream"] = stream
return stream
async def close(self):
open_stream = captured.get("open_stream")
captured["input_closed_before_client_close"] = open_stream is None or open_stream.input_stream.closed
captured["client_closed"] = True
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.AsyncBedrockRuntimeClient = FakeAsyncBedrockRuntimeClient
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.AsyncBedrockRuntimeConfig = FakeAsyncBedrockRuntimeConfig
_install_fake_sdk_modules(monkeypatch, client_module, config_module)
for env_var in (
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
@ -764,15 +810,33 @@ class TestBedrockRealtimeAwsAuth:
)
config_kwargs = stub_aws_sdk_client["config_kwargs"]
assert config_kwargs["aws_access_key_id"] == "litellm-params-access-key"
assert config_kwargs["aws_secret_access_key"] == "litellm-params-secret-key"
assert config_kwargs["aws_session_token"] == "litellm-params-session-token"
assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver)
resolver = config_kwargs["aws_credentials_identity_resolver"]
assert isinstance(resolver, FakeStaticCredentialsResolver)
assert resolver.identity.access_key_id == "litellm-params-access-key"
assert resolver.identity.secret_access_key == "litellm-params-secret-key"
assert resolver.identity.session_token == "litellm-params-session-token"
assert config_kwargs["region"] == "us-east-1"
assert config_kwargs["endpoint_uri"] == "https://bedrock-runtime.us-east-1.amazonaws.com"
assert isinstance(config_kwargs["transport"], FakeAWSCRTHTTPClient)
assert stub_aws_sdk_client["client_config"].kwargs is config_kwargs
assert stub_aws_sdk_client["operation_input"].model_id == "amazon.nova-sonic-v1:0"
assert websocket.closed
@pytest.mark.asyncio
async def test_api_base_overrides_default_endpoint(self, stub_aws_sdk_client):
await BedrockRealtime().async_realtime(
model="amazon.nova-sonic-v1:0",
websocket=RealtimeClientWS(),
logging_obj=FakeLogging(),
aws_region_name="us-east-1",
aws_access_key_id="k",
aws_secret_access_key="s",
api_base="https://vpce-bedrock.example.internal",
aws_bedrock_runtime_endpoint="https://ignored.example.internal",
)
assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://vpce-bedrock.example.internal"
@pytest.mark.asyncio
async def test_role_assumption_params_forwarded_to_get_credentials(self, stub_aws_sdk_client):
handler = StubCredentialsBedrockRealtime(
@ -805,11 +869,11 @@ class TestBedrockRealtimeAwsAuth:
"aws_sts_endpoint": None,
"aws_external_id": "realtime-external-id",
}
config_kwargs = stub_aws_sdk_client["config_kwargs"]
assert config_kwargs["aws_access_key_id"] == "assumed-access-key"
assert config_kwargs["aws_secret_access_key"] == "assumed-secret-key"
assert config_kwargs["aws_session_token"] == "assumed-session-token"
assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver)
resolver = stub_aws_sdk_client["config_kwargs"]["aws_credentials_identity_resolver"]
assert isinstance(resolver, FakeStaticCredentialsResolver)
assert resolver.identity.access_key_id == "assumed-access-key"
assert resolver.identity.secret_access_key == "assumed-secret-key"
assert resolver.identity.session_token == "assumed-session-token"
@pytest.mark.asyncio
async def test_unresolvable_credentials_raise_clear_auth_error(self, stub_aws_sdk_client):
@ -826,5 +890,118 @@ class TestBedrockRealtimeAwsAuth:
assert "config_kwargs" not in stub_aws_sdk_client
class TestBedrockRealtimeSdkLifecycle:
"""aws-sdk-bedrock-runtime 0.10/0.11: async config, async client, CRT transport, close() (LIT-7938 regression)"""
AWS_ARGS = {
"model": "amazon.nova-sonic-v1:0",
"aws_region_name": "us-east-1",
"aws_access_key_id": "k",
"aws_secret_access_key": "s",
}
@pytest.mark.asyncio
async def test_client_closed_after_input_stream_on_normal_completion(self, stub_aws_sdk_client):
await BedrockRealtime().async_realtime(websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS)
assert stub_aws_sdk_client["client_closed"]
assert stub_aws_sdk_client["input_closed_before_client_close"]
@pytest.mark.asyncio
async def test_client_closed_when_stream_open_fails(self, stub_aws_sdk_client):
stub_aws_sdk_client["streams"] = [ServiceUnavailableException("bedrock unavailable")]
with pytest.raises(ServiceUnavailableException):
await BedrockRealtime().async_realtime(
websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS
)
assert stub_aws_sdk_client["client_closed"]
@pytest.mark.asyncio
async def test_client_closed_when_provider_stream_fails_mid_session(self, stub_aws_sdk_client):
stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver)]
with pytest.raises(BedrockError):
await BedrockRealtime().async_realtime(
websocket=ConnectedClientWS([]), logging_obj=FakeLogging(), **self.AWS_ARGS
)
assert stub_aws_sdk_client["client_closed"]
assert stub_aws_sdk_client["input_closed_before_client_close"]
@pytest.mark.asyncio
async def test_client_without_close_completes_session(self, monkeypatch):
class ClientWithoutClose:
def __init__(self, config):
pass
async def invoke_model_with_bidirectional_stream(self, operation_input):
return ScriptedBedrockStream([])
class ConfigWithoutCapture:
@classmethod
async def resolve(cls, **kwargs):
return cls()
client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
client_module.AsyncBedrockRuntimeClient = ClientWithoutClose
config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
config_module.AsyncBedrockRuntimeConfig = ConfigWithoutCapture
_install_fake_sdk_modules(monkeypatch, client_module, config_module)
websocket = RealtimeClientWS()
await BedrockRealtime().async_realtime(websocket=websocket, logging_obj=FakeLogging(), **self.AWS_ARGS)
assert websocket.closed
class TestBedrockRealtimeSdkImportErrors:
"""Init errors must tell 'SDK not installed' apart from 'SDK installed but unsupported version' (LIT-7938)"""
@pytest.mark.asyncio
async def test_absent_sdk_names_install_extra(self, monkeypatch):
monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime", None)
handler = BedrockRealtime(sdk_version_lookup=lambda: None)
with pytest.raises(ImportError) as exc_info:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging()
)
message = str(exc_info.value)
assert message.startswith("Missing aws_sdk_bedrock_runtime")
assert "litellm[bedrock-realtime]" in message
assert "is installed but" not in message
close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode()
assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason
assert "pip install 'litellm[bedrock-realtime]'" in close_reason
@pytest.mark.asyncio
async def test_incompatible_sdk_names_installed_version_and_supported_range(self, monkeypatch):
legacy_client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
legacy_client_module.BedrockRuntimeClient = object
legacy_config_module = types.ModuleType("aws_sdk_bedrock_runtime.config")
legacy_config_module.Config = object
_install_fake_sdk_modules(monkeypatch, legacy_client_module, legacy_config_module)
handler = BedrockRealtime(sdk_version_lookup=lambda: "0.7.0")
with pytest.raises(ImportError) as exc_info:
await handler.async_realtime(
model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging()
)
message = str(exc_info.value)
assert "aws-sdk-bedrock-runtime 0.7.0 is installed but" in message
assert ">=0.10.0,<0.12.0" in message
assert not message.startswith("Missing aws_sdk_bedrock_runtime")
assert isinstance(exc_info.value.__cause__, ImportError)
assert str(exc_info.value.__cause__) not in message
assert "cannot import name" not in message
close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode()
assert "0.7.0 is installed" in close_reason
assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason
if __name__ == "__main__":
pytest.main([__file__, "-v"])

2
tests/test_litellm/proxy/client/cli/test_encryption_commands.py
View file

@ -1,4 +1,4 @@
"""CLI tests for the ``litellm-proxy encryption migrate`` command.
"""CLI tests for the ``lite encryption migrate`` command.
The HTTP client is mocked, so these assert the command's request routing (GET
check vs POST migrate, dry-run param) and its residual-state messaging without a

34
tests/test_litellm/proxy/client/cli/test_global_options.py
View file

@ -1,7 +1,9 @@
# stdlib imports
import json
import os
import sys
from pathlib import Path
from typing import Final
from unittest.mock import Mock, patch
import pytest
@ -9,7 +11,8 @@ from click.testing import CliRunner
import litellm.proxy.client.cli
from litellm._version import version as litellm_version
from litellm.proxy.client.cli import cli
from litellm.proxy.client.cli import cli, litellm_proxy_cli
from litellm.proxy.client.cli.main import LITELLM_PROXY_DEPRECATION_NOTICE
@pytest.fixture
@ -234,3 +237,32 @@ def test_version_flag_never_sends_api_key_to_unnamed_server(cli_runner, isolated
assert all(url.startswith("https://flag-proxy.example.com") for url in requested_urls)
sent_keys = [call.kwargs["headers"].get("Authorization") for call in mock_request.call_args_list]
assert sent_keys == ["Bearer sk-intended-for-flag-proxy"] * len(requested_urls)
def test_litellm_proxy_entrypoint_prints_deprecation_notice_on_stderr_and_still_runs(monkeypatch, capsys, requests_mock):
requests_mock.get("http://localhost:4000/health/readiness", json={"litellm_version": "1.2.3"})
monkeypatch.setattr(sys, "argv", ["litellm-proxy", "--version"])
monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000")
with pytest.raises(SystemExit) as exit_info:
litellm_proxy_cli()
captured: Final = capsys.readouterr()
assert exit_info.value.code == 0
assert captured.err.strip() == LITELLM_PROXY_DEPRECATION_NOTICE
assert f"LiteLLM Proxy CLI Version: {litellm_version}" in captured.out
assert "LiteLLM Proxy Server Version: 1.2.3" in captured.out
assert "deprecated" not in captured.out
def test_lite_entrypoint_prints_nothing_on_stderr(monkeypatch, capsys, requests_mock):
requests_mock.get("http://localhost:4000/health/readiness", json={"litellm_version": "1.2.3"})
monkeypatch.setattr(sys, "argv", ["lite", "--version"])
monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000")
with pytest.raises(SystemExit) as exit_info:
cli()
captured: Final = capsys.readouterr()
assert exit_info.value.code == 0
assert "LiteLLM Proxy Server Version: 1.2.3" in captured.out
assert f"LiteLLM Proxy CLI Version: {litellm_version}" in captured.out
assert captured.err == ""

174
tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
View file

@ -4,7 +4,7 @@ import sys
import types
from datetime import datetime, timedelta, timezone
from datetime import time as dt_time
from typing import Any, Dict, Final, List
from typing import Any, Dict, Final, List, Optional
from unittest.mock import AsyncMock, MagicMock
import httpx
@ -16,6 +16,7 @@ from litellm.proxy._types import LiteLLM_VerificationToken
from litellm.proxy.common_utils import reset_budget_job as reset_budget_job_module
from litellm.constants import (
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
RESET_BUDGET_JOB_BATCH_SIZE,
RESET_BUDGET_JOB_LOCK_TTL_SECONDS,
RESET_BUDGET_JOB_NAME,
)
@ -31,13 +32,36 @@ class MockTable:
self.find_many_calls: List[Dict[str, Any]] = []
self.update_many_calls: List[Dict[str, Any]] = []
self._find_many_results: List[Any] = []
self._find_many_error: Optional[tuple[int, Exception]] = None
def set_find_many_results(self, results: List[Any]):
self._find_many_results = results
async def find_many(self, where: Dict[str, Any]) -> List[Any]:
self.find_many_calls.append({"where": where})
return self._find_many_results
def set_find_many_error(self, after_reads: int, error: Exception):
"""Fail every read past the first ``after_reads``, the way a connection
dropping partway through a paged walk does."""
self._find_many_error = (after_reads, error)
async def find_many(
self,
where: Dict[str, Any],
order: Optional[Dict[str, str]] = None,
take: Optional[int] = None,
) -> List[Any]:
"""Replays canned rows, honouring the keyset cursor + ``take`` a paged
caller relies on: without that a paged walk never advances and the
test would hang instead of failing."""
if self._find_many_error is not None and len(self.find_many_calls) >= self._find_many_error[0]:
raise self._find_many_error[1]
paging = {k: v for k, v in (("order", order), ("take", take)) if v is not None}
self.find_many_calls.append({"where": where, **paging})
rows = list(self._find_many_results)
for field, condition in where.items():
if isinstance(condition, dict) and "gt" in condition and field != "spend":
rows = [row for row in rows if getattr(row, field, "") > condition["gt"]]
for field, direction in (order or {}).items():
rows.sort(key=lambda row: getattr(row, field, ""), reverse=direction == "desc")
return rows[:take] if take is not None else rows
async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]:
self.update_many_calls.append({"where": where, "data": data})
@ -801,10 +825,16 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock
},
]
# Verify find_many was called to fetch NULL-budget-id end users
# The post-commit invalidation walk covers both branches, so implicitly
# created customers on the default tier get their cached spend dropped too,
# and it is paged rather than reading the whole customer population.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 1
assert find_many_calls[0]["where"] == {"budget_id": None, "spend": {"gt": 0}}
assert find_many_calls[0]["where"]["OR"] == [
{"budget_id": {"in": [default_budget_id]}},
{"budget_id": None},
]
assert find_many_calls[0]["take"] == RESET_BUDGET_JOB_BATCH_SIZE
litellm.max_end_user_budget_id = None
@ -835,9 +865,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_configured(
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
# Should NOT have queried for NULL-budget-id end users
# The invalidation walk must not reach for NULL-budget-id customers: they
# ride a default tier that is not expiring, so their spend stays put.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 0
assert [call["where"] for call in find_many_calls] == [
{"budget_id": {"in": ["some-budget"]}, "user_id": {"gt": ""}}
]
litellm.max_end_user_budget_id = None
@ -872,9 +905,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_in_reset_li
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
# Should NOT have queried for NULL-budget-id end users
# The invalidation walk must not reach for NULL-budget-id customers: they
# ride a default tier that is not expiring, so their spend stays put.
find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert len(find_many_calls) == 0
assert [call["where"] for call in find_many_calls] == [
{"budget_id": {"in": ["other-budget"]}, "user_id": {"gt": ""}}
]
litellm.max_end_user_budget_id = None
@ -1252,6 +1288,21 @@ def _make_counter_invalidation_job(monkeypatch):
user_api_key_cache = MagicMock()
user_api_key_cache.async_delete_cache = AsyncMock()
# Batch deletes fan out to the same per-key calls the real DualCache makes,
# so an assertion reads "this key was invalidated" whether the caller went
# one key at a time or a page at a time.
async def _delete_counter_keys(keys):
for key in keys:
spend_counter_cache.in_memory_cache.delete_cache(key=key)
await spend_counter_cache.redis_cache.async_delete_cache(key=key)
async def _delete_management_keys(keys):
for key in keys:
await user_api_key_cache.async_delete_cache(key=key)
spend_counter_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_counter_keys)
user_api_key_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_management_keys)
fake_module = types.ModuleType("litellm.proxy.proxy_server")
fake_module.spend_counter_cache = spend_counter_cache
fake_module.user_api_key_cache = user_api_key_cache
@ -1586,7 +1637,7 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j
"user_id": "customer-42",
},
)
mock_prisma_client.data["enduser"] = [test_enduser]
mock_prisma_client.db.litellm_endusertable.set_find_many_results([test_enduser])
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
@ -1596,6 +1647,107 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j
assert "end_user_id:customer-42" in deleted
def test_enduser_invalidation_is_paged_and_batched(reset_budget_job, mock_prisma_client, monkeypatch):
"""The post-commit invalidation walk stays bounded in memory and in round trips.
Reading every customer on an expiring tier into one result set puts a
customer-count-sized list in the proxy's heap on every tick, which is an OOM
on a large enough deployment rather than a slow tick. Awaiting one cache call
per customer makes the last customer wait out every customer ahead of it.
Both regress silently, so pin the page size, the strictly advancing cursor,
and one batched call per page.
"""
counter_cache: Final = _make_counter_invalidation_job(monkeypatch)
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
population: Final = RESET_BUDGET_JOB_BATCH_SIZE * 2 + 3
mock_prisma_client.db.litellm_endusertable.set_find_many_results(
[
type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"})
for i in range(population)
]
)
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
reads: Final = mock_prisma_client.db.litellm_endusertable.find_many_calls
assert [read["take"] for read in reads] == [RESET_BUDGET_JOB_BATCH_SIZE] * 3
assert [read["where"]["user_id"]["gt"] for read in reads] == [
"",
f"cust-{RESET_BUDGET_JOB_BATCH_SIZE - 1:06d}",
f"cust-{RESET_BUDGET_JOB_BATCH_SIZE * 2 - 1:06d}",
]
assert counter_cache.async_delete_cache_keys.await_count == 3
assert counter_cache.user_api_key_cache.async_delete_cache_keys.await_count == 3
counter_cache.async_delete_cache.assert_not_called()
invalidated: Final = {
key for call in counter_cache.async_delete_cache_keys.await_args_list for key in call.args[0]
}
assert invalidated == {f"spend:end_user:cust-{i:06d}" for i in range(population)}
evicted: Final = {
key for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list for key in call.args[0]
}
assert evicted == {f"end_user_id:cust-{i:06d}" for i in range(population)}
def test_enduser_invalidation_reports_a_page_read_failure_instead_of_a_clean_finish(
mock_prisma_client, monkeypatch
):
"""A page that fails to read is not the end of the customer list.
The tier's window is already advanced by the time this walk runs, so no later
tick comes back for the customers past the page that failed: their cached
spend goes on rejecting requests until it expires. Returning the same empty
page normal end-of-data returns hid that behind a report of a clean pass.
"""
_make_counter_invalidation_job(monkeypatch)
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
endusers: Final = mock_prisma_client.db.litellm_endusertable
endusers.set_find_many_results(
[
type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"})
for i in range(RESET_BUDGET_JOB_BATCH_SIZE + 3)
]
)
endusers.set_find_many_error(1, RuntimeError("connection reset while paging customers"))
logging_obj: Final = RecordingProxyLogging()
job: Final = ResetBudgetJob(proxy_logging_obj=logging_obj, prisma_client=mock_prisma_client)
_run_and_drain_hooks(job.reset_budget_for_litellm_budget_table)
metadata: Final = logging_obj.service_logging_obj.success_calls[0]["event_metadata"]
assert metadata["enduser_invalidation_truncated"] is True
assert metadata["num_endusers_updated"] == RESET_BUDGET_JOB_BATCH_SIZE
def test_a_failed_counter_batch_still_evicts_the_management_cache(
reset_budget_job, mock_prisma_client, monkeypatch
):
"""The spend counters and the management cache are invalidated independently.
Sharing one handler meant a Redis failure on the counters returned before the
management cache was touched at all. The commit has already zeroed those rows
by then, so the cached objects keep authorizing against their pre-reset spend
until they expire.
"""
counter_cache: Final = _make_counter_invalidation_job(monkeypatch)
counter_cache.async_delete_cache_keys = AsyncMock(side_effect=RuntimeError("redis unavailable"))
mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")]
mock_prisma_client.db.litellm_endusertable.set_find_many_results(
[type("EndUser", (), {"user_id": "customer-42", "spend": 5.0, "budget_id": "budget-1"})]
)
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
evicted: Final = {
key
for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list
for key in call.args[0]
}
assert "end_user_id:customer-42" in evicted
def test_budget_table_reset_commits_even_when_cache_eviction_fails(reset_budget_job, mock_prisma_client, monkeypatch):
"""Eviction runs after the commit, so a broken cache cannot undo the write."""

53
tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py
View file

@ -82,6 +82,19 @@ class FakeRedisCache(RedisCache):
async def async_delete_cache(self, key: str): # type: ignore[override]
self._store.pop(key, None)
async def delete_cache_keys(self, keys): # type: ignore[override]
for key in keys:
self._store.pop(key, None)
class PartitionFailingRedisCache(FakeRedisCache):
"""Fails the batch delete for the key-object partition and no other."""
async def delete_cache_keys(self, keys): # type: ignore[override]
if any(is_user_key_cache_key(key) for key in keys):
raise ConnectionError("redis unavailable")
await super().delete_cache_keys(keys)
def _make_key_obj(token: str = "tok") -> UserAPIKeyAuth:
# Minimal object (UserAPIKeyAuth inherits token from base view).
@ -331,6 +344,46 @@ class TestUserKeyObjectPartition:
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None
assert await redis.async_get_cache(HASHED_TOKEN) is None
@pytest.mark.asyncio
async def test_batch_delete_routes_each_key_to_its_partition(self):
"""A batch delete has to clear the same partition the single delete does.
``DualCache``'s batch delete only knows about the main in-memory cache, so
inheriting it unchanged leaves a key object sitting in ``key_object_cache``
with its pre-reset spend, and the next request is authorized against that
stale copy until the local entry expires.
"""
redis = FakeRedisCache()
cache = UserApiKeyCache(redis_cache=redis)
await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth)
await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"})
await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")])
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None
assert await cache.async_get_cache(end_user_cache_key("u1")) is None
assert await redis.async_get_cache(HASHED_TOKEN) is None
assert await redis.async_get_cache(end_user_cache_key("u1")) is None
@pytest.mark.asyncio
async def test_batch_delete_clears_the_other_partition_when_one_fails(self):
"""One partition failing must not cost the other its deletions.
A caller batching these has already committed the rows they cache, so a
partition that is skipped keeps authorizing against pre-reset spend until
the entry expires. The failure is still raised for the caller to report.
"""
redis = PartitionFailingRedisCache()
cache = UserApiKeyCache(redis_cache=redis)
await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth)
await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"})
with pytest.raises(ConnectionError):
await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")])
assert await cache.async_get_cache(end_user_cache_key("u1")) is None
assert await redis.async_get_cache(end_user_cache_key("u1")) is None
@pytest.mark.asyncio
async def test_pipeline_write_routes_each_entry_to_its_partition(self):
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(max_size_in_memory=2))

70
tests/test_litellm/proxy/image_endpoints/test_endpoints.py
View file

@ -95,18 +95,14 @@ async def test_image_generation_prompt_rerouting(monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None)
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", {})
monkeypatch.setattr(
"litellm.proxy.proxy_server.proxy_logging_obj", fake_proxy_logger
)
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", fake_proxy_logger)
monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None)
monkeypatch.setattr("litellm.proxy.proxy_server.version", "test-version")
monkeypatch.setattr(
"litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers",
classmethod(lambda *args, **kwargs: {}),
)
monkeypatch.setattr(
"litellm.proxy.image_endpoints.endpoints.route_request", fake_route_request
)
monkeypatch.setattr("litellm.proxy.image_endpoints.endpoints.route_request", fake_route_request)
result = await endpoints.image_generation(
request=request,
@ -141,6 +137,60 @@ def _image_edit_client(monkeypatch, captured: Dict[str, Any]) -> TestClient:
return TestClient(app)
def test_image_edit_image_array_alias_is_not_forwarded(monkeypatch):
"""The documented `image[]` alias must reach the provider only as `image`."""
captured: Dict[str, Any] = {}
response = _image_edit_client(monkeypatch, captured).post(
"/v1/images/edits",
files={"image[]": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png")},
data={"model": "gpt-image-1", "prompt": "add a hat"},
)
assert response.status_code == 200
assert "image[]" not in captured
assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"]
assert [buffer.name for buffer in captured["image"]] == ["tree.png"]
def test_image_edit_mask_array_alias_is_not_forwarded(monkeypatch):
"""`mask[]` has the same shape as `image[]` and must be dropped the same way."""
captured: Dict[str, Any] = {}
response = _image_edit_client(monkeypatch, captured).post(
"/v1/images/edits",
files={
"image": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png"),
"mask[]": ("mask.png", b"\x89PNG\r\n\x1a\nmask", "image/png"),
},
data={"model": "gpt-image-1", "prompt": "add a hat"},
)
assert response.status_code == 200
assert "mask[]" not in captured
assert [buffer.getvalue() for buffer in captured["mask"]] == [b"\x89PNG\r\n\x1a\nmask"]
assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"]
def test_image_edit_canonical_file_fields_still_reach_the_provider(monkeypatch):
"""Dropping the bracketed aliases must not touch the canonical fields."""
captured: Dict[str, Any] = {}
response = _image_edit_client(monkeypatch, captured).post(
"/v1/images/edits",
files={
"image": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png"),
"mask": ("mask.png", b"\x89PNG\r\n\x1a\nmask", "image/png"),
},
data={"model": "gpt-image-1", "prompt": "add a hat"},
)
assert response.status_code == 200
assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"]
assert [buffer.getvalue() for buffer in captured["mask"]] == [b"\x89PNG\r\n\x1a\nmask"]
assert captured["prompt"] == "add a hat"
def test_image_edit_multipart_n_reaches_the_provider_as_an_int(monkeypatch):
"""A multipart `n` must not arrive as the string Starlette parsed it into."""
captured: Dict[str, Any] = {}
@ -180,7 +230,9 @@ async def test_a_model_the_router_cannot_serve_answers_an_openai_typed_error(mon
async def fake_add_litellm_data_to_request(**kwargs: object) -> object:
return kwargs["data"]
async def fake_pre_call_hook(*, user_api_key_dict: UserAPIKeyAuth, data: dict[str, object], call_type: str) -> dict[str, object]:
async def fake_pre_call_hook(
*, user_api_key_dict: UserAPIKeyAuth, data: dict[str, object], call_type: str
) -> dict[str, object]:
return data
async def fake_post_call_failure_hook(**_: object) -> None:
@ -211,7 +263,9 @@ async def test_a_model_the_router_cannot_serve_answers_an_openai_typed_error(mon
request = Request({"type": "http", "method": "POST", "path": "/v1/images/generations", "headers": []}, receive)
with pytest.raises(ProxyException) as raised:
await endpoints.image_generation(request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth())
await endpoints.image_generation(
request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth()
)
assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "404")

772
tests/test_litellm/proxy/management_endpoints/management_v1/test_teams.py Normal file
View file

@ -0,0 +1,772 @@
"""`POST /management/v1/teams/{team_id}/members/bulk_update`: the per-member limit writes and the
HTTP contract around them.
The in-memory Prisma here follows the one in
`tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py`, extended with the budget
table and the membership/budget relation the bulk budget writer needs.
"""
import copy
from collections.abc import Mapping, Sequence
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.testclient import TestClient
from pydantic import BaseModel, ConfigDict, Field
from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
team_membership_auth_cache_key,
team_membership_reservation_cache_key,
)
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
from litellm.proxy.list_api.common import ManagementProblem, problem_response, request_validation_problem
from litellm.proxy.management_endpoints.management_v1 import router
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.management_helpers.bulk_team_member_budgets import bulk_update_team_member_budgets
from litellm.types.proxy.management_endpoints.team_endpoints import (
MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES,
BulkTeamMemberBudgetUpdateRequest,
TeamMemberBudgetUpdateResult,
)
ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin")
OUTSIDER: Final = UserAPIKeyAuth(user_id="outsider", user_role=LitellmUserRoles.INTERNAL_USER)
TEAM_ID: Final = "t1"
class _BudgetRow(BaseModel):
"""A `LiteLLM_BudgetTable` row, carrying every column the merge patch reads or writes."""
model_config = ConfigDict(extra="allow")
budget_id: str
max_budget: float | None = None
soft_budget: float | None = None
max_parallel_requests: int | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
model_max_budget: Mapping[str, object] | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
allowed_models: list[str] = Field(default_factory=list)
created_by: str | None = None
updated_by: str | None = None
class _MembershipRow(BaseModel):
"""A `LiteLLM_TeamMembership` row; `litellm_budget_table` is only filled on an `include` read."""
model_config = ConfigDict(extra="allow")
user_id: str
team_id: str
budget_id: str | None = None
litellm_budget_table: _BudgetRow | None = None
def _wanted(where: Mapping[str, object], field: str) -> set[str] | None:
clause: Final = where.get(field)
if isinstance(clause, dict) and "in" in clause:
return set(clause["in"])
if isinstance(clause, str):
return {clause}
return None
def _matches(row: Mapping[str, object], where: Mapping[str, object]) -> bool:
return all((wanted := _wanted(where, field)) is not None and row.get(field) in wanted for field in where)
class _BudgetTable:
def __init__(self, budgets: Sequence[_BudgetRow]) -> None:
self.rows: dict[str, _BudgetRow] = {b.budget_id: b for b in budgets}
async def find_unique(self, where: Mapping[str, str]) -> _BudgetRow | None:
return self.rows.get(where["budget_id"])
async def update(self, where: Mapping[str, str], data: Mapping[str, object]) -> _BudgetRow:
row: Final = self.rows[where["budget_id"]]
updated: Final = row.model_copy(update=dict(data))
self.rows[row.budget_id] = updated
return updated
async def create(self, data: Mapping[str, object], include: Mapping[str, bool] | None = None) -> _BudgetRow:
budget_id: Final = f"new-budget-{len(self.rows) + 1}"
row: Final = _BudgetRow.model_validate({**data, "budget_id": budget_id})
self.rows[budget_id] = row
return row
class _MembershipTable:
def __init__(self, budgets: _BudgetTable, memberships: Sequence[_MembershipRow]) -> None:
self._budgets = budgets
self.rows: list[_MembershipRow] = list(memberships)
def _index_of(self, user_id: str, team_id: str) -> int | None:
return next(
(i for i, r in enumerate(self.rows) if r.user_id == user_id and r.team_id == team_id),
None,
)
async def find_many(
self, where: Mapping[str, object], include: Mapping[str, bool] | None = None
) -> list[_MembershipRow]:
matched: Final = [r for r in self.rows if _matches(r.model_dump(), where)]
if not include:
return matched
return [
r.model_copy(update={"litellm_budget_table": self._budgets.rows.get(r.budget_id or "")}) for r in matched
]
async def update(self, where: Mapping[str, Mapping[str, str]], data: Mapping[str, object]) -> _MembershipRow:
key: Final = where["user_id_team_id"]
index: Final = self._index_of(key["user_id"], key["team_id"])
assert index is not None, f"no membership row for {key}"
relation: Final = data.get("litellm_budget_table")
if isinstance(relation, dict) and relation.get("disconnect"):
self.rows[index] = self.rows[index].model_copy(update={"budget_id": None})
return self.rows[index]
async def upsert(self, where: Mapping[str, Mapping[str, str]], data: Mapping[str, object]) -> _MembershipRow:
key: Final = where["user_id_team_id"]
budget_id: Final = data["update"]["litellm_budget_table"]["connect"]["budget_id"]
index: Final = self._index_of(key["user_id"], key["team_id"])
if index is None:
self.rows.append(_MembershipRow(user_id=key["user_id"], team_id=key["team_id"], budget_id=budget_id))
return self.rows[-1]
self.rows[index] = self.rows[index].model_copy(update={"budget_id": budget_id})
return self.rows[index]
class _TeamTable:
"""`find_many` and `create` are what `RoutingPrismaWrapper` keys read routing off, so a fake
table without them would silently never route and pass a reader-staleness test on the writer."""
def __init__(self, teams: Sequence[LiteLLM_TeamTable]) -> None:
self.rows: dict[str, LiteLLM_TeamTable] = {t.team_id: t for t in teams}
async def find_unique(self, where: Mapping[str, str]) -> LiteLLM_TeamTable | None:
return self.rows.get(where["team_id"])
async def find_many(self, where: Mapping[str, object] | None = None) -> list[LiteLLM_TeamTable]:
return [t for t in self.rows.values() if where is None or _matches(t.model_dump(), where)]
async def create(self, data: Mapping[str, object]) -> LiteLLM_TeamTable:
row: Final = LiteLLM_TeamTable.model_validate(dict(data))
self.rows[row.team_id] = row
return row
class _Db:
def __init__(
self,
teams: Sequence[LiteLLM_TeamTable],
memberships: Sequence[_MembershipRow],
budgets: Sequence[_BudgetRow],
) -> None:
self.litellm_teamtable = _TeamTable(teams)
self.litellm_budgettable = _BudgetTable(budgets)
self.litellm_teammembership = _MembershipTable(self.litellm_budgettable, memberships)
class _FakePrisma:
def __init__(
self,
teams: Sequence[LiteLLM_TeamTable] = (),
memberships: Sequence[_MembershipRow] = (),
budgets: Sequence[_BudgetRow] = (),
) -> None:
self.db = _Db(teams, memberships, budgets)
@asynccontextmanager
async def tx(self, *, timeout: object = None):
snapshot: Final = copy.deepcopy(self.db)
try:
yield self.db
except BaseException:
self.db = snapshot
raise
class _ReplicatedPrisma:
"""A client whose reads route to a lagging replica, as a proxy with `DATABASE_URL_READ_REPLICA` does."""
def __init__(self, writer: _FakePrisma, reader: _FakePrisma) -> None:
self._writer = writer
self.db = RoutingPrismaWrapper(writer=writer.db, reader=reader.db) # pyright: ignore[reportArgumentType] # fake dbs stand in for PrismaWrapper
def tx(self, *, timeout: object = None):
return self._writer.tx(timeout=timeout)
class _UnreachableDb:
"""A `.db` whose every table access fails, as one behind a dropped connection does."""
def __getattr__(self, name: str) -> object:
raise RuntimeError("connection reset by peer")
class _UnreachablePrisma:
def __init__(self) -> None:
self.db = _UnreachableDb()
def _team(
*members: str,
team_id: str = TEAM_ID,
default_budget_id: str | None = None,
admins: Sequence[str] = (),
) -> LiteLLM_TeamTable:
return LiteLLM_TeamTable(
team_id=team_id,
metadata={"team_member_budget_id": default_budget_id} if default_budget_id else {},
members_with_roles=[
Member(user_id=m, user_email=f"{m}@example.com", role="admin" if m in admins else "user") for m in members
],
)
def _membership(user_id: str, budget_id: str | None = None, team_id: str = TEAM_ID) -> _MembershipRow:
return _MembershipRow(user_id=user_id, team_id=team_id, budget_id=budget_id)
def _budget(
budget_id: str,
*,
max_budget: float | None = None,
tpm_limit: int | None = None,
rpm_limit: int | None = None,
budget_duration: str | None = None,
) -> _BudgetRow:
return _BudgetRow(
budget_id=budget_id,
max_budget=max_budget,
tpm_limit=tpm_limit,
rpm_limit=rpm_limit,
budget_duration=budget_duration,
)
async def _bulk_update(
prisma: _FakePrisma | _ReplicatedPrisma,
members: Sequence[Mapping[str, object]],
team_id: str = TEAM_ID,
caller: UserAPIKeyAuth = ADMIN,
cache: UserApiKeyCache | None = None,
) -> tuple[TeamMemberBudgetUpdateResult, ...]:
return await bulk_update_team_member_budgets(
team_id=team_id,
data=BulkTeamMemberBudgetUpdateRequest.model_validate({"members": list(members)}),
user_api_key_dict=caller,
prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient
user_api_key_cache=cache or UserApiKeyCache(),
)
def _budget_id_of(prisma: _FakePrisma, user_id: str, team_id: str = TEAM_ID) -> str | None:
row: Final = next(r for r in prisma.db.litellm_teammembership.rows if r.user_id == user_id and r.team_id == team_id)
return row.budget_id
def _budget_of(prisma: _FakePrisma, user_id: str, team_id: str = TEAM_ID) -> _BudgetRow:
budget_id: Final = _budget_id_of(prisma, user_id, team_id)
assert budget_id is not None, f"{user_id} has no budget"
return prisma.db.litellm_budgettable.rows[budget_id]
def _seeded_cache(*user_ids: str, team_id: str = TEAM_ID) -> UserApiKeyCache:
cache: Final = UserApiKeyCache()
for user_id in user_ids:
cache.set_cache(key=team_membership_auth_cache_key(team_id=team_id, user_id=user_id), value={"cap": "old"})
cache.set_cache(
key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id), value={"cap": "old"}
)
return cache
def _cached_keys(cache: UserApiKeyCache, user_id: str, team_id: str = TEAM_ID) -> tuple[object, object]:
return (
cache.get_cache(key=team_membership_auth_cache_key(team_id=team_id, user_id=user_id)),
cache.get_cache(key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id)),
)
@pytest.mark.asyncio
async def test_patching_one_member_of_a_shared_budget_row_forks_it_and_leaves_the_other_member_untouched():
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "shared-b"), _membership("m2", "shared-b")],
budgets=[_budget("shared-b", max_budget=100.0, tpm_limit=900)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 50}])
assert [(r.user_id, r.success, r.max_budget) for r in results] == [("m1", True, 50.0)]
assert _budget_id_of(prisma, "m1") not in (None, "shared-b")
assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (50.0, 900)
assert _budget_id_of(prisma, "m2") == "shared-b"
assert prisma.db.litellm_budgettable.rows["shared-b"].max_budget == 100.0
assert results[0].budget_id == _budget_id_of(prisma, "m1")
@pytest.mark.asyncio
async def test_patching_members_of_the_team_default_budget_gives_each_their_own_row_and_leaves_the_default_alone():
prisma = _FakePrisma(
teams=[_team("m1", "m2", "m3", default_budget_id="team-default")],
memberships=[
_membership("m1", "team-default"),
_membership("m2", "team-default"),
_membership("m3", "team-default"),
],
budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)],
)
results = await _bulk_update(
prisma,
[{"user_id": "m1", "max_budget_in_team": 5}, {"user_id": "m2", "max_budget_in_team": 7}],
)
assert [r.success for r in results] == [True, True]
default = prisma.db.litellm_budgettable.rows["team-default"]
assert (default.max_budget, default.tpm_limit) == (25.0, 1000)
assert _budget_id_of(prisma, "m3") == "team-default"
patched = (_budget_id_of(prisma, "m1"), _budget_id_of(prisma, "m2"))
assert len(set(patched)) == 2 and "team-default" not in patched
assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (5.0, 1000)
assert (_budget_of(prisma, "m2").max_budget, _budget_of(prisma, "m2").tpm_limit) == (7.0, 1000)
@pytest.mark.asyncio
async def test_the_team_default_row_is_forked_even_when_only_one_membership_points_at_it():
prisma = _FakePrisma(
teams=[_team("m1", "m2", default_budget_id="team-default")],
memberships=[_membership("m1", "team-default")],
budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 5}])
assert [(r.success, r.max_budget, r.tpm_limit) for r in results] == [(True, 5.0, 1000)]
default = prisma.db.litellm_budgettable.rows["team-default"]
assert (default.max_budget, default.tpm_limit) == (25.0, 1000)
assert _budget_id_of(prisma, "m1") not in (None, "team-default")
@pytest.mark.asyncio
async def test_a_budget_row_only_one_member_points_at_is_updated_in_place():
prisma = _FakePrisma(
teams=[_team("m1", "m2", default_budget_id="team-default")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "team-default")],
budgets=[_budget("team-default", max_budget=25.0), _budget("priv-m1", max_budget=10.0, tpm_limit=5)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 20}])
assert [(r.success, r.budget_id, r.max_budget) for r in results] == [(True, "priv-m1", 20.0)]
assert set(prisma.db.litellm_budgettable.rows) == {"team-default", "priv-m1"}
assert _budget_id_of(prisma, "m1") == "priv-m1"
assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (20.0, 5)
@pytest.mark.asyncio
async def test_an_omitted_field_is_kept_an_explicit_null_clears_it_and_clearing_the_last_limit_disconnects():
prisma = _FakePrisma(
teams=[_team("m1")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", max_budget=10.0, tpm_limit=5, rpm_limit=7)],
)
kept = await _bulk_update(prisma, [{"user_id": "m1", "rpm_limit": 9}])
assert (kept[0].max_budget, kept[0].tpm_limit, kept[0].rpm_limit) == (10.0, 5, 9)
cleared = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": None}])
assert (cleared[0].max_budget, cleared[0].tpm_limit, cleared[0].rpm_limit) == (10.0, None, 9)
assert _budget_id_of(prisma, "m1") == "priv-m1"
emptied = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": None, "rpm_limit": None}])
assert (emptied[0].success, emptied[0].budget_id, emptied[0].max_budget) == (True, None, None)
assert _budget_id_of(prisma, "m1") is None
@pytest.mark.asyncio
async def test_budget_duration_seeds_a_reset_time_derived_from_the_duration_and_clearing_it_clears_the_reset():
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")],
budgets=[_budget("priv-m1", max_budget=10.0), _budget("priv-m2", max_budget=10.0)],
)
before = datetime.now(timezone.utc)
await _bulk_update(
prisma,
[{"user_id": "m1", "budget_duration": "2d"}, {"user_id": "m2", "budget_duration": "5d"}],
)
two_day = _budget_of(prisma, "m1").budget_reset_at
five_day = _budget_of(prisma, "m2").budget_reset_at
assert two_day is not None and five_day is not None
assert before < two_day <= before + timedelta(days=2)
assert before + timedelta(days=4) - timedelta(seconds=1) < five_day <= before + timedelta(days=5)
assert five_day - two_day == timedelta(days=3)
await _bulk_update(prisma, [{"user_id": "m1", "budget_duration": None}])
assert _budget_of(prisma, "m1").budget_reset_at is None
assert _budget_of(prisma, "m1").budget_duration is None
assert _budget_of(prisma, "m1").max_budget == 10.0
@pytest.mark.asyncio
async def test_a_member_named_twice_is_written_once_and_the_later_rows_report_the_duplicate():
prisma = _FakePrisma(
teams=[_team("m1")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", max_budget=1.0)],
)
results = await _bulk_update(
prisma,
[
{"user_id": "m1", "max_budget_in_team": 10},
{"user_id": "m1", "max_budget_in_team": 20},
{"user_email": "m1@example.com", "max_budget_in_team": 30},
],
)
assert [(r.success, r.error) for r in results] == [
(True, None),
(False, "Duplicate member in request"),
(False, "Duplicate member in request"),
]
assert _budget_of(prisma, "m1").max_budget == 10.0
@pytest.mark.asyncio
async def test_a_row_naming_somebody_off_the_team_fails_without_writing_while_the_rest_of_the_batch_lands():
prisma = _FakePrisma(
teams=[_team("m1")],
memberships=[_membership("m1", "priv-m1"), _membership("elsewhere", "priv-other")],
budgets=[_budget("priv-m1", max_budget=1.0), _budget("priv-other", max_budget=2.0)],
)
results = await _bulk_update(
prisma,
[
{"user_id": "elsewhere", "max_budget_in_team": 99},
{"user_email": "nobody@example.com", "max_budget_in_team": 99},
{"user_id": "m1", "max_budget_in_team": 10},
],
)
assert [(r.success, r.error) for r in results] == [
(False, "User not found in team"),
(False, "User not found in team"),
(True, None),
]
assert prisma.db.litellm_budgettable.rows["priv-other"].max_budget == 2.0
assert _budget_of(prisma, "m1").max_budget == 10.0
assert set(prisma.db.litellm_budgettable.rows) == {"priv-m1", "priv-other"}
@pytest.mark.asyncio
async def test_each_result_carries_the_limits_read_back_after_the_write_in_request_order():
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")],
budgets=[
_budget("priv-m1", tpm_limit=100, budget_duration="7d"),
_budget("priv-m2", rpm_limit=3),
],
)
results = await _bulk_update(
prisma,
[{"user_id": "m2", "rpm_limit": 8}, {"user_id": "m1", "max_budget_in_team": 42}],
)
assert [r.user_id for r in results] == ["m2", "m1"]
assert (results[1].max_budget, results[1].tpm_limit, results[1].budget_duration) == (42.0, 100, "7d")
assert (results[0].rpm_limit, results[0].max_budget) == (8, None)
@pytest.mark.asyncio
async def test_every_written_member_is_evicted_from_both_team_membership_cache_keys():
prisma = _FakePrisma(
teams=[_team("m1", "m2", "m3")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2"), _membership("m3", "priv-m3")],
budgets=[_budget("priv-m1", max_budget=1.0), _budget("priv-m2", max_budget=2.0), _budget("priv-m3")],
)
cache = _seeded_cache("m1", "m2", "m3")
await _bulk_update(
prisma,
[{"user_id": "m1", "max_budget_in_team": 10}, {"user_id": "m2", "max_budget_in_team": 20}],
cache=cache,
)
assert _cached_keys(cache, "m1") == (None, None)
assert _cached_keys(cache, "m2") == (None, None)
assert _cached_keys(cache, "m3") == ({"cap": "old"}, {"cap": "old"})
@pytest.mark.asyncio
async def test_a_member_with_no_cap_of_their_own_reports_the_team_default_cap_but_only_their_own_rate_limits():
prisma = _FakePrisma(
teams=[_team("m1", default_budget_id="team-default")],
memberships=[],
budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": 7}])
assert [(r.success, r.max_budget, r.max_budget_source, r.tpm_limit) for r in results] == [
(True, 25.0, "team_default", 7)
]
assert _budget_of(prisma, "m1").max_budget is None
default = prisma.db.litellm_budgettable.rows["team-default"]
assert (default.max_budget, default.tpm_limit) == (25.0, 1000)
@pytest.mark.asyncio
async def test_an_explicit_cap_reports_as_the_members_own_while_clearing_one_falls_back_to_the_team_default():
prisma = _FakePrisma(
teams=[_team("m1", "m2", default_budget_id="team-default")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")],
budgets=[
_budget("team-default", max_budget=25.0),
_budget("priv-m1", max_budget=5.0),
_budget("priv-m2", max_budget=9.0),
],
)
results = await _bulk_update(
prisma,
[{"user_id": "m1", "max_budget_in_team": 50}, {"user_id": "m2", "max_budget_in_team": None}],
)
assert [(r.user_id, r.max_budget, r.max_budget_source) for r in results] == [
("m1", 50.0, "member"),
("m2", 25.0, "team_default"),
]
assert results[1].budget_id is None
assert _budget_id_of(prisma, "m2") is None
assert prisma.db.litellm_budgettable.rows["team-default"].max_budget == 25.0
@pytest.mark.asyncio
async def test_a_team_with_no_default_budget_reports_no_effective_cap_for_a_member_without_one():
prisma = _FakePrisma(
teams=[_team("m1")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", tpm_limit=5)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "rpm_limit": 3}])
assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [(True, None, None)]
assert (results[0].tpm_limit, results[0].rpm_limit) == (5, 3)
@pytest.mark.asyncio
async def test_a_zero_team_default_reports_no_cap_because_enforcement_reads_zero_there_as_uncapped():
prisma = _FakePrisma(
teams=[_team("m1", default_budget_id="team-default")],
memberships=[_membership("m1", None)],
budgets=[_budget("team-default", max_budget=0.0)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": 9}])
assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [(True, None, None)]
assert results[0].tpm_limit == 9
@pytest.mark.asyncio
async def test_a_row_that_names_nobody_on_the_team_reports_no_cap_and_no_source():
prisma = _FakePrisma(
teams=[_team("m1", default_budget_id="team-default")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("team-default", max_budget=25.0), _budget("priv-m1", max_budget=5.0)],
)
results = await _bulk_update(
prisma,
[{"user_id": "ghost", "max_budget_in_team": 1}, {"user_id": "m1", "max_budget_in_team": 6}],
)
assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [
(False, None, None),
(True, 6.0, "member"),
]
@pytest.mark.asyncio
async def test_the_roster_authz_read_runs_on_the_writer_so_a_lagging_replica_cannot_let_a_demoted_admin_write():
writer = _FakePrisma(
teams=[_team("lead", "m1")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", max_budget=1.0)],
)
replica = _FakePrisma(teams=[_team("lead", "m1", admins=("lead",))])
demoted = UserAPIKeyAuth(user_id="lead", user_role=LitellmUserRoles.INTERNAL_USER)
with pytest.raises(ManagementProblem) as raised:
await _bulk_update(
_ReplicatedPrisma(writer=writer, reader=replica),
[{"user_id": "m1", "max_budget_in_team": 99}],
caller=demoted,
)
assert raised.value.problem.status == 403
assert writer.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
app = FastAPI()
@app.exception_handler(ManagementProblem)
async def management_problem_exception_handler(request: Request, exc: ManagementProblem):
return problem_response(exc.problem)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
return problem_response(request_validation_problem(exc.errors()))
app.include_router(router)
client = TestClient(app)
BULK_UPDATE_PATH: Final = f"{MANAGEMENT_V1_PREFIX}/teams/{TEAM_ID}/members/bulk_update"
@pytest.fixture
def as_proxy_admin():
app.dependency_overrides[user_api_key_auth] = lambda: ADMIN
yield
app.dependency_overrides.clear()
@pytest.fixture
def as_outsider():
app.dependency_overrides[user_api_key_auth] = lambda: OUTSIDER
yield
app.dependency_overrides.clear()
@pytest.fixture
def prisma(monkeypatch):
fake = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", max_budget=1.0)],
)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake)
return fake
def _post(body: object, path: str = BULK_UPDATE_PATH):
return client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"})
def test_unknown_fields_empty_and_oversized_batches_are_422_problem_documents(prisma, as_proxy_admin):
bodies = (
{"members": [{"user_id": "m1", "max_budget": 10}]},
{"members": [{"user_id": "m1"}], "team_id": TEAM_ID},
{"members": []},
{"members": [{"user_id": f"u{i}"} for i in range(MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES + 1)]},
)
for body in bodies:
response = _post(body)
assert response.status_code == 422, body
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:invalid-request-body"
assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
def test_an_unknown_team_is_a_404_problem_document(prisma, as_proxy_admin):
response = _post(
{"members": [{"user_id": "m1", "max_budget_in_team": 10}]},
path=f"{MANAGEMENT_V1_PREFIX}/teams/nope/members/bulk_update",
)
assert response.status_code == 404
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:team-not-found"
assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
def test_a_caller_who_administers_neither_the_team_nor_its_org_is_a_403_problem_document(prisma, as_outsider):
response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]})
assert response.status_code == 403
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:forbidden"
assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
def test_a_team_admin_may_bulk_update_their_own_teams_members(prisma, monkeypatch):
prisma.db.litellm_teamtable.rows[TEAM_ID] = _team("lead", "m1", admins=("lead",))
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_id="lead", user_role=LitellmUserRoles.INTERNAL_USER
)
try:
response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]})
finally:
app.dependency_overrides.clear()
assert response.status_code == 200
assert [(r["user_id"], r["success"], r["max_budget"]) for r in response.json()["data"]] == [("m1", True, 10.0)]
@pytest.mark.parametrize("duration", ("0d", "nonsense"))
def test_a_budget_duration_no_reset_can_be_scheduled_from_is_a_422_naming_its_row_and_writes_nothing(
prisma, as_proxy_admin, duration
):
response = _post(
{
"members": [
{"user_id": "m1", "max_budget_in_team": 10},
{"user_id": "m2", "budget_duration": duration},
]
}
)
assert response.status_code == 422
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:invalid-request-body"
assert "members.1.budget_duration" in response.json()["detail"]
assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
def test_an_unconnected_database_is_a_503_problem_document(monkeypatch, as_proxy_admin):
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]})
assert response.status_code == 503
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:database-not-connected"
def test_a_driver_error_answers_as_a_problem_document_without_leaking_the_exception(monkeypatch, as_proxy_admin):
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _UnreachablePrisma())
response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]})
assert response.status_code == 500
assert response.headers["content-type"] == "application/problem+json"
assert response.json()["type"] == "urn:litellm:error:internal-server-error"
assert "connection reset by peer" not in response.text

48
tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
View file

@ -7,7 +7,8 @@ from typing import Final
from unittest.mock import AsyncMock, MagicMock, call
import pytest
from fastapi import HTTPException
from fastapi import FastAPI, HTTPException
from httpx import ASGITransport, AsyncClient
from pytest_mock import MockerFixture
from litellm.proxy._types import (
@ -31,6 +32,7 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import (
_handle_group_membership_changes,
_handle_team_membership_changes,
_parse_member_entries,
_premium_user_check,
_process_group_patch_operations,
_recompute_scim_member_roles,
_resolve_group_member_ids,
@ -45,8 +47,10 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import (
patch_group,
patch_team_membership,
patch_user,
scim_router,
update_group,
update_user,
user_api_key_auth,
)
from litellm.types.proxy.management_endpoints.scim_v2 import (
SCIM_ENTERPRISE_USER_SCHEMA,
@ -484,6 +488,48 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp
)
@pytest.fixture
def scim_test_client():
"""An in-process SCIM application with authorization dependencies bypassed."""
app = FastAPI()
app.dependency_overrides[_premium_user_check] = lambda: None
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
app.include_router(scim_router)
return AsyncClient(transport=ASGITransport(app=app), base_url="http://test")
@pytest.mark.asyncio
@pytest.mark.parametrize("endpoint", ["Users", "Groups"])
@pytest.mark.parametrize(("requested_count", "effective_count"), [(0, 0), (200, 100), (1000, 100)])
async def test_scim_collection_endpoints_clamp_requested_page_size(
scim_test_client, endpoint, requested_count, effective_count, mocker
):
"""SCIM list endpoints accept zero and cap larger client page requests."""
mock_prisma_client = MagicMock()
mock_prisma_client.db = MagicMock()
table = MagicMock()
table.find_many = AsyncMock(return_value=[])
table.count = AsyncMock(return_value=0)
mock_prisma_client.db.litellm_usertable = table
mock_prisma_client.db.litellm_teamtable = table
mocker.patch( # test-quality-ok: HTTP validation requires an in-memory database boundary.
"litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception",
AsyncMock(return_value=mock_prisma_client),
)
async with scim_test_client as client:
response = await client.get(f"/scim/v2/{endpoint}?startIndex=1&count={requested_count}")
assert response.status_code == 200
table.find_many.assert_awaited_once_with(
where={},
skip=0,
take=effective_count,
order={"created_at": "desc"},
)
assert response.json()["itemsPerPage"] == 0
@pytest.mark.asyncio
async def test_get_users_filters_username_by_exposed_scim_username_for_okta(mocker):
"""

6
tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
View file

@ -15424,8 +15424,8 @@ async def test_team_info_reports_what_the_caller_may_edit(caller, org_admin, ena
assert response["team_info"].caller_edit_access.model_dump(mode="json") == expected
def test_build_member_budget_patch_maps_temp_budget_fields() -> None:
from litellm.proxy.management_endpoints.team_endpoints import _build_member_budget_patch
def test_member_budget_patch_maps_temp_budget_fields() -> None:
from litellm.proxy.management_endpoints.common_utils import member_budget_patch
expiry: Final = datetime(2030, 1, 1, tzinfo=timezone.utc)
request: Final = TeamMemberUpdateRequest(
@ -15434,7 +15434,7 @@ def test_build_member_budget_patch_maps_temp_budget_fields() -> None:
temp_budget_increase=50.0,
temp_budget_expiry=expiry,
)
assert _build_member_budget_patch(request) == {
assert member_budget_patch(request) == {
"temp_budget_increase": 50.0,
"temp_budget_expiry": expiry,
}

23
tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py
View file

@ -4,15 +4,23 @@ Static checks that every proxy Docker image installs the `bedrock-realtime` extr
Bedrock Nova Sonic speech-to-speech (`/v1/realtime`) needs `aws-sdk-bedrock-runtime`,
which only ships in the `bedrock-realtime` extra. An image whose `uv sync` stages
omit the extra fails every Nova Sonic realtime session with
"Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime".
"Missing aws_sdk_bedrock_runtime: pip install 'litellm[bedrock-realtime]' ...".
"""
import os
import re
import sys
from typing import Final
import pytest
from litellm.constants import BEDROCK_REALTIME_SDK_DISTRIBUTION, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE
if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib
REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "..", "..")
PROXY_DOCKERFILES: Final = (
@ -54,3 +62,16 @@ def test_every_uv_sync_installs_bedrock_realtime_extra(relative_path: str):
"`--extra bedrock-realtime`, so aws-sdk-bedrock-runtime is absent and Bedrock Nova Sonic "
"/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'"
)
def test_bedrock_realtime_extra_pins_the_range_named_in_the_runtime_error():
with open(os.path.join(REPO_ROOT, "pyproject.toml"), "rb") as f:
extra_specs: Final = tomllib.load(f)["project"]["optional-dependencies"]["bedrock-realtime"]
sdk_specs: Final = tuple(spec for spec in extra_specs if spec.startswith(BEDROCK_REALTIME_SDK_DISTRIBUTION))
assert len(sdk_specs) == 1, f"expected exactly one {BEDROCK_REALTIME_SDK_DISTRIBUTION} spec, got {extra_specs}"
requirement: Final = sdk_specs[0].split(";")[0].strip()
assert requirement == f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}", (
f"pyproject pins {requirement!r} but the handler's install hint names "
f"{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE!r} with the awscrt extra; keep them in sync"
)

99
ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx
View file

@ -1,7 +1,7 @@
import React from "react";
import { describe, it, expect, vi, beforeEach } from "vitest";
import userEvent from "@testing-library/user-event";
import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils";
import { renderWithProviders, screen, waitFor, within } from "../../../../tests/test-utils";
import {
GuardrailInformation,
makeBedrockResponse,
@ -24,6 +24,42 @@ const skippedPreCall: Partial<GuardrailInformation> = {
duration: null,
};
const untimedPreCall: Partial<GuardrailInformation> = {
guardrail_name: "conduct",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: null,
end_time: null,
duration: null,
};
const timedPreCall: Partial<GuardrailInformation> = {
guardrail_name: "timed-pre-rail",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: 1_700_000_000,
end_time: 1_700_000_000.1,
duration: 0.1,
};
const latePreCall: Partial<GuardrailInformation> = {
guardrail_name: "late-pre-rail",
guardrail_status: "success",
guardrail_mode: "pre_call",
start_time: 1_700_000_500,
end_time: 1_700_000_500.1,
duration: 0.1,
};
const untimedPostCall: Partial<GuardrailInformation> = {
guardrail_name: "untimed-post-rail",
guardrail_status: "success",
guardrail_mode: "post_call",
start_time: null,
end_time: null,
duration: null,
};
const ranPostCall: Partial<GuardrailInformation> = {
guardrail_name: "ran-rail",
guardrail_status: "success",
@ -98,6 +134,67 @@ describe("GuardrailViewer", () => {
expect(screen.getByText("—")).toBeInTheDocument();
});
it("keeps a guardrail that ran without any timing on the lifecycle", () => {
renderWithProviders(<GuardrailViewer data={makeGuardrailInformation(untimedPreCall)} />);
expect(screen.getByText("Request received")).toBeInTheDocument();
expect(screen.getByText(/Pre-call guardrail: conduct/)).toBeInTheDocument();
expect(screen.getByText("LLM call")).toBeInTheDocument();
expect(screen.getByText("Response returned")).toBeInTheDocument();
expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument();
});
it("keeps an untimed guardrail ahead of a timed one recorded after it in the same phase", () => {
const untimed = makeGuardrailInformation(untimedPreCall);
const timedPre = makeGuardrailInformation(timedPreCall);
renderWithProviders(<GuardrailViewer data={[untimed, timedPre]} />);
const rows = screen.getAllByTestId("lifecycle-row");
const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null);
const untimedIndex = rowIndex(/Pre-call guardrail: conduct/);
const timedIndex = rowIndex(/Pre-call guardrail: timed-pre-rail/);
expect(untimedIndex).toBeGreaterThanOrEqual(0);
expect(timedIndex).toBeGreaterThanOrEqual(0);
expect(untimedIndex).toBeLessThan(timedIndex);
});
it("orders each phase on its own clock when a later pre-call outlives an earlier post-call", () => {
const latePre = makeGuardrailInformation(latePreCall);
const untimedPost = makeGuardrailInformation(untimedPostCall);
const earlyPost = makeGuardrailInformation(ranPostCall);
renderWithProviders(<GuardrailViewer data={[latePre, untimedPost, earlyPost]} />);
const rows = screen.getAllByTestId("lifecycle-row");
const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null);
const untimedIndex = rowIndex(/Post-call guardrail: untimed-post-rail/);
const earlyIndex = rowIndex(/Post-call guardrail: ran-rail/);
expect(untimedIndex).toBeGreaterThanOrEqual(0);
expect(earlyIndex).toBeGreaterThanOrEqual(0);
expect(untimedIndex).toBeLessThan(earlyIndex);
});
it("anchors offsets on the timed entries and gives the untimed one no fabricated offset", () => {
const untimed = makeGuardrailInformation(untimedPreCall);
const ran = makeGuardrailInformation(ranPostCall);
renderWithProviders(<GuardrailViewer data={[untimed, ran]} />);
const lifecycleRow = (label: string | RegExp): HTMLElement => {
const row = screen.getAllByTestId("lifecycle-row").find((r) => within(r).queryByText(label) !== null);
if (row === undefined) throw new Error(`no lifecycle row labelled ${label}`);
return row;
};
expect(within(lifecycleRow("Request received")).getByText("T+0ms")).toBeInTheDocument();
expect(within(lifecycleRow(/Post-call guardrail: ran-rail/)).getByText("T+250ms")).toBeInTheDocument();
expect(within(lifecycleRow("Response returned")).getByText("T+251ms")).toBeInTheDocument();
const untimedRow = within(lifecycleRow(/Pre-call guardrail: conduct/));
expect(untimedRow.getByText("—")).toBeInTheDocument();
expect(untimedRow.queryByText(/^T\+/)).not.toBeInTheDocument();
});
it("calculates and displays masked entity totals", async () => {
const user = userEvent.setup();
const data = makeGuardrailInformation({

60
ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx
View file

@ -361,7 +361,7 @@ const GenericGuardrailResponse = ({ response }: { response: any }) => {
interface TimelineEntry {
type: "request" | "guardrail" | "llm" | "response";
label: string;
offsetMs: number;
offsetMs: number | null;
outcome?: EntryOutcome;
}
@ -370,73 +370,85 @@ type TimedGuardrailInformation = GuardrailInformation & { start_time: number; en
const isTimed = (e: GuardrailInformation): e is TimedGuardrailInformation =>
typeof e.start_time === "number" && typeof e.end_time === "number";
const belongsOnLifecycle = (e: GuardrailInformation): boolean => isTimed(e) || getEntryOutcome(e) !== "not_run";
// Sorts a phase's timed entries by start time while leaving its untimed entries in the
// slots they were recorded in. Applied per phase, never globally: an entry can land in
// more than one phase bucket, so a global pass can reorder one phase by another's clock.
const orderWithinPhase = (group: GuardrailInformation[]): GuardrailInformation[] => {
const byStart = group.filter(isTimed).sort((a, b) => a.start_time - b.start_time);
const timedSlots = new Map(group.flatMap((e, i) => (isTimed(e) ? [i] : [])).map((slot, k) => [slot, byStart[k]]));
return group.map((e, i) => timedSlots.get(i) ?? e);
};
const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
const sorted = useMemo(() => entries.filter(isTimed).sort((a, b) => a.start_time - b.start_time), [entries]);
const sorted = useMemo(() => entries.filter(belongsOnLifecycle), [entries]);
const timeline = useMemo(() => {
if (sorted.length === 0) return [];
const baseTime = sorted[0].start_time;
const timed = sorted.filter(isTimed);
const baseTime = timed.length > 0 ? Math.min(...timed.map((e) => e.start_time)) : null;
const offsetOf = (e: GuardrailInformation): number | null =>
baseTime === null || !isTimed(e) ? null : Math.round((e.end_time - baseTime) * 1000);
const items: TimelineEntry[] = [];
// Request received
items.push({ type: "request", label: "Request received", offsetMs: 0 });
items.push({ type: "request", label: "Request received", offsetMs: baseTime === null ? null : 0 });
// Pre-call guardrails — use modeMatches so array modes (e.g. ["pre_call", "post_call"])
// place the entry in every matching bucket.
const preCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call"));
const postCalls = sorted.filter(
(e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only"),
const preCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call")));
const postCalls = orderWithinPhase(
sorted.filter((e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only")),
);
const duringCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call"));
const duringCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call")));
for (const e of preCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `Pre-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// LLM call — infer from gap between pre-call end and post-call start
const lastPreEnd = preCalls.length > 0 ? Math.max(...preCalls.map((e) => e.end_time)) : baseTime;
const firstPostStart = postCalls.length > 0 ? Math.min(...postCalls.map((e) => e.start_time)) : undefined;
const llmEndTime = firstPostStart ?? lastPreEnd + 1;
const llmOffsetMs = Math.round((llmEndTime - baseTime) * 1000);
const timedPre = preCalls.filter(isTimed);
const timedPost = postCalls.filter(isTimed);
const lastPreEnd = timedPre.length > 0 ? Math.max(...timedPre.map((e) => e.end_time)) : baseTime;
const firstPostStart = timedPost.length > 0 ? Math.min(...timedPost.map((e) => e.start_time)) : undefined;
const llmEndTime = firstPostStart ?? (lastPreEnd === null ? null : lastPreEnd + 1);
items.push({
type: "llm",
label: "LLM call",
offsetMs: llmOffsetMs,
offsetMs: llmEndTime === null || baseTime === null ? null : Math.round((llmEndTime - baseTime) * 1000),
});
// During-call guardrails (rare)
for (const e of duringCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `During-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// Post-call guardrails
for (const e of postCalls) {
const offsetMs = Math.round((e.end_time - baseTime) * 1000);
items.push({
type: "guardrail",
label: `Post-call guardrail: ${getDisplayName(e)}`,
offsetMs,
offsetMs: offsetOf(e),
outcome: getEntryOutcome(e),
});
}
// Response returned
const maxEnd = Math.max(...sorted.map((e) => e.end_time));
const responseOffsetMs = Math.round((maxEnd - baseTime) * 1000) + 1;
const maxEnd = timed.length > 0 ? Math.max(...timed.map((e) => e.end_time)) : null;
const responseOffsetMs = maxEnd === null || baseTime === null ? null : Math.round((maxEnd - baseTime) * 1000) + 1;
items.push({ type: "response", label: "Response returned", offsetMs: responseOffsetMs });
return items;
@ -447,7 +459,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
<h4 className="text-xs font-semibold text-muted-foreground uppercase tracking-wider mb-4">Request Lifecycle</h4>
<div className="relative">
{timeline.map((item, idx) => (
<div key={idx} className="flex items-start gap-3 relative">
<div key={idx} data-testid="lifecycle-row" className="flex items-start gap-3 relative">
{/* Vertical line */}
<div className="flex flex-col items-center">
<div className="shrink-0">
@ -475,7 +487,9 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {
{OUTCOME_LABEL[item.outcome]}
</span>
)}
<span className="text-xs text-muted-foreground font-mono ml-auto shrink-0">T+{item.offsetMs}ms</span>
<span className="text-xs text-muted-foreground font-mono ml-auto shrink-0">
{item.offsetMs === null ? "—" : `T+${item.offsetMs}ms`}
</span>
</div>
</div>
</div>

143
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -8544,6 +8544,45 @@ export interface paths {
patch?: never;
trace?: never;
};
"/management/v1/teams/{team_id}/members/bulk_update": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Bulk Update Team Member Budgets Action
* @description Set per-member limits for up to 500 members of one team in one call. Same
* authorization and member addressing as `/team/member_update`: proxy admins, the team's
* admins, and admins of the team's organization, with each member named by exactly one of
* `user_id` or `user_email`. Unknown body fields are a 422 and an unknown team is a 404.
*
* Each row is a merge patch of that member's limits: a field left out is untouched, a
* field sent as null is cleared, and clearing the last limit drops the member back to the
* team default. A budget row shared by several memberships, the team default included, is
* copied for the member being patched rather than written in place, so one member's new
* cap never lands on anybody else.
*
* `data` holds one result per requested member, in request order, carrying the limits in
* force after the write. A row is `success: false` with an `error` when it names nobody on
* the team or repeats an earlier row. Roles are not part of this route; `/team/member_update`
* still owns them.
*
* Example curl:
* ```
* curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_update' --header 'Authorization: Bearer sk-1234' --header 'Content-Type: application/json' --data '{"members": [{"user_id": "user-1", "max_budget_in_team": 10}, {"user_email": "user-2@example.com", "max_budget_in_team": 10, "budget_duration": "30d"}]}'
* ```
*/
post: operations["bulk_update_team_member_budgets_action_management_v1_teams__team_id__members_bulk_update_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/management/v1/users/bulk": {
parameters: {
query?: never;
@ -24930,6 +24969,22 @@ export interface components {
[key: string]: unknown;
} | null;
};
/**
* BulkTeamMemberBudgetUpdateRequest
* @description Body of `POST /management/v1/teams/{team_id}/members/bulk_update`.
*/
BulkTeamMemberBudgetUpdateRequest: {
/** Members */
members: components["schemas"]["TeamMemberBudgetPatch"][];
};
/**
* BulkTeamMemberBudgetUpdateResponse
* @description `{data: [...]}` with one `TeamMemberBudgetUpdateResult` per requested member, in request order.
*/
BulkTeamMemberBudgetUpdateResponse: {
/** Data */
data: components["schemas"]["TeamMemberBudgetUpdateResult"][];
};
/**
* BulkTeamMemberDeleteRequest
* @description Body of `POST /management/v1/teams/{team_id}/members/bulk_delete`.
@ -37945,6 +38000,57 @@ export interface components {
/** User Id */
user_id?: string | null;
};
/**
* TeamMemberBudgetPatch
* @description One member's per-member limits, merge-patch style: a field left out of the row is
* untouched, a field sent as null is cleared, and clearing the last limit drops the
* member back to the team default.
*/
TeamMemberBudgetPatch: {
/** Allowed Models */
allowed_models?: string[] | null;
/** Budget Duration */
budget_duration?: string | null;
/** Max Budget In Team */
max_budget_in_team?: number | null;
/** Rpm Limit */
rpm_limit?: number | null;
/** Tpm Limit */
tpm_limit?: number | null;
/** User Email */
user_email?: string | null;
/** User Id */
user_id?: string | null;
};
/**
* TeamMemberBudgetUpdateResult
* @description Outcome for one requested member, in request order, carrying the limits in force
* after the write rather than the ones that were asked for.
*/
TeamMemberBudgetUpdateResult: {
/** Allowed Models */
allowed_models?: string[] | null;
/** Budget Duration */
budget_duration?: string | null;
/** Budget Id */
budget_id?: string | null;
/** Error */
error?: string | null;
/** Max Budget */
max_budget?: number | null;
/** Max Budget Source */
max_budget_source?: ("member" | "team_default") | null;
/** Rpm Limit */
rpm_limit?: number | null;
/** Success */
success: boolean;
/** Tpm Limit */
tpm_limit?: number | null;
/** User Email */
user_email?: string | null;
/** User Id */
user_id?: string | null;
};
/** TeamMemberDeleteRequest */
TeamMemberDeleteRequest: {
/** Team Id */
@ -38000,7 +38106,7 @@ export interface components {
};
/**
* TeamMemberRef
* @description One member to remove, named by exactly one of `user_id` or `user_email`.
* @description One member, named by exactly one of `user_id` or `user_email`.
*/
TeamMemberRef: {
/** User Email */
@ -52113,6 +52219,41 @@ export interface operations {
};
};
};
bulk_update_team_member_budgets_action_management_v1_teams__team_id__members_bulk_update_post: {
parameters: {
query?: never;
header?: never;
path: {
team_id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["BulkTeamMemberBudgetUpdateRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["BulkTeamMemberBudgetUpdateResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
bulk_create_users_route_management_v1_users_bulk_post: {
parameters: {
query?: never;

49
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-09-12T22:48:38.53978Z"
exclude-newer = "2026-09-14T20:32:38.482736111Z"
exclude-newer-span = "P3D"
[manifest]
@ -535,16 +535,21 @@ wheels = [
[[package]]
name = "aws-sdk-bedrock-runtime"
version = "0.7.0"
version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", extra = ["aiohttp"], marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/67/8a/ed3fd98775273b0b7f6006b4970aa876d506668b7fe29145f54fcb941c3b/aws_sdk_bedrock_runtime-0.7.0.tar.gz", hash = "sha256:0cb172cbc03ff060e5c1d6f9cfa9a8ac5e71d9e0d58d3117006ebf614cbb4677", size = 170304, upload-time = "2026-06-23T04:04:52.382Z" }
sdist = { url = "https://files.pythonhosted.org/packages/8e/b3/9c225cbfe9f17ea2e3d75a0fdd0b325ef79839b9c09a376bda63a7bf3bb3/aws_sdk_bedrock_runtime-0.11.0.tar.gz", hash = "sha256:f2c45d34625bf6a7b56375e29a53a16b376880bda771e4bbf7d84491622eb193", size = 173854, upload-time = "2026-08-24T21:17:16.304Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9e/e1/f86d50f0ad9c8200645f315c524d285e86b30b94bb65118e1108597714e6/aws_sdk_bedrock_runtime-0.7.0-py3-none-any.whl", hash = "sha256:de67ede6f441bbb77ef61c237945d559513843fc827abe1af12535c2519650c5", size = 94948, upload-time = "2026-06-23T04:04:51.281Z" },
{ url = "https://files.pythonhosted.org/packages/29/0c/9512304ed017ce49992df6661eac2b914550247e13bccb55be6ca594170d/aws_sdk_bedrock_runtime-0.11.0-py3-none-any.whl", hash = "sha256:ef01c26ddfd83a5d3e438ab72ebb3c13b41fc0ef11d81095b22c8016f97e9795", size = 97112, upload-time = "2026-08-24T21:17:17.396Z" },
]
[package.optional-dependencies]
awscrt = [
{ name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
]
[[package]]
@ -4483,7 +4488,7 @@ dependencies = [
[package.optional-dependencies]
bedrock-realtime = [
{ name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12'" },
{ name = "aws-sdk-bedrock-runtime", extra = ["awscrt"], marker = "python_full_version >= '3.12'" },
]
caching = [
{ name = "diskcache" },
@ -4693,7 +4698,7 @@ requires-dist = [
{ name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" },
{ name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" },
{ name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.0.19,<1.0" },
{ name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.7.0,<0.8.0" },
{ name = "aws-sdk-bedrock-runtime", extras = ["awscrt"], marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.10.0,<0.12.0" },
{ name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = ">=1.0.0,<2.0" },
{ name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = ">=1.25.2,<2.0" },
{ name = "azure-identity", marker = "extra == 'proxy'", specifier = ">=1.25.2,<2.0" },
@ -4884,7 +4889,7 @@ source = { editable = "enterprise" }
[[package]]
name = "litellm-proxy-extras"
version = "0.4.98"
version = "0.4.99"
source = { editable = "litellm-proxy-extras" }
[[package]]
@ -9126,16 +9131,16 @@ wheels = [
[[package]]
name = "smithy-aws-core"
version = "0.7.0"
version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
{ name = "smithy-http", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fc/a8/37bfde59519f45d2047d0033b791aca6574d867aaf57bb56a6de42ab5c26/smithy_aws_core-0.7.0.tar.gz", hash = "sha256:34e82d09fc808acd5ffc80f03828d0609c6a211f49f0884dc6ee7ca095a1b6af", size = 15670, upload-time = "2026-06-23T04:04:50.365Z" }
sdist = { url = "https://files.pythonhosted.org/packages/7d/d3/501c0023548173416109ac42298ca33b708469dc922005770811a597949f/smithy_aws_core-0.11.0.tar.gz", hash = "sha256:29ee89976a520a87e3db557e03e115fdc21a0a60b81161e95174395a1b064da1", size = 38791, upload-time = "2026-08-24T21:16:59.631Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fd/54/2d06dd9a3972a380d71bb8c3312e317aa8f1ea68dd28cffc06955ccf0220/smithy_aws_core-0.7.0-py3-none-any.whl", hash = "sha256:6c60c8fbb9431c60e80ea7f2d37e7ae48409cc1541f587fe073f202eca067e92", size = 24894, upload-time = "2026-06-23T04:04:49.349Z" },
{ url = "https://files.pythonhosted.org/packages/e4/f6/fefda9aab809fa1a62bf7073bd6d8ab427bd9989f39b13c0d6e29d4d1045/smithy_aws_core-0.11.0-py3-none-any.whl", hash = "sha256:77cf130c22deac14a8cbeb8ccc4bcfe5a91798f4b38cb53a987080ec58c89f23", size = 58855, upload-time = "2026-08-24T21:16:58.657Z" },
]
[package.optional-dependencies]
@ -9160,41 +9165,45 @@ wheels = [
[[package]]
name = "smithy-core"
version = "0.6.0"
version = "0.8.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/e9/45/688d52c61cd4d843bb230694259e91d4c7d6954eeecbadf452a168001d45/smithy_core-0.6.0.tar.gz", hash = "sha256:ba2e5d860d716aff75004a23f53e09dfaca3e2b94f8a00c1f76dcb355b769ce0", size = 52095, upload-time = "2026-06-23T04:04:44.687Z" }
sdist = { url = "https://files.pythonhosted.org/packages/7c/c6/93e9eea3c6163228dfe972c3e989e0553047858805ab7aa4a59f074ba129/smithy_core-0.8.1.tar.gz", hash = "sha256:3d2f8fca5960d74bd7ef380f70901c7bcdebe53f929d2d3d2fa6cb790b3f5214", size = 54259, upload-time = "2026-08-20T17:55:30.354Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/b6/06795faa9844b9667ae492e6293370393e19e7f0c2df8da1b4bf7e5f6ed9/smithy_core-0.6.0-py3-none-any.whl", hash = "sha256:51e347ed309d60ab9d36b783dbf88de614c460d51bec79d39cd403956b00f063", size = 66879, upload-time = "2026-06-23T04:04:43.596Z" },
{ url = "https://files.pythonhosted.org/packages/0c/23/c6430bbf406477fc7d16254b9908a723b299a4a21a94c99db9d12c84a8bf/smithy_core-0.8.1-py3-none-any.whl", hash = "sha256:44bd9bdf702f76919af58e44a6a1bb3dc136a745b2f955281743022ce767e347", size = 68805, upload-time = "2026-08-20T17:55:29.366Z" },
]
[[package]]
name = "smithy-http"
version = "0.4.2"
version = "0.5.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/58/5a772d212e066d6fc1398946c4aae19bcdaa75209879d776f641b6a06b5b/smithy_http-0.4.2.tar.gz", hash = "sha256:50d11b6a55e42448450a01e3d0f605ccee65a72abf52d02eed82862a15be5937", size = 29616, upload-time = "2026-06-23T04:04:45.687Z" }
sdist = { url = "https://files.pythonhosted.org/packages/98/78/b5f3113d6c8f0bc1f9777a7f5ca84b892d29efac05850e14f7d4f7e645b5/smithy_http-0.5.0.tar.gz", hash = "sha256:bb4a19672f7c7eeb872a308f777eb505281a5bafb1ee3d1ea9c760c06c352510", size = 31122, upload-time = "2026-08-24T21:16:56.488Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/57/3e/7b2464d40893bec0b5d1f479d25116d4aa09f9f66536b4c4b3126202215d/smithy_http-0.4.2-py3-none-any.whl", hash = "sha256:a158f107e9fab925289d20772c2e38b0bba94e55c05d0edc9290310f22a60454", size = 41025, upload-time = "2026-06-23T04:04:46.764Z" },
{ url = "https://files.pythonhosted.org/packages/27/27/e414082643028846b73afa52a1a8f934548196ee12b187a06803f02a3e66/smithy_http-0.5.0-py3-none-any.whl", hash = "sha256:af273d5f42e7733ce7a6e9bd6fdd6a59ef1b61f6cd1f4a89dd53dfce99da7bef", size = 42198, upload-time = "2026-08-24T21:16:57.52Z" },
]
[package.optional-dependencies]
aiohttp = [
{ name = "aiohttp", marker = "python_full_version >= '3.12'" },
{ name = "yarl", marker = "python_full_version >= '3.12'" },
]
awscrt = [
{ name = "awscrt", marker = "python_full_version >= '3.12'" },
]
[[package]]
name = "smithy-json"
version = "0.2.3"
version = "0.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ijson", marker = "python_full_version >= '3.12'" },
{ name = "smithy-core", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/6c/418b5687d8933b7a135d5e1a98c61fe814b98f72517dbae0e666860cb876/smithy_json-0.2.3.tar.gz", hash = "sha256:686e9b55a36dacb08e472732b358573ef78009055e05e9fce2e806d61490b2b3", size = 7805, upload-time = "2026-06-23T04:04:47.71Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c7/ac/04164eefb3da7479f52f6535b4b39cc8384c292cb2bb74279f2acc4f4b4d/smithy_json-0.3.0.tar.gz", hash = "sha256:c81c7034587e01bc64767cbbecb05a7d65ca9070612fd94e8a03e80540290a22", size = 7956, upload-time = "2026-08-20T17:55:32.177Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3c/14/eabb26b355415bcd9feef27fb5b18f1dad3fabd4208cfcbaf152025fa9ae/smithy_json-0.2.3-py3-none-any.whl", hash = "sha256:594e1bbe3d480963237f8fd0fc648dbd4e988b4503fea90157b5f07706796327", size = 10252, upload-time = "2026-06-23T04:04:48.46Z" },
{ url = "https://files.pythonhosted.org/packages/9d/cf/0104c40a0e18fa307ea3da4310eba949f474a5bc1df3cc2b5851a72e8486/smithy_json-0.3.0-py3-none-any.whl", hash = "sha256:ffb73d2e60cf5e616e5d0a1019e7b9f518edba076cb423f10981457725dcddc4", size = 10252, upload-time = "2026-08-20T17:55:31.204Z" },
]
[[package]]