diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py index e6b3744019c..c9c91e3283b 100644 --- a/cookbook/litellm_proxy_server/cli_token_usage.py +++ b/cookbook/litellm_proxy_server/cli_token_usage.py @@ -3,7 +3,7 @@ Example: Using CLI token with LiteLLM SDK This example shows how to use the CLI authentication token -in your Python scripts after running `litellm-proxy login`. +in your Python scripts after running `lite login`. """ from textwrap import indent @@ -22,7 +22,7 @@ def main(): api_key = litellm.get_litellm_gateway_api_key() if not api_key: - print("āŒ No CLI token found. Please run 'litellm-proxy login' first.") + print("āŒ No CLI token found. Please run 'lite login' first.") return print("āœ… Found CLI token.") @@ -58,6 +58,6 @@ if __name__ == "__main__": main() print("\nšŸ’” Tips:") - print("1. Run 'litellm-proxy login' to authenticate first") + print("1. Run 'lite login' to authenticate first") print("2. Replace 'https://your-proxy.com' with your actual proxy URL") print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none") diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 914b9c5a14b..604ffc3abd4 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.98" +version = "0.4.99" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.98" +version = "0.4.99" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 81e2af45686..66be77dbb40 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -521,6 +521,18 @@ class DualCache(BaseCache): if self.redis_cache is not None: await self.redis_cache.async_delete_cache(key) + async def async_delete_cache_keys(self, keys: Sequence[str]) -> None: + """Batch twin of ``async_delete_cache``, chunked because Redis takes the + whole list as one DELETE command.""" + if not keys: + return + for key in keys: + self.in_memory_cache.delete_cache(key) + if self.redis_cache is None: + return + for start in range(0, len(keys), DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE): + await self.redis_cache.delete_cache_keys(keys[start : start + DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE]) + async def async_get_ttl(self, key: str) -> int | None: """ Get the remaining TTL of a key in in-memory cache or redis diff --git a/litellm/constants.py b/litellm/constants.py index 8409a161800..d4827bb7483 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -320,6 +320,8 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update" BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed" BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure" +BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime" +BEDROCK_REALTIME_SDK_SUPPORTED_RANGE: Final = ">=0.10.0,<0.12.0" CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model" MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved" REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged" diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 95099924dcf..373435fa4ee 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -508,7 +508,8 @@ class AnthropicMessagesHandler(BaseTranslation): chat_completion_compatible_request, _tool_name_mapping, ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( - anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()) + anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()), + preserve_midturn_system=True, ) return chat_completion_compatible_request diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index ed01d16bd1b..7eb56ae55d3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -118,6 +118,10 @@ from litellm.llms.anthropic.common_utils import ( from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, ) +from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import ( + convert_mid_conversation_system_turns, + is_system_role_message, +) from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( openai_chat_refusal_text, refusal_stop_details, @@ -176,6 +180,7 @@ from litellm.types.llms.openai import ( ToolMessageContentPart, ) from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicStreamWrapper @@ -186,6 +191,12 @@ if TYPE_CHECKING: ToolResultContent: TypeAlias = str | list[ToolMessageContentPart] +def target_supports_mid_conversation_system(model: str | None, custom_llm_provider: str | None) -> bool: + if not model: + return False + return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) + + class AnthropicAdapter: def __init__(self) -> None: pass @@ -418,10 +429,28 @@ class LiteLLMAnthropicMessagesAdapter: self, messages: list[AllAnthropicPassThroughMessageValues], model: str | None = None, + *, + custom_llm_provider: str | None = None, + preserve_midturn_system: bool = False, ) -> list: new_messages: Final[list[AllMessageValues]] = [] replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) - for m in replayable_messages: + leading_count: Final = next( + (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), + len(replayable_messages), + ) + trailing_messages: Final = replayable_messages[leading_count:] + keeps_midturn_system: Final = ( + preserve_midturn_system + or not any(is_system_role_message(m) for m in trailing_messages) + or target_supports_mid_conversation_system(model, custom_llm_provider) + ) + ordered_messages: Final = ( + replayable_messages + if keeps_midturn_system + else (*replayable_messages[:leading_count], *convert_mid_conversation_system_turns(trailing_messages)) + ) + for m in ordered_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] @@ -494,7 +523,7 @@ class LiteLLMAnthropicMessagesAdapter: if isinstance(m.get("content"), str): assistant_message_str = str(m.get("content", "")) elif isinstance(m.get("content"), list): - for content in m.get("content", []): + for content in cast(list, m.get("content", [])): # cast-ok: untrusted client payload if isinstance(content, str): assistant_message_str = str(content) elif isinstance(content, dict): @@ -1154,6 +1183,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + preserve_midturn_system: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1175,6 +1205,8 @@ class LiteLLMAnthropicMessagesAdapter: new_messages = self.translate_anthropic_messages_to_openai( messages=messages_list, model=anthropic_message_request.get("model"), + custom_llm_provider=custom_llm_provider, + preserve_midturn_system=preserve_midturn_system, ) ## ADD SYSTEM MESSAGE TO MESSAGES self._add_system_message_to_messages(new_messages, anthropic_message_request) diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index ecaf8f2e7e1..ebd0342b10c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -765,7 +765,8 @@ def _count_effective_tokens( messages=cast( "list[AllAnthropicPassThroughMessageValues]", messages_without_compaction, - ) + ), + preserve_midturn_system=True, ) except Exception as e: verbose_logger.debug( @@ -920,7 +921,8 @@ def _build_summary_messages( messages=cast( "list[AllAnthropicPassThroughMessageValues]", stripped, - ) + ), + preserve_midturn_system=True, ) except Exception as e: verbose_logger.warning( diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py b/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py new file mode 100644 index 00000000000..ddefec6bac9 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py @@ -0,0 +1,77 @@ +from collections.abc import Mapping, Sequence +from itertools import groupby +from typing import Final + +CONVERTED_SYSTEM_NOTE: Final = ( + "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." +) + + +def as_system_content_blocks(value: object) -> list[object]: + if value is None: + return [] + if isinstance(value, list): + return list(value) + if isinstance(value, str): + return [{"type": "text", "text": value}] + return [value] + + +def is_system_role_message(message: object) -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + +def system_role_message_as_user(message: Mapping[str, object]) -> Mapping[str, object]: + return { + "role": "user", + "content": as_system_content_blocks(CONVERTED_SYSTEM_NOTE) + as_system_content_blocks(message.get("content")), + } + + +def opens_with_tool_results(message: object) -> bool: + if not isinstance(message, dict) or message.get("role") != "user": + return False + content: Final = message.get("content") + return ( + isinstance(content, list) + and len(content) > 0 + and isinstance(content[0], dict) + and content[0].get("type") == "tool_result" + ) + + +def system_run_placed_after_tool_results( + system_run: Sequence[Mapping[str, object]], follower_run: Sequence[Mapping[str, object]] +) -> tuple[Mapping[str, object], ...]: + if follower_run and opens_with_tool_results(follower_run[0]): + return (follower_run[0], *system_run, *follower_run[1:]) + return (*system_run, *follower_run) + + +def system_turns_after_tool_results( + messages: Sequence[Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + runs: Final = tuple(tuple(run) for _, run in groupby(messages, key=is_system_role_message)) + if not runs: + return () + first_system_run: Final = 0 if is_system_role_message(runs[0][0]) else 1 + paired_runs: Final = tuple( + (runs[i], runs[i + 1] if i + 1 < len(runs) else ()) for i in range(first_system_run, len(runs), 2) + ) + return ( + *(runs[0] if first_system_run else ()), + *( + m + for system_run, follower_run in paired_runs + for m in system_run_placed_after_tool_results(system_run, follower_run) + ), + ) + + +def convert_mid_conversation_system_turns( + messages: Sequence[Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + return tuple( + system_role_message_as_user(m) if is_system_role_message(m) else m + for m in system_turns_after_tool_results(messages) + ) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 27cdac34116..5fa686b7560 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -27,6 +27,11 @@ from ...common_utils import ( strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) +from .mid_conversation_system import ( + as_system_content_blocks, + convert_mid_conversation_system_turns, + is_system_role_message, +) DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -151,73 +156,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param - @staticmethod - def _as_system_content_blocks(value: object) -> list: - if value is None: - return [] - if isinstance(value, list): - return list(value) - if isinstance(value, str): - return [{"type": "text", "text": value}] - return [value] - - @staticmethod - def _is_system_role_message(message: object) -> bool: - return isinstance(message, dict) and message.get("role") == "system" - - _CONVERTED_SYSTEM_NOTE: Final = ( - "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." - ) - - def _system_role_message_as_user(self, message: Mapping) -> Mapping: - return { - "role": "user", - "content": self._as_system_content_blocks(self._CONVERTED_SYSTEM_NOTE) - + self._as_system_content_blocks(message.get("content")), - } - - @staticmethod - def _opens_with_tool_results(message: object) -> bool: - if not isinstance(message, dict) or message.get("role") != "user": - return False - content: Final = message.get("content") - return ( - isinstance(content, list) - and len(content) > 0 - and isinstance(content[0], dict) - and content[0].get("type") == "tool_result" - ) - - def _system_run_before(self, messages: Sequence, index: int) -> Sequence: - start: Final = next( - (j + 1 for j in range(index - 1, -1, -1) if not self._is_system_role_message(messages[j])), - 0, - ) - return messages[start:index] - - def _system_run_end(self, messages: Sequence, index: int) -> int: - return next( - (j for j in range(index, len(messages)) if not self._is_system_role_message(messages[j])), - len(messages), - ) - - def _reordered_around_tool_results(self, messages: Sequence, index: int) -> tuple: - message: Final = messages[index] - if self._opens_with_tool_results(message): - return (message, *self._system_run_before(messages, index)) - if not self._is_system_role_message(message): - return (message,) - run_end: Final = self._system_run_end(messages, index) - follower: Final = messages[run_end] if run_end < len(messages) else None - return () if self._opens_with_tool_results(follower) else (message,) - - def _system_turns_after_tool_results(self, messages: Sequence) -> tuple: - return tuple( - message - for index in range(len(messages)) - for message in self._reordered_around_tool_results(messages, index) - ) - def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, @@ -254,7 +192,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if not isinstance(messages, list): return leading_count: Final = next( - (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + (i for i, m in enumerate(messages) if not is_system_role_message(m)), len(messages), ) hoisted: Final = messages[:leading_count] @@ -265,10 +203,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self.custom_llm_provider, key="supports_mid_conversation_system", ) - else [ - self._system_role_message_as_user(m) if self._is_system_role_message(m) else m - for m in self._system_turns_after_tool_results(messages[leading_count:]) - ] + else list(convert_mid_conversation_system_turns(messages[leading_count:])) ) if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining @@ -278,7 +213,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): anthropic_messages_request.get("system"), *(m.get("content") for m in hoisted), ) - for block in self._as_system_content_blocks(source) + for block in as_system_content_blocks(source) ] filtered_system: Final = self._filter_billing_headers_from_system(system_content) if filtered_system: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index cb2c70e74c8..20c8258e440 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -34,6 +34,7 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" +_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") def error_response_text(response: httpx.Response) -> str: @@ -878,9 +879,10 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool: """ Whether Converse ``cachePoint`` blocks may be sent to this model. - Bedrock rejects requests carrying cachePoint blocks for models without prompt - caching support ("You invoked an unsupported model or your request did not allow - prompt caching"), so a model whose cost-map entry does not declare + OpenAI-family models only support implicit caching and never accept explicit + ``cachePoint`` blocks. Bedrock rejects requests carrying cachePoint blocks for + models without prompt caching support ("You invoked an unsupported model or your + request did not allow prompt caching"), so a model whose cost-map entry does not declare ``supports_prompt_caching`` must not receive them. A model absent from the map (an application inference profile ARN, a model newer than the map) keeps emitting so existing caching setups never silently degrade. ``litellm.utils.supports_prompt_caching`` @@ -888,6 +890,8 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool: """ if model is None: return True + if _OPENAI_FAMILY_MODEL_RE.search(model): + return False entries: Final = tuple( entry for candidate in (model, get_bedrock_base_model(model)) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 2c1ce6068b2..fe3822629a7 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -6,11 +6,12 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic. import asyncio import contextlib +import importlib.metadata import json -from collections.abc import AsyncIterator, Mapping, MutableMapping +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping from dataclasses import dataclass from types import MappingProxyType -from typing import Final, NoReturn, Protocol +from typing import Final, NoReturn, Protocol, runtime_checkable from pydantic import JsonValue, TypeAdapter @@ -19,6 +20,8 @@ from litellm._logging import _redact_string, verbose_proxy_logger from litellm.constants import ( BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY, + BEDROCK_REALTIME_SDK_DISTRIBUTION, + BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) @@ -121,6 +124,38 @@ class BedrockBidirectionalStream(Protocol): async def await_output(self) -> tuple[object, BedrockOutputStream]: ... +@runtime_checkable +class ClosableBedrockRuntimeClient(Protocol): + async def close(self) -> None: ... + + +def _installed_sdk_version() -> str | None: + try: + return importlib.metadata.version(BEDROCK_REALTIME_SDK_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return None + + +def _sdk_import_error(installed_version: str | None, cause: ImportError) -> ImportError: + install_hint: Final = "pip install 'litellm[bedrock-realtime]'" + requirement: Final = f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}" + verbose_proxy_logger.error("Bedrock Realtime: SDK import failed (installed=%s): %s", installed_version, cause) + if installed_version is None: + return ImportError(f"Missing aws_sdk_bedrock_runtime: {install_hint} ({requirement})") + return ImportError( + f"{BEDROCK_REALTIME_SDK_DISTRIBUTION} {installed_version} is installed but Bedrock realtime needs " + f"[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}: {install_hint}" + ) + + +async def _close_bedrock_client(bedrock_client: object) -> None: + if not isinstance(bedrock_client, ClosableBedrockRuntimeClient): + return + with contextlib.suppress(Exception): + await bedrock_client.close() + verbose_proxy_logger.debug("Bedrock Realtime: closed SDK client") + + @dataclass(frozen=True, slots=True) class _BridgeOutcome: logged_events: tuple[OpenAIRealtimeEvents, ...] @@ -199,8 +234,9 @@ async def _ack_session_update( class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" - def __init__(self): + def __init__(self, sdk_version_lookup: Callable[[], str | None] = _installed_sdk_version): super().__init__() + self._sdk_version_lookup: Final = sdk_version_lookup async def async_realtime( self, @@ -234,14 +270,13 @@ class BedrockRealtime(BaseAWSLLM): Various AWS authentication parameters """ try: - from aws_sdk_bedrock_runtime.client import ( - BedrockRuntimeClient, - InvokeModelWithBidirectionalStreamOperationInput, - ) - from aws_sdk_bedrock_runtime.config import Config - from smithy_aws_core.identity import StaticCredentialsResolver - except ImportError: - raise ImportError("Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime") + from aws_sdk_bedrock_runtime.client import AsyncBedrockRuntimeClient + from aws_sdk_bedrock_runtime.config import AsyncBedrockRuntimeConfig + from aws_sdk_bedrock_runtime.models import InvokeModelWithBidirectionalStreamOperationInput + from smithy_aws_core.identity import AWSCredentialsIdentity, StaticCredentialsResolver + from smithy_http.aio.crt import AWSCRTHTTPClient + except ImportError as e: + raise _sdk_import_error(self._sdk_version_lookup(), e) from e pending_session_update: Final = _pending_session_update(websocket.scope) @@ -285,22 +320,37 @@ class BedrockRealtime(BaseAWSLLM): ) frozen_credentials: Final = await run_aws_signing(credentials.get_frozen_credentials) - # Initialize Bedrock client with aws_sdk_bedrock_runtime - config: Final = Config( + credentials_identity: Final = AWSCredentialsIdentity( + access_key_id=frozen_credentials.access_key, + secret_access_key=frozen_credentials.secret_key, + session_token=frozen_credentials.token, + ) + config: Final = await AsyncBedrockRuntimeConfig.resolve( endpoint_uri=endpoint_uri, region=aws_region_name, - aws_access_key_id=frozen_credentials.access_key, - aws_secret_access_key=frozen_credentials.secret_key, - aws_session_token=frozen_credentials.token, - aws_credentials_identity_resolver=StaticCredentialsResolver(), + aws_credentials_identity_resolver=StaticCredentialsResolver(identity=credentials_identity), + transport=AWSCRTHTTPClient(), ) - bedrock_client: Final = BedrockRuntimeClient(config=config) + bedrock_client: Final = AsyncBedrockRuntimeClient(config=config) async def open_bidirectional_stream() -> BedrockBidirectionalStream: return await bedrock_client.invoke_model_with_bidirectional_stream( InvokeModelWithBidirectionalStreamOperationInput(model_id=model) ) + try: + await self._run_session(websocket, open_bidirectional_stream, model, logging_obj, pending_session_update) + finally: + await _close_bedrock_client(bedrock_client) + + async def _run_session( + self, + websocket: RealtimeClientWebSocket, + open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]], + model: str, + logging_obj: LiteLLMLogging, + pending_session_update: str | None, + ) -> None: transformation_config: Final = BedrockRealtimeConfig() bedrock_stream: Final = await open_bidirectional_stream() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c565b6ecc4b..96cdbcffd90 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7605,7 +7605,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7733,7 +7733,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7887,7 +7887,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, @@ -7956,7 +7956,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8856,7 +8856,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8955,7 +8955,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9054,7 +9054,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -10987,14 +10987,14 @@ "supports_vision": true }, "azure_ai/FW-Kimi-K3": { - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ "low", "high", @@ -23788,7 +23788,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -24114,7 +24114,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -45245,7 +45245,7 @@ "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -45482,6 +45482,7 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-09-29", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -45503,7 +45504,7 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://api.together.xyz/v1/models", + "source": "https://api.together.ai/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -45528,6 +45529,7 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, + "deprecation_date": "2026-09-29", "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -45552,7 +45554,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -45567,7 +45569,7 @@ "supports_vision": true }, "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", "max_input_tokens": 514, @@ -45680,7 +45682,7 @@ "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", @@ -50573,7 +50575,7 @@ "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, "output_cost_per_token": 1.7e-07, @@ -50584,7 +50586,7 @@ "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, "output_cost_per_token": 1.3e-07, @@ -50593,6 +50595,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "deprecation_date": "2026-03-04", "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, @@ -50603,6 +50606,7 @@ "mode": "chat" }, "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "deprecation_date": "2026-08-04", "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -50612,6 +50616,7 @@ "mode": "chat" }, "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { + "deprecation_date": "2026-08-25", "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -50622,6 +50627,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "deprecation_date": "2026-08-04", "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, @@ -50632,6 +50638,7 @@ "mode": "chat" }, "wandb/moonshotai/Kimi-K2-Instruct": { + "deprecation_date": "2026-03-04", "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -50656,6 +50663,7 @@ "supports_vision": true }, "wandb/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-08-25", "max_tokens": 197000, "max_input_tokens": 197000, "max_output_tokens": 197000, @@ -50670,7 +50678,7 @@ }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, "output_cost_per_token": 2.2e-07, @@ -50690,6 +50698,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "deprecation_date": "2026-03-04", "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, @@ -50700,6 +50709,7 @@ "mode": "chat" }, "wandb/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-03-04", "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -50719,6 +50729,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + "deprecation_date": "2026-04-21", "max_tokens": 64000, "max_input_tokens": 64000, "max_output_tokens": 64000, @@ -50728,6 +50739,7 @@ "mode": "chat" }, "wandb/microsoft/Phi-4-mini-instruct": { + "deprecation_date": "2026-08-04", "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -56692,7 +56704,8 @@ "supports_function_calling": true, "supports_vision": true, "supports_web_search": true, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "supports_response_schema": false }, "gemini-3.8-live-extended-thinking": { "input_cost_per_audio_token": 3e-06, @@ -56726,7 +56739,8 @@ "supports_vision": true, "supports_web_search": true, "gemini_audio_only_live": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -60970,10 +60984,11 @@ "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, "max_tokens": 1048576, - "max_input_tokens": 1048576, + "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 7e-08, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -60983,7 +60998,7 @@ "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 7e-08, @@ -60996,10 +61011,11 @@ "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, "max_tokens": 1048576, - "max_input_tokens": 1048576, + "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, "cache_read_input_token_cost": 2e-07, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61009,7 +61025,7 @@ "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, "litellm_provider": "wandb", @@ -61018,8 +61034,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/ibm-granite/granite-4.1-8b": { + "deprecation_date": "2026-10-05", "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, "litellm_provider": "wandb", @@ -61028,8 +61045,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, "litellm_provider": "wandb", @@ -61038,8 +61056,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, "litellm_provider": "wandb", @@ -61050,7 +61069,7 @@ "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, "cache_read_input_token_cost": 5e-08, @@ -61063,7 +61082,7 @@ "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61076,7 +61095,7 @@ "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61089,10 +61108,10 @@ "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 5e-08, + "max_input_tokens": 262000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 4e-08, "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61102,10 +61121,10 @@ "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, - "input_cost_per_token": 7.5e-07, - "output_cost_per_token": 2.75e-06, - "cache_read_input_token_cost": 1.5e-07, + "max_input_tokens": 262000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.15e-06, + "cache_read_input_token_cost": 1e-07, "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61113,8 +61132,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/OpenPipe/Qwen3-14B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 32768, - "max_input_tokens": 32768, + "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, "litellm_provider": "wandb", @@ -61125,7 +61145,7 @@ "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61138,7 +61158,7 @@ "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "wandb", @@ -61149,10 +61169,11 @@ "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, "cache_read_input_token_cost": 1.2e-07, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61160,9 +61181,10 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "deprecation_date": "2026-10-05", "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "wandb", @@ -61171,8 +61193,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "deprecation_date": "2026-10-05", "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "wandb", @@ -61187,6 +61210,7 @@ "input_cost_per_token": 1.31e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, + "max_input_tokens": 1049000, "supports_prompt_caching": true, "source": "https://wandb.ai/site/pricing/tokens/" }, @@ -61197,13 +61221,14 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, "cache_read_input_token_cost": 5e-08, + "max_input_tokens": 131000, "supports_prompt_caching": true, "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -62866,7 +62891,7 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -69158,5 +69183,19 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_vision": true + }, + "wandb/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://wandb.ai/site/pricing/tokens/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 74f38b3ca6d..216b9f6def6 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -38680,8 +38680,7 @@ "required": false, "schema": { "default": 10, - "maximum": 100, - "minimum": 1, + "minimum": 0, "title": "Count", "type": "integer" } @@ -39385,8 +39384,7 @@ "required": false, "schema": { "default": 10, - "maximum": 100, - "minimum": 1, + "minimum": 0, "title": "Count", "type": "integer" } diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d9529f7d575..0b1612559ab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -850,6 +850,7 @@ class LiteLLMRoutes(enum.Enum): "/team/member_add", "/team/member_delete", "/management/v1/teams/{team_id}/members/bulk_delete", + "/management/v1/teams/{team_id}/members/bulk_update", "/team/member_update", "/team/{team_id}/member/{user_id}/reset_spend", "/team/permissions_list", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 166a0500cee..0a6b618805d 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -31,6 +31,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( # team "/team/new", "/management/v1/teams/{team_id}/members/bulk_delete", + "/management/v1/teams/{team_id}/members/bulk_update", "/team/update", "/team/delete", "/team/block", @@ -767,6 +768,7 @@ class RouteChecks: "/user/bulk_update", "/team/new", "/management/v1/teams/{team_id}/members/bulk_delete", + "/management/v1/teams/{team_id}/members/bulk_update", "/team/update", "/team/delete", "/model/new", diff --git a/litellm/proxy/client/cli/__init__.py b/litellm/proxy/client/cli/__init__.py index 843a0095878..7634cabb3b3 100644 --- a/litellm/proxy/client/cli/__init__.py +++ b/litellm/proxy/client/cli/__init__.py @@ -1,5 +1,5 @@ """CLI package for LiteLLM Proxy Client.""" -from .main import cli +from .main import cli, litellm_proxy_cli -__all__ = ["cli"] +__all__ = ["cli", "litellm_proxy_cli"] diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index 4c6ab94191e..f9a9356d0d6 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -36,8 +36,8 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): resumable; safe to re-run after an interruption. Examples: - litellm-proxy encryption migrate --check # attestation scan, no writes - litellm-proxy encryption migrate # perform the migration + lite encryption migrate --check # attestation scan, no writes + lite encryption migrate # perform the migration """ client: Final = HTTPClient(ctx.obj["base_url"], ctx.obj["api_key"]) diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 05fb877d0f1..63e38c93221 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -168,5 +168,16 @@ cli.add_command(configure_group) cli.add_command(unconfigure_group) +LITELLM_PROXY_DEPRECATION_NOTICE: Final = ( + "The `litellm-proxy` command is deprecated and will be removed in a future release; " + "run `lite` instead, it takes the same commands and options." +) + + +def litellm_proxy_cli() -> None: + click.secho(LITELLM_PROXY_DEPRECATION_NOTICE, err=True, fg="yellow") + cli() + + if __name__ == "__main__": cli() diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index acb51e73daf..1299a4df243 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -26,7 +26,6 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import ( DB_RETRY_SAFE_ERROR_TYPES, LiteLLM_BudgetTableFull, - LiteLLM_EndUserTable, Litellm_EntityType, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -193,13 +192,6 @@ def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]: return (end_user_cache_key(row.user_id),) -def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float: - if not caps: - return 0.0 - effective_budget_id: Final[str | None] = row.budget_id or litellm.max_end_user_budget_id - return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None) - - def _budget_link_where( budget_ids: Sequence[str], extra: Mapping[str, object] = MappingProxyType({}), @@ -207,6 +199,19 @@ def _budget_link_where( return {"budget_id": {"in": list(budget_ids)}, **extra} +def _enduser_invalidation_where(budget_ids: Sequence[str]) -> dict[str, object]: + """Customers whose cached spend a committed reset of these tiers invalidated. + + Mirrors ``_queue_enduser_resets`` without its ``spend > 0`` filter, which + post-commit would match nobody. + """ + linked: Final = _budget_link_where(budget_ids) + default_budget_id: Final = litellm.max_end_user_budget_id + if default_budget_id is None or default_budget_id not in budget_ids: + return linked + return {"OR": [linked, {"budget_id": None}]} # mutable-ok: prisma where filter must be a dict + + def _queue_budget_linked_resets( writes: LinkedSpendResetWrites, cascade: "_BudgetCascade", @@ -265,16 +270,29 @@ class _BudgetCascade: budgets: tuple[LiteLLM_BudgetTableFull, ...] = () budget_ids: tuple[str, ...] = () budget_resets: tuple[tuple[str, datetime], ...] = () - endusers: tuple[_EndUserRow, ...] = () counter_resets: tuple[tuple[str, float], ...] = () cache_keys: tuple[str, ...] = () rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) +@dataclass(frozen=True, slots=True) +class _EndUserWalk: + """Where the customer walk stands. ``cursor`` is None once it is done, and + ``truncated`` says a failed page read cut it short of the tail.""" + + cursor: str | None = "" + invalidated: int = 0 + truncated: bool = False + + +_ENDUSER_WALK_DONE: Final = _EndUserWalk(cursor=None) + + @dataclass(frozen=True, slots=True) class _BudgetCascadeCommitted: cascade: _BudgetCascade advanced: int + endusers: _EndUserWalk @dataclass(frozen=True, slots=True) @@ -285,6 +303,8 @@ class _BudgetCascadeFailed: _EMPTY_CASCADE: Final = _BudgetCascade() +_InvalidatedCache = Literal["spend counter", "user_api_key_cache"] + @dataclass(frozen=True, slots=True) class _ChunkOutcome: @@ -416,10 +436,12 @@ _WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = ( ) -def _budget_cascade_event_metadata(cascade: _BudgetCascade) -> dict[str, object]: +def _budget_cascade_event_metadata( + cascade: _BudgetCascade, endusers: _EndUserWalk = _ENDUSER_WALK_DONE +) -> dict[str, object]: return { "num_budgets_found": len(cascade.budgets), - "num_endusers_found": len(cascade.endusers), + "num_endusers_found": endusers.invalidated, } @@ -593,6 +615,38 @@ class ResetBudgetJob: e, ) + @staticmethod + async def _invalidate_caches(counter_keys: Sequence[str], cache_keys: Sequence[str]) -> None: + """Batch twin of ``_invalidate_spend_counter`` and + ``_invalidate_user_api_key_cache_entry``, after the commit like both: + one round trip per chunk where a tier's dependents are unbounded.""" + await ResetBudgetJob._invalidate_cache("spend counter", counter_keys) + await ResetBudgetJob._invalidate_cache("user_api_key_cache", cache_keys) + + @staticmethod + async def _invalidate_cache(cache: _InvalidatedCache, keys: Sequence[str]) -> None: + """One cache's share of a batch, awaited separately so either failing + still leaves the other invalidated.""" + if not keys: + return + try: + from litellm.proxy.proxy_server import spend_counter_cache, user_api_key_cache + + match cache: + case "spend counter": + await spend_counter_cache.async_delete_cache_keys(keys) + case "user_api_key_cache": + await user_api_key_cache.async_delete_cache_keys(keys) + case _: + assert_never(cache) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to invalidate %d %s entries: %s. Budgets may be over-enforced until they expire.", + len(keys), + cache, + e, + ) + async def _fetch_linked_rows( self, table: SpendLinkedTable[_RowT], @@ -612,18 +666,57 @@ class ResetBudgetJob: verbose_proxy_logger.warning("Failed to fetch %s for counter invalidation: %s", log_subject, e) return () - async def _collect_endusers_to_reset(self, budget_ids: Sequence[str]) -> tuple[_EndUserRow, ...]: - linked: Final[Sequence[_EndUserRow] | None] = await self._with_db_retry( - lambda: self.prisma_client.get_data( - table_name="enduser", - query_type="find_all", - budget_id_list=list(budget_ids), - ), - reason="reset_budget_read_endusers_failure", + async def _invalidate_enduser_caches(self, budget_ids: Sequence[str]) -> _EndUserWalk: + """Drop the cached spend of every customer the committed tier reset zeroed. + + Paged like ``_reset_windows_for``, and capless for its reason too: the + customers on one tier are unbounded, and a cap cannot keep its position + across pod elections, so it would restart at the first customer forever. + """ + if not budget_ids: + return _ENDUSER_WALK_DONE + where: Final = _enduser_invalidation_where(budget_ids) + walk = _EndUserWalk() + while walk.cursor is not None: + walk = await self._invalidate_enduser_page(where=where, cursor=walk.cursor, reached=walk.invalidated) + return walk + + async def _invalidate_enduser_page(self, where: Mapping[str, object], cursor: str, reached: int) -> _EndUserWalk: + """Invalidate one page of customers and say where the walk goes next.""" + try: + rows: Final = await self._fetch_enduser_page(where=where, cursor=cursor) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to fetch end users for cache invalidation after %s customers (cursor %r): %s. " + "The customers past that page keep their cached spend until it expires.", + reached, + cursor, + e, + ) + return _EndUserWalk(cursor=None, invalidated=reached, truncated=True) + if not rows: + return _EndUserWalk(cursor=None, invalidated=reached) + await self._invalidate_caches( + counter_keys=tuple(_enduser_counter_key(row) for row in rows), + cache_keys=tuple(key for row in rows for key in _enduser_cache_keys(row)), + ) + walked: Final = reached + len(rows) + if len(rows) < RESET_BUDGET_JOB_BATCH_SIZE: + return _EndUserWalk(cursor=None, invalidated=walked) + return _EndUserWalk(cursor=rows[-1].user_id, invalidated=walked) + + async def _fetch_enduser_page(self, where: Mapping[str, object], cursor: str) -> tuple[_EndUserRow, ...]: + """One keyset page of customers, ordered by primary key so the cursor never repeats a row.""" + return tuple( + await self._with_db_retry( + lambda: EndUserRepository(self.prisma_client).table.find_many( + where={**where, "user_id": {"gt": cursor}}, # mutable-ok: prisma where filter must be a dict + order={"user_id": "asc"}, # mutable-ok: prisma order filter must be a dict + take=RESET_BUDGET_JOB_BATCH_SIZE, + ), + reason="reset_budget_read_endusers_failure", + ) ) - if litellm.max_end_user_budget_id is None or litellm.max_end_user_budget_id not in budget_ids: - return tuple(linked or ()) - return (*(linked or ()), *await self._get_endusers_with_no_budget_id()) async def _collect_budget_cascade(self, budgets_to_reset: Sequence[LiteLLM_BudgetTableFull]) -> _BudgetCascade: """Resolve every row the expiring budget tiers gate, before any write. @@ -670,7 +763,6 @@ class ResetBudgetJob: if _rollover_enabled() else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType ) - endusers: Final[tuple[_EndUserRow, ...]] = await self._collect_endusers_to_reset(budget_ids) return _BudgetCascade( budgets=tuple(budgets_to_reset), budget_ids=budget_ids, @@ -682,7 +774,6 @@ class ResetBudgetJob: for b in budgets_to_reset if b.budget_id is not None and b.budget_duration is not None ), - endusers=endusers, counter_resets=( *( (_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps)) @@ -695,7 +786,6 @@ class ResetBudgetJob: (_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in model_access_groups ), - *((_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) for row in endusers), ), rollover_caps=rollover_caps, cache_keys=( @@ -704,7 +794,6 @@ class ResetBudgetJob: *(key for row in orgs for key in _org_cache_keys(row)), *(key for row in tags for key in _tag_cache_keys(row)), *(key for row in model_access_groups for key in _model_access_group_cache_keys(row)), - *(key for row in endusers for key in _enduser_cache_keys(row)), ), ) @@ -736,10 +825,10 @@ class ResetBudgetJob: uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at) async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None: - for counter_key, _ in cascade.counter_resets: - await self._invalidate_spend_counter(counter_key) - for cache_key in cascade.cache_keys: - await self._invalidate_user_api_key_cache_entry(cache_key) + await self._invalidate_caches( + counter_keys=tuple(counter_key for counter_key, _ in cascade.counter_resets), + cache_keys=cascade.cache_keys, + ) async def _reset_expired_budget_cascade(self) -> _BudgetCascadeCommitted | _BudgetCascadeFailed: now: Final = datetime.now(timezone.utc) @@ -769,6 +858,7 @@ class ResetBudgetJob: (reset_at for _, reset_at in cascade.budget_resets), cutoff=datetime.now(timezone.utc), ), + endusers=await self._invalidate_enduser_caches(cascade.budget_ids), ) async def reset_budget_for_litellm_budget_table(self) -> None: @@ -788,7 +878,7 @@ class ResetBudgetJob: end_time: Final = time.time() match outcome: - case _BudgetCascadeCommitted(cascade=cascade, advanced=advanced): + case _BudgetCascadeCommitted() as committed: asyncio.create_task( self.proxy_logging_obj.service_logging_obj.async_service_success_hook( service=ServiceTypes.RESET_BUDGET_JOB, @@ -797,13 +887,14 @@ class ResetBudgetJob: start_time=start_time, end_time=end_time, event_metadata={ - **_budget_cascade_event_metadata(cascade), - "num_endusers_updated": len(cascade.endusers), + **_budget_cascade_event_metadata(committed.cascade, committed.endusers), + "num_endusers_updated": committed.endusers.invalidated, "num_endusers_failed": 0, + "enduser_invalidation_truncated": committed.endusers.truncated, }, ) ) - return _ChunkOutcome(fetched=len(cascade.budgets), advanced=advanced) + return _ChunkOutcome(fetched=len(committed.cascade.budgets), advanced=committed.advanced) case _BudgetCascadeFailed(cascade=cascade, error=error): verbose_proxy_logger.exception( "Failed to reset the budget table cascade (team member, enduser, org, tag and model access " @@ -827,27 +918,6 @@ class ResetBudgetJob: case _: assert_never(outcome) - async def _get_endusers_with_no_budget_id( - self, - ) -> list[LiteLLM_EndUserTable]: - """ - Fetch end users that have no explicit budget_id set (NULL) and have - accumulated spend > 0. These are implicitly-created end users that - rely on the default budget (litellm.max_end_user_budget_id) applied - in-memory during auth checks. - """ - table: Final = EndUserRepository(self.prisma_client).table - rows: Final = await self._with_db_retry( - lambda: table.find_many( - where={ - "budget_id": None, - "spend": {"gt": 0}, - }, - ), - reason="reset_budget_read_endusers_without_budget_id_failure", - ) - return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows] - async def _write_key_reset_updates(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None: """ Write per-row {spend, budget_reset_at} updates for keys. diff --git a/litellm/proxy/common_utils/timezone_utils.py b/litellm/proxy/common_utils/timezone_utils.py index a50daf40144..99e89210e43 100644 --- a/litellm/proxy/common_utils/timezone_utils.py +++ b/litellm/proxy/common_utils/timezone_utils.py @@ -78,3 +78,27 @@ def get_budget_reset_time(budget_duration: str) -> datetime: `BudgetResetSettings` by injection (creation/update endpoints, startup backfill). """ return compute_budget_reset_at(budget_duration, get_budget_reset_settings()) + + +def _is_persistable_budget_duration(budget_duration: str) -> bool: + from litellm.litellm_core_utils.duration_parser import duration_in_seconds + + try: + if duration_in_seconds(budget_duration) <= 0: + return False + get_budget_reset_time(budget_duration=budget_duration) + except (ValueError, OverflowError): + return False + return True + + +def budget_duration_error(budget_duration: str | None) -> str | None: + """Why `budget_duration` cannot be persisted, or None when it is usable. + + A non-positive duration resolves to a reset time of "now", which leaves the row + permanently due: the reset job re-reads it every tick and, once enough of them + exist, they fill each batch and starve every other tenant's reset. + """ + if budget_duration is None or _is_persistable_budget_duration(budget_duration): + return None + return f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'." diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 1c7a379897f..89ff113c6d3 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import re from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, TypeVar, cast, overload @@ -221,6 +222,24 @@ class UserApiKeyCache(DualCache): return await super().async_delete_cache(key) + async def async_delete_cache_keys(self, keys: Sequence[str]) -> None: + """Batch twin of ``async_delete_cache``, partitioned like + ``async_set_cache_pipeline``. + + Both partitions are cleared even when one raises, because a caller + batching these has already committed the rows they cache. + """ + key_object_keys: Final = tuple(key for key in keys if is_user_key_cache_key(key)) + other_keys: Final = tuple(key for key in keys if not is_user_key_cache_key(key)) + outcomes: Final = await asyncio.gather( + self.key_object_cache.async_delete_cache_keys(key_object_keys), + super().async_delete_cache_keys(other_keys), + return_exceptions=True, + ) + failed: Final = tuple(outcome for outcome in outcomes if isinstance(outcome, BaseException)) + if failed: + raise failed[0] + def flush_cache(self) -> None: super().flush_cache() self.key_object_cache.in_memory_cache.flush_cache() diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 5b90c0ff830..b9580ba3948 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -36,6 +36,10 @@ router: Final = APIRouter() IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams)) +IMAGE_ARRAY_FIELD: Final = "image[]" +MASK_ARRAY_FIELD: Final = "mask[]" +BRACKETED_FILE_FIELDS: Final = frozenset({IMAGE_ARRAY_FIELD, MASK_ARRAY_FIELD}) + async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO: """ @@ -244,9 +248,9 @@ async def image_edit_api( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), image: list[UploadFile] | None = File(None), - image_array: list[UploadFile] | None = File(None, alias="image[]"), + image_array: list[UploadFile] | None = File(None, alias=IMAGE_ARRAY_FIELD), mask: list[UploadFile] | None = File(None), - mask_array: list[UploadFile] | None = File(None, alias="mask[]"), + mask_array: list[UploadFile] | None = File(None, alias=MASK_ARRAY_FIELD), model: str | None = None, ): """ @@ -294,12 +298,14 @@ async def image_edit_api( ######################################################### # Read request body and convert UploadFiles to BytesIO ######################################################### - data: Final = dict( - coerce_numeric_form_fields( + data: Final = { + key: value + for key, value in coerce_numeric_form_fields( parsed_body=await _read_request_body(request=request), numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS, - ) - ) + ).items() + if key not in BRACKETED_FILE_FIELDS + } image_files: Final = await batch_to_bytesio(image) mask_files: Final = await batch_to_bytesio(mask) if image_files: diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 546078cce09..6f88b05a8ae 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -1,5 +1,6 @@ import math from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Optional, Union from fastapi import HTTPException, status @@ -33,23 +34,11 @@ def validate_budget_duration(budget_duration: str | None, status_code: int = 400 enough of them exist, they fill each batch and starve every other tenant's reset. """ - if budget_duration is None: - return + from litellm.proxy.common_utils.timezone_utils import budget_duration_error - from litellm.litellm_core_utils.duration_parser import duration_in_seconds - from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time - - try: - if duration_in_seconds(budget_duration) <= 0: - raise ValueError("budget_duration must be positive") - get_budget_reset_time(budget_duration=budget_duration) - except (ValueError, OverflowError): - raise HTTPException( - status_code=status_code, - detail={ - "error": f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'." - }, - ) + error: Final = budget_duration_error(budget_duration) + if error is not None: + raise HTTPException(status_code=status_code, detail={"error": error}) from litellm._logging import verbose_proxy_logger @@ -492,6 +481,35 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = ( ) +MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType( + { + "max_budget_in_team": "max_budget", + "tpm_limit": "tpm_limit", + "rpm_limit": "rpm_limit", + "budget_duration": "budget_duration", + "allowed_models": "allowed_models", + "temp_budget_increase": "temp_budget_increase", + "temp_budget_expiry": "temp_budget_expiry", + } +) + + +def _prisma_value(value: object) -> object: + return list(value) if isinstance(value, tuple) else value + + +def member_budget_patch(source: BaseModel) -> dict[str, Any]: + """Map the per-member limit fields a request actually set to their budget-table + columns (merge-patch: a sent value updates, an explicit null clears, an absent + field is left untouched).""" + provided: Final = source.model_dump(exclude_unset=True) + return { + column: _prisma_value(provided[request_field]) + for request_field, column in MEMBER_BUDGET_PATCH_FIELDS.items() + if request_field in provided + } + + def _is_set_budget_value(value: object) -> bool: if value is None: return False @@ -515,6 +533,7 @@ async def _upsert_budget_and_membership( user_api_key_dict: UserAPIKeyAuth, budget_patch: dict[str, Any], team_default_budget_id: str | None = None, + shared_budget_ids: frozenset[str] | None = None, ): """ Apply a merge-patch of per-member budget fields to a team membership. @@ -529,6 +548,10 @@ async def _upsert_budget_and_membership( (from team metadata.team_member_budget_id). When the membership still points at it, we clone-on-write so editing one member's budget does not mutate the shared default that every other member points at. + + ``shared_budget_ids`` extends that protection to any other row more than one + membership points at, which a caller patching several members at once has + already counted; a row listed there is cloned rather than written in place. """ if not budget_patch: return @@ -540,10 +563,8 @@ async def _upsert_budget_and_membership( get_budget_reset_time(budget_duration=duration) if duration is not None else None ) - is_shared_default: Final = ( - existing_budget_id is not None - and team_default_budget_id is not None - and existing_budget_id == team_default_budget_id + is_shared_default: Final = existing_budget_id is not None and ( + existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset()) ) async def _disconnect(): diff --git a/litellm/proxy/management_endpoints/management_v1/teams.py b/litellm/proxy/management_endpoints/management_v1/teams.py index ba384bfb028..eee6f486a4f 100644 --- a/litellm/proxy/management_endpoints/management_v1/teams.py +++ b/litellm/proxy/management_endpoints/management_v1/teams.py @@ -1,4 +1,4 @@ -"""`POST /management/v1/teams/{team_id}/members/bulk_delete`.""" +"""`POST /management/v1/teams/{team_id}/members/bulk_delete` and `.../members/bulk_update`.""" from typing import Annotated, Final @@ -9,12 +9,15 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_helpers.bulk_team_member_budgets import bulk_update_team_member_budgets from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped ) from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail from litellm.types.proxy.management_endpoints.team_endpoints import ( + BulkTeamMemberBudgetUpdateRequest, + BulkTeamMemberBudgetUpdateResponse, BulkTeamMemberDeleteRequest, BulkTeamMemberDeleteResponse, ) @@ -92,3 +95,80 @@ async def bulk_delete_team_members_action( detail="Failed to remove team members.", ) ) + + +@router.post( + "/teams/{team_id}/members/bulk_update", + tags=["team management"], # mutable-ok: FastAPI types `tags` as list[str], not Sequence + dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)), + response_model=BulkTeamMemberBudgetUpdateResponse, +) +@management_endpoint_wrapper +async def bulk_update_team_member_budgets_action( + team_id: str, + data: BulkTeamMemberBudgetUpdateRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> BulkTeamMemberBudgetUpdateResponse: + """ + Set per-member limits for up to 500 members of one team in one call. Same + authorization and member addressing as `/team/member_update`: proxy admins, the team's + admins, and admins of the team's organization, with each member named by exactly one of + `user_id` or `user_email`. Unknown body fields are a 422 and an unknown team is a 404. + + Each row is a merge patch of that member's limits: a field left out is untouched, a + field sent as null is cleared, and clearing the last limit drops the member back to the + team default. A budget row shared by several memberships, the team default included, is + copied for the member being patched rather than written in place, so one member's new + cap never lands on anybody else. + + `data` holds one result per requested member, in request order, carrying the limits in + force after the write. A row is `success: false` with an `error` when it names nobody on + the team or repeats an earlier row. Roles are not part of this route; `/team/member_update` + still owns them. + + Example curl: + ``` + curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_update' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{"members": [{"user_id": "user-1", "max_budget_in_team": 10}, {"user_email": "user-2@example.com", "max_budget_in_team": 10, "budget_duration": "30d"}]}' + ``` + """ + try: + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + if prisma_client is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}database-not-connected", + title="Database not connected", + status=503, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + ) + + results: Final = await bulk_update_team_member_budgets( + team_id=team_id, + data=data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + return BulkTeamMemberBudgetUpdateResponse(data=results) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.management_v1.teams.bulk_update_team_member_budgets_action(): " + "Exception occured - %s", + e, + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to update team member budgets.", + ) + ) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index ceb67e3eee8..34c1ad42435 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -264,6 +264,8 @@ scim_router: Final = APIRouter( dependencies=[Depends(_premium_user_check)], ) +SCIM_MAX_PAGE_SIZE: Final = 100 + # Helper functions for common operations async def _get_prisma_client_or_raise_exception(): @@ -1572,12 +1574,13 @@ def _parse_scim_eq_filter(scim_filter: str) -> tuple[str, str] | None: ) async def get_users( startIndex: int = Query(1, ge=1), - count: int = Query(10, ge=1, le=100), + count: int = Query(10, ge=0), filter: str | None = Query(None), ): """ Get a list of users according to SCIM v2 protocol """ + page_size: Final = min(count, SCIM_MAX_PAGE_SIZE) verbose_proxy_logger.debug( "SCIM GET USERS request: startIndex=%s count=%s filter=%s", startIndex, @@ -1607,7 +1610,7 @@ async def get_users( users: Final[Sequence[LiteLLM_UserTable]] = await _table(UserRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), - take=count, + take=page_size, order={"created_at": "desc"}, ) @@ -1623,7 +1626,7 @@ async def get_users( return SCIMListResponse( totalResults=total_count, startIndex=startIndex, - itemsPerPage=min(count, len(scim_users)), + itemsPerPage=len(scim_users), Resources=scim_users, ) @@ -2399,12 +2402,13 @@ class _TeamWhereConditions(TypedDict, total=False): ) async def get_groups( startIndex: int = Query(1, ge=1), - count: int = Query(10, ge=1, le=100), + count: int = Query(10, ge=0), filter: str | None = Query(None), ): """ Get a list of groups according to SCIM v2 protocol """ + page_size: Final = min(count, SCIM_MAX_PAGE_SIZE) verbose_proxy_logger.debug( "SCIM GET GROUPS request: startIndex=%s count=%s filter=%s", startIndex, @@ -2425,7 +2429,7 @@ async def get_groups( teams: Final = await _table(TeamRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), - take=count, + take=page_size, order={"created_at": "desc"}, ) @@ -2462,7 +2466,7 @@ async def get_groups( return SCIMListResponse( totalResults=total_count, startIndex=startIndex, - itemsPerPage=min(count, len(scim_groups)), + itemsPerPage=len(scim_groups), Resources=scim_groups, ) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0eb0f59e09c..fae3ef33c55 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -129,6 +129,7 @@ from litellm.proxy.management_endpoints.common_utils import ( _update_metadata_fields, _upsert_budget_and_membership, _user_has_admin_view, + member_budget_patch, validate_budget_duration, validate_team_model_max_budget, ) @@ -3686,29 +3687,6 @@ async def team_member_delete( return existing_team_row -_MEMBER_BUDGET_PATCH_FIELDS: Final = { - "max_budget_in_team": "max_budget", - "tpm_limit": "tpm_limit", - "rpm_limit": "rpm_limit", - "budget_duration": "budget_duration", - "allowed_models": "allowed_models", - "temp_budget_increase": "temp_budget_increase", - "temp_budget_expiry": "temp_budget_expiry", -} - - -def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> dict[str, object]: - """Map the budget fields the request actually set (merge-patch: a sent - value updates, an explicit null clears, an absent field is left untouched) - to their budget-table columns.""" - provided: Final = data.model_dump(exclude_unset=True) - return { - column: provided[request_field] - for request_field, column in _MEMBER_BUDGET_PATCH_FIELDS.items() - if request_field in provided - } - - @router.post( "/team/member_update", tags=["team management"], @@ -3814,7 +3792,7 @@ async def team_member_update( team_default_budget_id = raw_default_budget_id ### upsert new budget - budget_patch: Final = _build_member_budget_patch(data) + budget_patch: Final = member_budget_patch(data) async with prisma_client.tx() as tx: await _upsert_budget_and_membership( tx=tx, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 329443148a2..00cf357d89d 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -354,7 +354,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: status_code=400, detail=( "Your litellm CLI is out of date and uses a login flow this proxy no longer supports. " - "Upgrade it with `pip install -U 'litellm[proxy]'` and run `litellm-proxy login` again." + "Upgrade it with `pip install -U 'litellm[proxy]'` and run `lite login` again." ), ) if not _is_valid_cli_sso_login_id(login_id): @@ -375,7 +375,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: raise HTTPException( status_code=400, detail=( - "CLI login session not found or expired. Run `litellm-proxy login` again. " + "CLI login session not found or expired. Run `lite login` again. " "If this happens immediately after starting a login, the proxy is likely running multiple " "replicas without a shared cache; configure a Redis cache " "so every replica can see the login session." diff --git a/litellm/proxy/management_helpers/bulk_team_member_budgets.py b/litellm/proxy/management_helpers/bulk_team_member_budgets.py new file mode 100644 index 00000000000..b24712ab1b4 --- /dev/null +++ b/litellm/proxy/management_helpers/bulk_team_member_budgets.py @@ -0,0 +1,192 @@ +"""Batched per-member limit writes behind `POST /management/v1/teams/{team_id}/members/bulk_update`. + +Every read runs on the writer inside the batch transaction, so the write plan can never be +built from a lagging read replica. Any budget row that more than one membership points at, +the team's shared default included, is cloned before it is written, so raising one member's +cap never moves another member's. +""" + +from collections.abc import Sequence +from datetime import timedelta +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import invalidate_team_member_spend_state +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient +from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_update uses + _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # same check /team/member_update uses + _upsert_budget_and_membership, # pyright: ignore[reportPrivateUsage] # the single-member write, shared so the two surfaces cannot drift + member_budget_patch, +) +from litellm.proxy.management_helpers.bulk_user_deletion import ( + _duplicate_member_indexes, # pyright: ignore[reportPrivateUsage] # same duplicate rule as members/bulk_delete + _eq_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete + _forbidden, # pyright: ignore[reportPrivateUsage] # same problem shape as members/bulk_delete + _in_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete + _team_not_found, # pyright: ignore[reportPrivateUsage] # same problem shape as members/bulk_delete + _team_users_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete +) +from litellm.proxy.utils import PrismaClient +from litellm.repositories.team_repository import TeamRepository +from litellm.types.proxy.management_endpoints.team_endpoints import ( + BulkTeamMemberBudgetUpdateRequest, + TeamMemberBudgetPatch, + TeamMemberBudgetUpdateResult, +) + +if TYPE_CHECKING: + from prisma import Prisma + from prisma import models as prisma_models + + from litellm.repositories.prisma_protocols import TableActions + +_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) +_NO_METADATA: Final = MappingProxyType({}) +_WITH_BUDGET: Final = MappingProxyType({"litellm_budget_table": True}) + + +def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]": + return tx.litellm_teammembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _budget_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_BudgetTable]": + return tx.litellm_budgettable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +def _roster_user_id(member: TeamMemberBudgetPatch, roster: Sequence[Member]) -> str | None: + """The team member this row addresses, or None when it names nobody on the team.""" + if member.user_id is not None: + return member.user_id if any(m.user_id == member.user_id for m in roster) else None + return next((m.user_id for m in roster if m.user_email is not None and m.user_email == member.user_email), None) + + +def _team_default_budget_id(team: LiteLLM_TeamTable) -> str | None: + raw: Final = (team.metadata or _NO_METADATA).get("team_member_budget_id") + return raw if isinstance(raw, str) else None + + +async def _shared_budget_ids(tx: "Prisma", budget_ids: frozenset[str]) -> frozenset[str]: + """The rows in ``budget_ids`` more than one membership points at, counted across every + team so a row shared with another team is protected too.""" + if not budget_ids: + return frozenset() + rows: Final = await _membership_tx_db(tx).find_many(where=_in_filter("budget_id", budget_ids)) + return frozenset(budget_id for budget_id in budget_ids if sum(1 for row in rows if row.budget_id == budget_id) > 1) + + +def _result( + member: TeamMemberBudgetPatch, + user_id: str | None, + error: str | None, + budget_of: "MappingProxyType[str, prisma_models.LiteLLM_BudgetTable | None]", + team_default_max_budget: float | None, +) -> TeamMemberBudgetUpdateResult: + if error is not None or user_id is None: + return TeamMemberBudgetUpdateResult( + user_id=member.user_id, + user_email=member.user_email, + success=False, + error=error or "User not found in team", + ) + budget: Final = budget_of.get(user_id) + own_max_budget: Final = budget.max_budget if budget is not None else None + inherits: Final = own_max_budget is None and team_default_max_budget is not None and team_default_max_budget > 0 + return TeamMemberBudgetUpdateResult( + user_id=user_id, + user_email=member.user_email, + success=True, + budget_id=budget.budget_id if budget is not None else None, + max_budget=team_default_max_budget if inherits else own_max_budget, + max_budget_source=("team_default" if inherits else "member" if own_max_budget is not None else None), + tpm_limit=budget.tpm_limit if budget is not None else None, + rpm_limit=budget.rpm_limit if budget is not None else None, + budget_duration=budget.budget_duration if budget is not None else None, + allowed_models=tuple(budget.allowed_models) if budget is not None else None, + ) + + +async def bulk_update_team_member_budgets( + team_id: str, + data: BulkTeamMemberBudgetUpdateRequest, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> tuple[TeamMemberBudgetUpdateResult, ...]: + """Apply one merge patch of per-member limits per requested member, in one transaction.""" + team: Final = await TeamRepository(WriterPinnedClient(prisma_client.db)).find_by_id(team_id) + if team is None: + raise _team_not_found(team_id) + + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team) + and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team) + ): + raise _forbidden( + "Call not allowed. User not proxy admin OR team admin OR org admin for this team. " + f"route='/management/v1/teams/{team_id}/members/bulk_update'" + ) + + roster: Final = team.members_with_roles or () + named: Final = tuple(_roster_user_id(member, roster) for member in data.members) + duplicates: Final = _duplicate_member_indexes(data.members) | frozenset( + index for index, user_id in enumerate(named) if user_id is not None and user_id in named[:index] + ) + applied: Final = tuple( + (index, user_id) for index, user_id in enumerate(named) if user_id is not None and index not in duplicates + ) + if not applied: + return tuple( + _result( + member, None, "Duplicate member in request" if index in duplicates else None, MappingProxyType({}), None + ) + for index, member in enumerate(data.members) + ) + + user_ids: Final = sorted(user_id for _, user_id in applied) + default_budget_id: Final = _team_default_budget_id(team) + team_members_filter: Final = _team_users_filter(team_id, user_ids) + + async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: + memberships: Final = await _membership_tx_db(tx).find_many(where=team_members_filter) + budget_id_of: Final = MappingProxyType({m.user_id: m.budget_id for m in memberships}) + shared: Final = await _shared_budget_ids( + tx, frozenset(budget_id for budget_id in budget_id_of.values() if budget_id is not None) + ) + for index, user_id in applied: + await _upsert_budget_and_membership( + tx=tx, + team_id=team_id, + user_id=user_id, + existing_budget_id=budget_id_of.get(user_id), + user_api_key_dict=user_api_key_dict, + budget_patch=member_budget_patch(data.members[index]), + team_default_budget_id=default_budget_id, + shared_budget_ids=shared, + ) + written: Final = await _membership_tx_db(tx).find_many(where=team_members_filter, include=_WITH_BUDGET) + team_default: Final = ( + await _budget_tx_db(tx).find_unique(where=_eq_filter("budget_id", default_budget_id)) + if default_budget_id is not None + else None + ) + + for user_id in user_ids: + await invalidate_team_member_spend_state( + user_id=user_id, team_id=team_id, user_api_key_cache=user_api_key_cache + ) + + budget_of: Final = MappingProxyType({m.user_id: m.litellm_budget_table for m in written}) + return tuple( + _result( + member, + named[index], + "Duplicate member in request" if index in duplicates else None, + budget_of, + team_default.max_budget if team_default is not None else None, + ) + for index, member in enumerate(data.members) + ) diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 5f5be81ee4b..4524c47ec38 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,6 +1,6 @@ from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from litellm.proxy._types import ( KeyManagementRoutes, @@ -10,12 +10,15 @@ from litellm.proxy._types import ( Member, MemberDeleteRequest, ) +from litellm.proxy.common_utils.timezone_utils import budget_duration_error from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse TeamIdSearchMatch = Literal["exact", "prefix"] MAX_BULK_TEAM_MEMBER_DELETES: Final = 500 +MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES: Final = 500 + class GetTeamMemberPermissionsRequest(BaseModel): """Request to get the team member permissions for a team""" @@ -123,7 +126,7 @@ class BulkTeamMemberAddResponse(BaseModel): class TeamMemberRef(MemberDeleteRequest): - """One member to remove, named by exactly one of `user_id` or `user_email`.""" + """One member, named by exactly one of `user_id` or `user_email`.""" model_config = ConfigDict(extra="forbid") @@ -155,6 +158,55 @@ class BulkTeamMemberDeleteResponse(ResourceResponse[tuple[TeamMemberDeleteResult """`{data: [...]}` with one `TeamMemberDeleteResult` per requested member, in request order.""" +class TeamMemberBudgetPatch(TeamMemberRef): + """One member's per-member limits, merge-patch style: a field left out of the row is + untouched, a field sent as null is cleared, and clearing the last limit drops the + member back to the team default.""" + + max_budget_in_team: float | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: tuple[str, ...] | None = None + + @field_validator("budget_duration") + @classmethod + def persistable_budget_duration(cls, value: str | None) -> str | None: + error: Final = budget_duration_error(value) + if error is not None: + raise ValueError(error) + return value + + +class BulkTeamMemberBudgetUpdateRequest(BaseModel): + """Body of `POST /management/v1/teams/{team_id}/members/bulk_update`.""" + + model_config = ConfigDict(extra="forbid") + + members: tuple[TeamMemberBudgetPatch, ...] = Field(min_length=1, max_length=MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES) + + +class TeamMemberBudgetUpdateResult(BaseModel): + """Outcome for one requested member, in request order, carrying the limits in force + after the write rather than the ones that were asked for.""" + + user_id: str | None = None + user_email: str | None = None + success: bool + error: str | None = None + budget_id: str | None = None + max_budget: float | None = None + max_budget_source: Literal["member", "team_default"] | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: tuple[str, ...] | None = None + + +class BulkTeamMemberBudgetUpdateResponse(ResourceResponse[tuple[TeamMemberBudgetUpdateResult, ...]]): + """`{data: [...]}` with one `TeamMemberBudgetUpdateResult` per requested member, in request order.""" + + class TeamMemberInfoResponse(LiteLLM_TeamMembership): """Response for GET /team/{team_id}/members/me ā€” caller's own membership row.""" diff --git a/litellm/utils.py b/litellm/utils.py index 073aa2e8bb5..2c9200fbad7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2916,6 +2916,15 @@ def supports_none_reasoning_effort(model: str, custom_llm_provider: str | None = return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_none_reasoning_effort") +def supports_mid_conversation_system(model: str, custom_llm_provider: str | None = None) -> bool: + """ + Check if the given model accepts a system role message after the leading system block and return a boolean value. + """ + return _supports_factory( + model=model, custom_llm_provider=custom_llm_provider, key="supports_mid_conversation_system" + ) + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c565b6ecc4b..96cdbcffd90 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7605,7 +7605,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7733,7 +7733,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7887,7 +7887,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, @@ -7956,7 +7956,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8856,7 +8856,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8955,7 +8955,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -9054,7 +9054,7 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -10987,14 +10987,14 @@ "supports_vision": true }, "azure_ai/FW-Kimi-K3": { - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ "low", "high", @@ -23788,7 +23788,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -24114,7 +24114,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": false }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -45245,7 +45245,7 @@ "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -45482,6 +45482,7 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-09-29", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -45503,7 +45504,7 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://api.together.xyz/v1/models", + "source": "https://api.together.ai/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -45528,6 +45529,7 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, + "deprecation_date": "2026-09-29", "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -45552,7 +45554,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -45567,7 +45569,7 @@ "supports_vision": true }, "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", "max_input_tokens": 514, @@ -45680,7 +45682,7 @@ "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-15", + "deprecation_date": "2026-09-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", @@ -50573,7 +50575,7 @@ "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, "output_cost_per_token": 1.7e-07, @@ -50584,7 +50586,7 @@ "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, "output_cost_per_token": 1.3e-07, @@ -50593,6 +50595,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "deprecation_date": "2026-03-04", "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, @@ -50603,6 +50606,7 @@ "mode": "chat" }, "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "deprecation_date": "2026-08-04", "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -50612,6 +50616,7 @@ "mode": "chat" }, "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { + "deprecation_date": "2026-08-25", "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -50622,6 +50627,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "deprecation_date": "2026-08-04", "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, @@ -50632,6 +50638,7 @@ "mode": "chat" }, "wandb/moonshotai/Kimi-K2-Instruct": { + "deprecation_date": "2026-03-04", "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -50656,6 +50663,7 @@ "supports_vision": true }, "wandb/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-08-25", "max_tokens": 197000, "max_input_tokens": 197000, "max_output_tokens": 197000, @@ -50670,7 +50678,7 @@ }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, "output_cost_per_token": 2.2e-07, @@ -50690,6 +50698,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "deprecation_date": "2026-03-04", "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, @@ -50700,6 +50709,7 @@ "mode": "chat" }, "wandb/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-03-04", "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -50719,6 +50729,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + "deprecation_date": "2026-04-21", "max_tokens": 64000, "max_input_tokens": 64000, "max_output_tokens": 64000, @@ -50728,6 +50739,7 @@ "mode": "chat" }, "wandb/microsoft/Phi-4-mini-instruct": { + "deprecation_date": "2026-08-04", "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -56692,7 +56704,8 @@ "supports_function_calling": true, "supports_vision": true, "supports_web_search": true, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "supports_response_schema": false }, "gemini-3.8-live-extended-thinking": { "input_cost_per_audio_token": 3e-06, @@ -56726,7 +56739,8 @@ "supports_vision": true, "supports_web_search": true, "gemini_audio_only_live": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -60970,10 +60984,11 @@ "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, "max_tokens": 1048576, - "max_input_tokens": 1048576, + "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 7e-08, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -60983,7 +60998,7 @@ "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 7e-08, @@ -60996,10 +61011,11 @@ "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, "max_tokens": 1048576, - "max_input_tokens": 1048576, + "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, "cache_read_input_token_cost": 2e-07, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61009,7 +61025,7 @@ "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, "litellm_provider": "wandb", @@ -61018,8 +61034,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/ibm-granite/granite-4.1-8b": { + "deprecation_date": "2026-10-05", "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, "litellm_provider": "wandb", @@ -61028,8 +61045,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 131072, - "max_input_tokens": 131072, + "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, "litellm_provider": "wandb", @@ -61038,8 +61056,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, "litellm_provider": "wandb", @@ -61050,7 +61069,7 @@ "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, "cache_read_input_token_cost": 5e-08, @@ -61063,7 +61082,7 @@ "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61076,7 +61095,7 @@ "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61089,10 +61108,10 @@ "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 5e-08, + "max_input_tokens": 262000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 4e-08, "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61102,10 +61121,10 @@ "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, - "input_cost_per_token": 7.5e-07, - "output_cost_per_token": 2.75e-06, - "cache_read_input_token_cost": 1.5e-07, + "max_input_tokens": 262000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.15e-06, + "cache_read_input_token_cost": 1e-07, "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61113,8 +61132,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/OpenPipe/Qwen3-14B-Instruct": { + "deprecation_date": "2026-10-05", "max_tokens": 32768, - "max_input_tokens": 32768, + "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, "litellm_provider": "wandb", @@ -61125,7 +61145,7 @@ "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 1.5e-07, @@ -61138,7 +61158,7 @@ "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "wandb", @@ -61149,10 +61169,11 @@ "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, "cache_read_input_token_cost": 1.2e-07, + "deprecation_date": "2026-10-05", "supports_prompt_caching": true, "litellm_provider": "wandb", "mode": "chat", @@ -61160,9 +61181,10 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "deprecation_date": "2026-10-05", "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, "litellm_provider": "wandb", @@ -61171,8 +61193,9 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "deprecation_date": "2026-10-05", "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "wandb", @@ -61187,6 +61210,7 @@ "input_cost_per_token": 1.31e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, + "max_input_tokens": 1049000, "supports_prompt_caching": true, "source": "https://wandb.ai/site/pricing/tokens/" }, @@ -61197,13 +61221,14 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, "cache_read_input_token_cost": 5e-08, + "max_input_tokens": 131000, "supports_prompt_caching": true, "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_tokens": 262144, - "max_input_tokens": 262144, + "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -62866,7 +62891,7 @@ "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -69158,5 +69183,19 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_vision": true + }, + "wandb/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://wandb.ai/site/pricing/tokens/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/pyproject.toml b/pyproject.toml index 93ff55c4069..dfe84a28d52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.98", + "litellm-proxy-extras==0.4.99", "litellm-enterprise==0.1.68", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", @@ -143,8 +143,9 @@ bedrock-realtime = [ # InvokeModelWithBidirectionalStream API, which boto3 cannot do. This # experimental AWS SDK (with its smithy-* deps, pulled transitively) # provides the bidirectional stream; imported lazily in the realtime - # handler so litellm core stays usable without it. - "aws-sdk-bedrock-runtime>=0.7.0,<0.8.0; python_version >= '3.12'", + # handler so litellm core stays usable without it. The awscrt extra is + # required: the SDK's default aiohttp transport has no duplex streaming. + "aws-sdk-bedrock-runtime[awscrt]>=0.10.0,<0.12.0; python_version >= '3.12'", ] proxy-runtime = [ # Historically bundled in the proxy Docker images via requirements.txt. @@ -173,7 +174,7 @@ proxy-runtime = [ [project.scripts] litellm = "litellm:run_server" lite = "litellm.proxy.client.cli:cli" -litellm-proxy = "litellm.proxy.client.cli:cli" +litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index 8fd1efff0da..97522e5728c 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -10,9 +10,10 @@ from hashlib import sha256 from typing import Final, TypeVar import httpx -from integration._support.database import read_rows from pydantic import JsonValue, TypeAdapter +from tests.integration._support.database import read_rows + JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) T = TypeVar("T") diff --git a/tests/integration/_support/generation.py b/tests/integration/_support/generation.py index afb3ec2e768..50c1a6f2ad4 100644 --- a/tests/integration/_support/generation.py +++ b/tests/integration/_support/generation.py @@ -6,7 +6,7 @@ from contextlib import contextmanager import httpx from hypothesis import Phase, settings -from integration._support.client import Gateway +from tests.integration._support.client import Gateway LIFECYCLE_SETTINGS: Final = settings( max_examples=20, diff --git a/tests/integration/authorization/test_warmed_policy.py b/tests/integration/authorization/test_warmed_policy.py index 8dbf364f69f..a9bee196ddd 100644 --- a/tests/integration/authorization/test_warmed_policy.py +++ b/tests/integration/authorization/test_warmed_policy.py @@ -10,9 +10,9 @@ from pydantic import JsonValue from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test -from integration._support.client import Gateway, eventually, object_value -from integration._support.database import read_rows -from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests +from tests.integration._support.client import Gateway, eventually, object_value +from tests.integration._support.database import read_rows +from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests def assert_serving(gateway: Gateway, model: str, key: str, status: int, error_type: str = "auth_error") -> None: diff --git a/tests/integration/configuration/test_effective_settings.py b/tests/integration/configuration/test_effective_settings.py index 7fa440d1d8d..8e164acbe03 100644 --- a/tests/integration/configuration/test_effective_settings.py +++ b/tests/integration/configuration/test_effective_settings.py @@ -4,8 +4,8 @@ from typing import Final import httpx import pytest -from integration._support.client import Gateway, object_value, string_value -from integration._support.database import read_rows +from tests.integration._support.client import Gateway, object_value, string_value +from tests.integration._support.database import read_rows def model_identity(gateway: Gateway, alias: str) -> str: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index adb9fcd57f3..342952d44d4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -12,9 +12,9 @@ import pytest import httpx from redis import Redis -from integration._support.client import Gateway, eventually, gateway_from_environment -from integration._support.manifest import OWNED_DIRECTORIES, contracts -from integration._support.generation import LIFECYCLE_SETTINGS +from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.manifest import OWNED_DIRECTORIES, contracts +from tests.integration._support.generation import LIFECYCLE_SETTINGS COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() diff --git a/tests/integration/management/test_key_updates.py b/tests/integration/management/test_key_updates.py index 6f2e850b17a..b460190f0ba 100644 --- a/tests/integration/management/test_key_updates.py +++ b/tests/integration/management/test_key_updates.py @@ -3,8 +3,8 @@ from hashlib import sha256 import pytest -from integration._support.client import Gateway, object_value -from integration._support.database import read_rows +from tests.integration._support.client import Gateway, object_value +from tests.integration._support.database import read_rows @pytest.mark.covers("mgmt.key.update.preserves_independent_fields") diff --git a/tests/integration/management/test_partial_update_sequences.py b/tests/integration/management/test_partial_update_sequences.py index 3d1ef1374e1..d79c145a685 100644 --- a/tests/integration/management/test_partial_update_sequences.py +++ b/tests/integration/management/test_partial_update_sequences.py @@ -5,11 +5,12 @@ from typing import Final import pytest from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test -from integration._support.client import Gateway, object_value -from integration._support.database import read_rows -from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests from pydantic import JsonValue +from tests.integration._support.client import Gateway, object_value +from tests.integration._support.database import read_rows +from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests + def _key_rows(digest: str) -> list[dict[str, JsonValue]]: return read_rows( diff --git a/tests/integration/pricing/test_configured_prices.py b/tests/integration/pricing/test_configured_prices.py index b1df012e870..655d74c1402 100644 --- a/tests/integration/pricing/test_configured_prices.py +++ b/tests/integration/pricing/test_configured_prices.py @@ -6,8 +6,8 @@ import uuid import pytest import yaml -from integration._support.client import Gateway, eventually, object_value, string_value -from integration._support.database import read_rows +from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows @pytest.mark.covers("quota_management.spend_tracking.custom_price.matches_input_rates") diff --git a/tests/integration/providers/test_request_boundary.py b/tests/integration/providers/test_request_boundary.py index aad10843642..33663cd4c59 100644 --- a/tests/integration/providers/test_request_boundary.py +++ b/tests/integration/providers/test_request_boundary.py @@ -3,7 +3,7 @@ from typing import Final import httpx import pytest -from integration._support.client import Gateway, JSON_OBJECT, object_value +from tests.integration._support.client import Gateway, JSON_OBJECT, object_value @pytest.mark.covers("other.provider_wire.internal_parameters_filtered") diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index fe3c38a771f..32bcee7cb2a 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -102,21 +102,24 @@ def _wire_batcher_for_test(prisma_client, fail_commit=False): return batch_calls -def _wire_cascade_reads_for_test(prisma_client): +def _wire_cascade_reads_for_test(prisma_client, endusers=()): """ The budget tier's cascade reads the rows it is about to zero, so their spend counters can be invalidated after the commit. Give each of those tables an awaitable find_many so the reads resolve instead of falling into the job's warn-and-continue path. + + End users are read by the post-commit invalidation walk rather than by + ``get_data``, so callers that care about customers pass them here. """ for table in ( "litellm_teammembership", "litellm_verificationtoken", "litellm_organizationtable", "litellm_tagtable", - "litellm_endusertable", ): getattr(prisma_client.db, table).find_many = AsyncMock(return_value=[]) + prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=list(endusers)) @pytest.mark.asyncio @@ -556,7 +559,7 @@ async def test_reset_budget_continues_other_categories_on_failure(): **{u["user_id"]: u["spend"] for u in [user2]}, **{t["team_id"]: t["spend"] for t in [team1, team2]}, } - _wire_cascade_reads_for_test(prisma_client) + _wire_cascade_reads_for_test(prisma_client, endusers=[enduser1]) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -607,7 +610,10 @@ async def test_reset_budget_continues_other_categories_on_failure(): called_tables = { call.kwargs.get("table_name") for call in prisma_client.get_data.await_args_list } - assert called_tables == {"key", "user", "team", "budget", "enduser"} + assert called_tables == {"key", "user", "team", "budget"} + # Customers are not part of that set: the cascade zeroes them by budget link + # and reads them only afterwards, to invalidate their cached spend. + prisma_client.db.litellm_endusertable.find_many.assert_awaited() # Every category writes through the batch path now, so update_data is unused. prisma_client.update_data.assert_not_awaited() @@ -1029,7 +1035,7 @@ async def test_service_logger_endusers_success(): prisma_client.get_data = AsyncMock(side_effect=fake_get_data) prisma_client.update_data = AsyncMock() batch_calls = _wire_batcher_for_test(prisma_client) - _wire_cascade_reads_for_test(prisma_client) + _wire_cascade_reads_for_test(prisma_client, endusers=endusers) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -1094,7 +1100,7 @@ async def test_service_logger_endusers_failure(): prisma_client.get_data = AsyncMock(side_effect=fake_get_data) prisma_client.update_data = AsyncMock() _wire_batcher_for_test(prisma_client, fail_commit=True) - _wire_cascade_reads_for_test(prisma_client) + _wire_cascade_reads_for_test(prisma_client, endusers=endusers) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -1121,7 +1127,9 @@ async def test_service_logger_endusers_failure(): ) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args event_metadata = kwargs.get("event_metadata", {}) assert event_metadata.get("num_budgets_found") == len(budgets) - assert event_metadata.get("num_endusers_found") == len(endusers) + # Customers are read by the post-commit invalidation walk, which a failed + # commit never reaches, so a failure reports none touched. + assert event_metadata.get("num_endusers_found") == 0 assert "endusers_found" not in event_metadata assert "budgets_found" not in event_metadata proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called() diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index 44542558002..ca5058818e4 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -367,7 +367,7 @@ async def obtain_cli_sso_token_via_poll_flow( models: list[str], ) -> str: """ - Obtain a CLI SSO JWT through the same HTTP flow as `litellm-proxy login`: + Obtain a CLI SSO JWT through the same HTTP flow as `lite login`: /sso/cli/start -> (SSO callback) -> /sso/cli/complete -> /sso/cli/poll. When the proxy SSO session cache is not shared with the test runner (otel CI @@ -551,7 +551,7 @@ async def test_team_budget_enforcement(): @pytest.mark.asyncio async def test_team_budget_enforcement_cli_sso_token(): """ - Team budget enforcement for CLI SSO session tokens (litellm-proxy login JWT). + Team budget enforcement for CLI SSO session tokens (lite login JWT). 1. Create team with a tiny max_budget and a user on that team 2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint) diff --git a/tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py b/tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py index ed21734c5fc..e0f99835b44 100644 --- a/tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py +++ b/tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py @@ -20,7 +20,9 @@ import pytest IMAGE: Final = os.getenv("LITELLM_IMAGE") NON_ROOT_UID: Final = "12345:0" -IMPORT_PROBE: Final = "import aws_sdk_bedrock_runtime, smithy_aws_core; print('bedrock-realtime ok')" +IMPORT_PROBE: Final = ( + "import aws_sdk_bedrock_runtime, smithy_aws_core, smithy_http.aio.crt; print('bedrock-realtime ok')" +) pytestmark = [ pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"), @@ -52,7 +54,7 @@ def test_image_imports_bedrock_realtime_sdk(): ) assert probe.returncode == 0 and "bedrock-realtime ok" in probe.stdout, ( - f"{IMAGE} cannot import aws_sdk_bedrock_runtime as uid {NON_ROOT_UID}, so Bedrock Nova Sonic " - "/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'. Is `--extra bedrock-realtime` " + f"{IMAGE} cannot import aws_sdk_bedrock_runtime with its awscrt transport as uid {NON_ROOT_UID}, so " + "Bedrock Nova Sonic /v1/realtime sessions fail at SDK import. Is `--extra bedrock-realtime` " f"passed to every `uv sync` in its Dockerfile?\nstdout:\n{probe.stdout}\nstderr:\n{probe.stderr}" ) diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 95395878c25..5f59de9cca5 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.constants import DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache, _redis_circuit_breaker_guard, _redis_circuit_breaker_guard_sync @@ -759,3 +760,34 @@ async def test_redis_timeouts_falling_back_to_memory_log_once_per_interval(caplo " (199 more Redis timeouts since the previous Redis timeout line were logged at DEBUG)", ) ] + + +@pytest.mark.asyncio +async def test_async_delete_cache_keys_drops_memory_and_chunks_redis(): + """Batch delete clears both layers, and chunks Redis so one caller's large + key list cannot become a single oversized DELETE command.""" + redis_cache = MagicMock(spec=RedisCache) + redis_cache.delete_cache_keys = AsyncMock() + dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + keys = [f"key-{i}" for i in range(DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE + 7)] + for key in keys: + dual_cache.in_memory_cache.set_cache(key=key, value=1) + + await dual_cache.async_delete_cache_keys(keys) + + assert all(dual_cache.in_memory_cache.get_cache(key=key) is None for key in keys) + sent = [call.args[0] for call in redis_cache.delete_cache_keys.await_args_list] + assert [len(chunk) for chunk in sent] == [DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE, 7] + assert [key for chunk in sent for key in chunk] == keys + + +@pytest.mark.asyncio +async def test_async_delete_cache_keys_on_empty_list_touches_no_backend(): + """An empty page must not reach Redis: DELETE with no arguments is an error.""" + redis_cache = MagicMock(spec=RedisCache) + redis_cache.delete_cache_keys = AsyncMock() + dual_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + + await dual_cache.async_delete_cache_keys([]) + + redis_cache.delete_cache_keys.assert_not_awaited() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 03b9840b1c3..e6782b70d3e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -23,6 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im create_tool_name_mapping, truncate_tool_name, ) +from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import ( + CONVERTED_SYSTEM_NOTE, +) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.anthropic import ( AnthopicMessagesAssistantMessageParam, @@ -563,10 +566,19 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): @pytest.mark.parametrize( ("system_content", "expected_content"), [ - ("Use the corrected result.", "Use the corrected result."), + ( + "Use the corrected result.", + [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Use the corrected result."}, + ], + ), ( [{"type": "text", "text": "Use the corrected result."}], - [{"type": "text", "text": "Use the corrected result."}], + [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Use the corrected result."}, + ], ), ( [ @@ -576,7 +588,11 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): }, {"type": "text", "text": "Use the corrected result."}, ], - [{"type": "text", "text": "Use the corrected result."}], + [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "Use the corrected result."}, + ], ), ( [ @@ -584,13 +600,14 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): {"type": "text", "text": "Second correction."}, ], [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, {"type": "text", "text": "First correction."}, {"type": "text", "text": "Second correction."}, ], ), ], ) -def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correction( +def test_translate_anthropic_messages_to_openai_converts_midturn_system_correction( system_content: object, expected_content: object, ): @@ -646,7 +663,7 @@ def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correct "tool_call_id": "toolu_01234", "content": "Rainy, 55Ā°F", }, - {"role": "system", "content": expected_content}, + {"role": "user", "content": expected_content}, {"role": "user", "content": "Continue."}, ] @@ -752,8 +769,8 @@ def test_translate_anthropic_messages_to_openai_drops_empty_midturn_system( def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): """ Request level: the trusted top-level prompt is hoisted to index 0 exactly once and the - in-sequence correction keeps its own position and `role: "system"` -- no duplication of - either, and no reordering of the surrounding turns. + in-sequence correction keeps its own position as a user turn prefixed with the operator + note -- no duplication of either, and no reordering of the surrounding turns. """ openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request={ @@ -773,11 +790,140 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): {"role": "system", "content": "Trusted top-level prompt."}, {"role": "user", "content": "First question."}, {"role": "assistant", "content": "First answer.", "thinking_blocks": None}, - {"role": "system", "content": "Use the corrected result."}, + { + "role": "user", + "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Use the corrected result."}, + ], + }, {"role": "user", "content": "Continue."}, ] +_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST: Final = { + "max_tokens": 128, + "system": [{"type": "text", "text": "You are Claude Code."}], + "messages": [ + {"role": "user", "content": "say hi"}, + { + "role": "system", + "content": [{"type": "text", "text": "Keep answers to one sentence."}], + }, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "say bye"}, + ], +} + + +@pytest.mark.parametrize("custom_llm_provider", [None, "hosted_vllm"]) +def test_translate_anthropic_to_openai_converts_claude_code_midturn_system_turn(custom_llm_provider: str | None): + """ + Claude Code appends a system-role harness reminder after the user turn. On a chat-completions + target that does not declare ``supports_mid_conversation_system`` (a self-hosted model the cost + map knows nothing about) the outbound request must have exactly one system message, at index 0, + and the converted turn must carry the operator note first. + """ + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST}, + custom_llm_provider=custom_llm_provider, + ) + + roles = [m["role"] for m in openai_request["messages"]] + assert roles == ["system", "user", "user", "assistant", "user"] + converted = openai_request["messages"][2] + assert converted["content"][0]["text"] == CONVERTED_SYSTEM_NOTE + assert converted["content"][1]["text"] == "Keep answers to one sentence." + + +def test_translate_anthropic_to_openai_keeps_midturn_system_when_target_declares_support(monkeypatch): + """ + A chat-completions target flagged ``supports_mid_conversation_system`` in the cost map accepts + the role anywhere, so the harness reminder is forwarded in place with its role and content + untouched, the same rule the native Anthropic Messages path applies. + """ + model: Final = "system-role-anywhere-chat-model" + monkeypatch.setitem( + litellm.model_cost, + model, + {"litellm_provider": "openai", "mode": "chat", "supports_mid_conversation_system": True}, + ) + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={"model": model, **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST}, + custom_llm_provider="openai", + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": [{"type": "text", "text": "You are Claude Code."}]}, + {"role": "user", "content": "say hi"}, + { + "role": "system", + "content": [{"type": "text", "text": "Keep answers to one sentence."}], + }, + {"role": "assistant", "content": "Hi.", "thinking_blocks": None}, + {"role": "user", "content": "say bye"}, + ] + + +def test_translate_anthropic_to_openai_moves_midturn_system_after_tool_result(): + """ + A system entry wedged between an assistant tool_use turn and its tool_result turn is + emitted after the role: "tool" message, so the tool call stays paired with its result. + """ + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=[ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01234", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + }, + {"role": "system", "content": "Use the corrected result."}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01234", + "content": "Rainy, 55Ā°F", + } + ], + }, + ], + model="claude-3-5-sonnet-20240620", + ) + + assert [m["role"] for m in result] == ["assistant", "tool", "user"] + assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE + + +def test_translate_anthropic_messages_to_openai_converts_string_midturn_system(): + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=[ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "Keep it short."}, + ], + model="claude-3-5-sonnet-20240620", + ) + + assert result == [ + {"role": "user", "content": "hi"}, + { + "role": "user", + "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Keep it short."}, + ], + }, + ] + + def _claude_code_user_id(session_id: str) -> str: return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id}) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py new file mode 100644 index 00000000000..40a9f4c2536 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py @@ -0,0 +1,89 @@ +from collections import Counter + +from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import ( + CONVERTED_SYSTEM_NOTE, + convert_mid_conversation_system_turns, +) + + +class RoleReadCountingMessage(dict): + def __init__(self, role: str, content: object, reads: Counter): + super().__init__(role=role, content=content) + self.reads = reads + + def get(self, key, default=None): + self.reads[key] += 1 + return super().get(key, default) + + +def test_convert_mid_conversation_system_turns_converts_system_to_user_in_place(): + result = convert_mid_conversation_system_turns( + [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": [{"type": "text", "text": "Keep it short."}]}, + {"role": "assistant", "content": "Hi."}, + ] + ) + + assert result == ( + {"role": "user", "content": "hi"}, + { + "role": "user", + "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Keep it short."}, + ], + }, + {"role": "assistant", "content": "Hi."}, + ) + + +def test_convert_mid_conversation_system_turns_wraps_string_content(): + result = convert_mid_conversation_system_turns( + [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "Keep it short."}, + ] + ) + + assert result[1] == { + "role": "user", + "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "Keep it short."}, + ], + } + + +def test_convert_mid_conversation_system_turns_moves_system_after_tool_result(): + assistant_tool_use = { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}], + } + wedged_system = {"role": "system", "content": "Use the corrected result."} + tool_result = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}], + } + + result = convert_mid_conversation_system_turns([assistant_tool_use, wedged_system, tool_result]) + + assert result[0] is assistant_tool_use + assert result[1] is tool_result + assert result[2]["role"] == "user" + assert result[2]["content"][0]["text"] == CONVERTED_SYSTEM_NOTE + + +def test_convert_mid_conversation_system_turns_reads_each_role_a_bounded_number_of_times(): + reads = Counter() + system_run = [RoleReadCountingMessage("system", f"reminder {i}", reads) for i in range(2_000)] + tool_result = RoleReadCountingMessage( + "user", [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Rainy"}], reads + ) + messages = [RoleReadCountingMessage("user", "hi", reads), *system_run, tool_result] + + result = convert_mid_conversation_system_turns(messages) + + assert reads["role"] <= 3 * len(messages) + assert result[1] is tool_result + assert [m["content"][1]["text"] for m in result[2:]] == [m["content"] for m in system_run] diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index eba8d912fe0..9b28e42f93b 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1189,17 +1189,24 @@ def test_get_supported_openai_params_bedrock_converse(): @pytest.mark.parametrize( - "tools, expected_marker", + "tools, model, expected_marker", [ pytest.param( [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}], + "anthropic.claude-sonnet-4-5-20250929-v1:0", "dep-bedrock", id="tools-present-so-the-cachepoint-is-placed", ), - pytest.param(None, None, id="no-tools-so-nothing-is-placed"), + pytest.param(None, "anthropic.claude-sonnet-4-5-20250929-v1:0", None, id="no-tools-so-nothing-is-placed"), + pytest.param( + [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}], + "global.openai.gpt-6-astra", + None, + id="openai-family-implicit-caching-only", + ), ], ) -def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, expected_marker): +def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, model, expected_marker): """Spend attribution credits the gateway for breakpoints it placed, and a tool_config point becomes one here or nowhere. @@ -1213,7 +1220,7 @@ def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, expec optional_params["tools"] = tools data = AmazonConverseConfig()._transform_request_helper( - model="anthropic.claude-sonnet-4-5-20250929-v1:0", + model=model, system_content_blocks=[], optional_params=optional_params, messages=[{"role": "user", "content": "hi"}], @@ -5591,6 +5598,9 @@ def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model(): True, id="unmapped-arn-keeps-emitting", ), + pytest.param("global.openai.gpt-6-astra", False, id="openai-family-implicit-caching-only"), + pytest.param("openai.gpt-oss-120b-1:0", False, id="openai-gpt-oss"), + pytest.param("us.openai.gpt-99-unmapped", False, id="unmapped-openai-family-still-suppressed"), ], ) def test_cache_points_emitted_only_for_models_that_support_prompt_caching(model, expects_cache_points, monkeypatch): diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 09ebc1a3c95..80f917e0578 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -23,6 +23,9 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) +from litellm.llms.anthropic.experimental_pass_through.messages.mid_conversation_system import ( + as_system_content_blocks, +) from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, AmazonAnthropicClaudeMessagesStreamDecoder, @@ -2533,20 +2536,16 @@ def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_ def test_as_system_content_blocks_handles_each_shape(): - """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, + """``as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value (e.g. a bare content-block dict) -> wrapped in a single-element list.""" block = {"type": "text", "text": "x"} - assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(None) == [] - assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks("hello") == [ - {"type": "text", "text": "hello"} - ] + assert as_system_content_blocks(None) == [] + assert as_system_content_blocks("hello") == [{"type": "text", "text": "hello"}] blocks = [block] - out = AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(blocks) + out = as_system_content_blocks(blocks) assert out == blocks and out is not blocks - assert AmazonAnthropicClaudeMessagesConfig._as_system_content_blocks(block) == [ - block - ] + assert as_system_content_blocks(block) == [block] @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index ac3a43b742f..21838759acd 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -8,7 +8,11 @@ from unittest.mock import MagicMock import pytest import litellm -from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.constants import ( + BEDROCK_REALTIME_SDK_SUPPORTED_RANGE, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, + WEBSOCKET_CLOSE_REASON_MAX_BYTES, +) from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig @@ -207,7 +211,19 @@ class ScriptedBedrockStream: return (None, self._receiver) +class FakeAWSCredentialsIdentity: + def __init__(self, access_key_id, secret_access_key, session_token=None): + self.access_key_id = access_key_id + self.secret_access_key = secret_access_key + self.session_token = session_token + + class FakeStaticCredentialsResolver: + def __init__(self, identity=None): + self.identity = identity + + +class FakeAWSCRTHTTPClient: pass @@ -227,48 +243,32 @@ class StubCredentialsBedrockRealtime(BedrockRealtime): return SimpleNamespace(get_frozen_credentials=lambda: self.frozen_credentials) -@pytest.fixture -def stub_aws_sdk_client(monkeypatch): - captured = {} +class FakeOperationInput: + def __init__(self, model_id): + self.model_id = model_id - class CapturingConfig: - def __init__(self, **kwargs): - captured["config_kwargs"] = kwargs - self.kwargs = kwargs - - class FakeOperationInput: - def __init__(self, model_id): - self.model_id = model_id - - class FakeBedrockRuntimeClient: - def __init__(self, config): - captured["client_config"] = config - - async def invoke_model_with_bidirectional_stream(self, operation_input): - captured["operation_input"] = operation_input - if captured.get("streams"): - stream = captured["streams"].pop(0) - if isinstance(stream, Exception): - raise stream - return stream - return ScriptedBedrockStream(captured.get("scripted_payloads", [])) +def _install_fake_sdk_modules(monkeypatch, client_module, config_module): + """Wire fake aws_sdk_bedrock_runtime / smithy packages into sys.modules for the handler's lazy imports.""" package = types.ModuleType("aws_sdk_bedrock_runtime") - client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") - client_module.BedrockRuntimeClient = FakeBedrockRuntimeClient - client_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput - config_module = types.ModuleType("aws_sdk_bedrock_runtime.config") - config_module.Config = CapturingConfig models_module = types.ModuleType("aws_sdk_bedrock_runtime.models") models_module.BidirectionalInputPayloadPart = FakePayloadPart models_module.InvokeModelWithBidirectionalStreamInputChunk = FakeInputChunk + models_module.InvokeModelWithBidirectionalStreamOperationInput = FakeOperationInput package.client = client_module package.config = config_module package.models = models_module smithy_package = types.ModuleType("smithy_aws_core") identity_module = types.ModuleType("smithy_aws_core.identity") + identity_module.AWSCredentialsIdentity = FakeAWSCredentialsIdentity identity_module.StaticCredentialsResolver = FakeStaticCredentialsResolver smithy_package.identity = identity_module + smithy_http_package = types.ModuleType("smithy_http") + smithy_http_aio = types.ModuleType("smithy_http.aio") + crt_module = types.ModuleType("smithy_http.aio.crt") + crt_module.AWSCRTHTTPClient = FakeAWSCRTHTTPClient + smithy_http_aio.crt = crt_module + smithy_http_package.aio = smithy_http_aio stubbed_modules = { "aws_sdk_bedrock_runtime": package, @@ -277,10 +277,56 @@ def stub_aws_sdk_client(monkeypatch): "aws_sdk_bedrock_runtime.models": models_module, "smithy_aws_core": smithy_package, "smithy_aws_core.identity": identity_module, + "smithy_http": smithy_http_package, + "smithy_http.aio": smithy_http_aio, + "smithy_http.aio.crt": crt_module, } for module_name, module in stubbed_modules.items(): monkeypatch.setitem(sys.modules, module_name, module) + +@pytest.fixture +def stub_aws_sdk_client(monkeypatch): + """Fake of the aws-sdk-bedrock-runtime 0.10/0.11 surface: async config resolve, async client with close()""" + captured = {} + + class FakeAsyncBedrockRuntimeConfig: + def __init__(self, kwargs): + self.kwargs = kwargs + + @classmethod + async def resolve(cls, **kwargs): + captured["config_kwargs"] = kwargs + return cls(kwargs) + + class FakeAsyncBedrockRuntimeClient: + def __init__(self, config): + captured["client_config"] = config + captured["client_closed"] = False + + async def invoke_model_with_bidirectional_stream(self, operation_input): + captured["operation_input"] = operation_input + if captured.get("streams"): + stream = captured["streams"].pop(0) + if isinstance(stream, Exception): + raise stream + captured["open_stream"] = stream + return stream + stream = ScriptedBedrockStream(captured.get("scripted_payloads", [])) + captured["open_stream"] = stream + return stream + + async def close(self): + open_stream = captured.get("open_stream") + captured["input_closed_before_client_close"] = open_stream is None or open_stream.input_stream.closed + captured["client_closed"] = True + + client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") + client_module.AsyncBedrockRuntimeClient = FakeAsyncBedrockRuntimeClient + config_module = types.ModuleType("aws_sdk_bedrock_runtime.config") + config_module.AsyncBedrockRuntimeConfig = FakeAsyncBedrockRuntimeConfig + _install_fake_sdk_modules(monkeypatch, client_module, config_module) + for env_var in ( "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", @@ -764,15 +810,33 @@ class TestBedrockRealtimeAwsAuth: ) config_kwargs = stub_aws_sdk_client["config_kwargs"] - assert config_kwargs["aws_access_key_id"] == "litellm-params-access-key" - assert config_kwargs["aws_secret_access_key"] == "litellm-params-secret-key" - assert config_kwargs["aws_session_token"] == "litellm-params-session-token" - assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver) + resolver = config_kwargs["aws_credentials_identity_resolver"] + assert isinstance(resolver, FakeStaticCredentialsResolver) + assert resolver.identity.access_key_id == "litellm-params-access-key" + assert resolver.identity.secret_access_key == "litellm-params-secret-key" + assert resolver.identity.session_token == "litellm-params-session-token" assert config_kwargs["region"] == "us-east-1" + assert config_kwargs["endpoint_uri"] == "https://bedrock-runtime.us-east-1.amazonaws.com" + assert isinstance(config_kwargs["transport"], FakeAWSCRTHTTPClient) assert stub_aws_sdk_client["client_config"].kwargs is config_kwargs assert stub_aws_sdk_client["operation_input"].model_id == "amazon.nova-sonic-v1:0" assert websocket.closed + @pytest.mark.asyncio + async def test_api_base_overrides_default_endpoint(self, stub_aws_sdk_client): + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=RealtimeClientWS(), + logging_obj=FakeLogging(), + aws_region_name="us-east-1", + aws_access_key_id="k", + aws_secret_access_key="s", + api_base="https://vpce-bedrock.example.internal", + aws_bedrock_runtime_endpoint="https://ignored.example.internal", + ) + + assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://vpce-bedrock.example.internal" + @pytest.mark.asyncio async def test_role_assumption_params_forwarded_to_get_credentials(self, stub_aws_sdk_client): handler = StubCredentialsBedrockRealtime( @@ -805,11 +869,11 @@ class TestBedrockRealtimeAwsAuth: "aws_sts_endpoint": None, "aws_external_id": "realtime-external-id", } - config_kwargs = stub_aws_sdk_client["config_kwargs"] - assert config_kwargs["aws_access_key_id"] == "assumed-access-key" - assert config_kwargs["aws_secret_access_key"] == "assumed-secret-key" - assert config_kwargs["aws_session_token"] == "assumed-session-token" - assert isinstance(config_kwargs["aws_credentials_identity_resolver"], FakeStaticCredentialsResolver) + resolver = stub_aws_sdk_client["config_kwargs"]["aws_credentials_identity_resolver"] + assert isinstance(resolver, FakeStaticCredentialsResolver) + assert resolver.identity.access_key_id == "assumed-access-key" + assert resolver.identity.secret_access_key == "assumed-secret-key" + assert resolver.identity.session_token == "assumed-session-token" @pytest.mark.asyncio async def test_unresolvable_credentials_raise_clear_auth_error(self, stub_aws_sdk_client): @@ -826,5 +890,118 @@ class TestBedrockRealtimeAwsAuth: assert "config_kwargs" not in stub_aws_sdk_client +class TestBedrockRealtimeSdkLifecycle: + """aws-sdk-bedrock-runtime 0.10/0.11: async config, async client, CRT transport, close() (LIT-7938 regression)""" + + AWS_ARGS = { + "model": "amazon.nova-sonic-v1:0", + "aws_region_name": "us-east-1", + "aws_access_key_id": "k", + "aws_secret_access_key": "s", + } + + @pytest.mark.asyncio + async def test_client_closed_after_input_stream_on_normal_completion(self, stub_aws_sdk_client): + await BedrockRealtime().async_realtime(websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS) + + assert stub_aws_sdk_client["client_closed"] + assert stub_aws_sdk_client["input_closed_before_client_close"] + + @pytest.mark.asyncio + async def test_client_closed_when_stream_open_fails(self, stub_aws_sdk_client): + stub_aws_sdk_client["streams"] = [ServiceUnavailableException("bedrock unavailable")] + + with pytest.raises(ServiceUnavailableException): + await BedrockRealtime().async_realtime( + websocket=RealtimeClientWS(), logging_obj=FakeLogging(), **self.AWS_ARGS + ) + + assert stub_aws_sdk_client["client_closed"] + + @pytest.mark.asyncio + async def test_client_closed_when_provider_stream_fails_mid_session(self, stub_aws_sdk_client): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver)] + + with pytest.raises(BedrockError): + await BedrockRealtime().async_realtime( + websocket=ConnectedClientWS([]), logging_obj=FakeLogging(), **self.AWS_ARGS + ) + + assert stub_aws_sdk_client["client_closed"] + assert stub_aws_sdk_client["input_closed_before_client_close"] + + @pytest.mark.asyncio + async def test_client_without_close_completes_session(self, monkeypatch): + class ClientWithoutClose: + def __init__(self, config): + pass + + async def invoke_model_with_bidirectional_stream(self, operation_input): + return ScriptedBedrockStream([]) + + class ConfigWithoutCapture: + @classmethod + async def resolve(cls, **kwargs): + return cls() + + client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") + client_module.AsyncBedrockRuntimeClient = ClientWithoutClose + config_module = types.ModuleType("aws_sdk_bedrock_runtime.config") + config_module.AsyncBedrockRuntimeConfig = ConfigWithoutCapture + _install_fake_sdk_modules(monkeypatch, client_module, config_module) + websocket = RealtimeClientWS() + + await BedrockRealtime().async_realtime(websocket=websocket, logging_obj=FakeLogging(), **self.AWS_ARGS) + + assert websocket.closed + + +class TestBedrockRealtimeSdkImportErrors: + """Init errors must tell 'SDK not installed' apart from 'SDK installed but unsupported version' (LIT-7938)""" + + @pytest.mark.asyncio + async def test_absent_sdk_names_install_extra(self, monkeypatch): + monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime", None) + handler = BedrockRealtime(sdk_version_lookup=lambda: None) + + with pytest.raises(ImportError) as exc_info: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging() + ) + + message = str(exc_info.value) + assert message.startswith("Missing aws_sdk_bedrock_runtime") + assert "litellm[bedrock-realtime]" in message + assert "is installed but" not in message + close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode() + assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason + assert "pip install 'litellm[bedrock-realtime]'" in close_reason + + @pytest.mark.asyncio + async def test_incompatible_sdk_names_installed_version_and_supported_range(self, monkeypatch): + legacy_client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") + legacy_client_module.BedrockRuntimeClient = object + legacy_config_module = types.ModuleType("aws_sdk_bedrock_runtime.config") + legacy_config_module.Config = object + _install_fake_sdk_modules(monkeypatch, legacy_client_module, legacy_config_module) + handler = BedrockRealtime(sdk_version_lookup=lambda: "0.7.0") + + with pytest.raises(ImportError) as exc_info: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), logging_obj=FakeLogging() + ) + + message = str(exc_info.value) + assert "aws-sdk-bedrock-runtime 0.7.0 is installed but" in message + assert ">=0.10.0,<0.12.0" in message + assert not message.startswith("Missing aws_sdk_bedrock_runtime") + assert isinstance(exc_info.value.__cause__, ImportError) + assert str(exc_info.value.__cause__) not in message + assert "cannot import name" not in message + close_reason = message.encode()[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode() + assert "0.7.0 is installed" in close_reason + assert BEDROCK_REALTIME_SDK_SUPPORTED_RANGE in close_reason + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/proxy/client/cli/test_encryption_commands.py b/tests/test_litellm/proxy/client/cli/test_encryption_commands.py index 43e53cf5be2..3a86eb82593 100644 --- a/tests/test_litellm/proxy/client/cli/test_encryption_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_encryption_commands.py @@ -1,4 +1,4 @@ -"""CLI tests for the ``litellm-proxy encryption migrate`` command. +"""CLI tests for the ``lite encryption migrate`` command. The HTTP client is mocked, so these assert the command's request routing (GET check vs POST migrate, dry-run param) and its residual-state messaging without a diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index b73d1acc6e3..d46cc2ad120 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,7 +1,9 @@ # stdlib imports import json import os +import sys from pathlib import Path +from typing import Final from unittest.mock import Mock, patch import pytest @@ -9,7 +11,8 @@ from click.testing import CliRunner import litellm.proxy.client.cli from litellm._version import version as litellm_version -from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli import cli, litellm_proxy_cli +from litellm.proxy.client.cli.main import LITELLM_PROXY_DEPRECATION_NOTICE @pytest.fixture @@ -234,3 +237,32 @@ def test_version_flag_never_sends_api_key_to_unnamed_server(cli_runner, isolated assert all(url.startswith("https://flag-proxy.example.com") for url in requested_urls) sent_keys = [call.kwargs["headers"].get("Authorization") for call in mock_request.call_args_list] assert sent_keys == ["Bearer sk-intended-for-flag-proxy"] * len(requested_urls) + + +def test_litellm_proxy_entrypoint_prints_deprecation_notice_on_stderr_and_still_runs(monkeypatch, capsys, requests_mock): + requests_mock.get("http://localhost:4000/health/readiness", json={"litellm_version": "1.2.3"}) + monkeypatch.setattr(sys, "argv", ["litellm-proxy", "--version"]) + monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000") + with pytest.raises(SystemExit) as exit_info: + litellm_proxy_cli() + + captured: Final = capsys.readouterr() + assert exit_info.value.code == 0 + assert captured.err.strip() == LITELLM_PROXY_DEPRECATION_NOTICE + assert f"LiteLLM Proxy CLI Version: {litellm_version}" in captured.out + assert "LiteLLM Proxy Server Version: 1.2.3" in captured.out + assert "deprecated" not in captured.out + + +def test_lite_entrypoint_prints_nothing_on_stderr(monkeypatch, capsys, requests_mock): + requests_mock.get("http://localhost:4000/health/readiness", json={"litellm_version": "1.2.3"}) + monkeypatch.setattr(sys, "argv", ["lite", "--version"]) + monkeypatch.setenv("LITELLM_PROXY_URL", "http://localhost:4000") + with pytest.raises(SystemExit) as exit_info: + cli() + + captured: Final = capsys.readouterr() + assert exit_info.value.code == 0 + assert "LiteLLM Proxy Server Version: 1.2.3" in captured.out + assert f"LiteLLM Proxy CLI Version: {litellm_version}" in captured.out + assert captured.err == "" diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 1ccf9be37b9..e96069ffa99 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -4,7 +4,7 @@ import sys import types from datetime import datetime, timedelta, timezone from datetime import time as dt_time -from typing import Any, Dict, Final, List +from typing import Any, Dict, Final, List, Optional from unittest.mock import AsyncMock, MagicMock import httpx @@ -16,6 +16,7 @@ from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.common_utils import reset_budget_job as reset_budget_job_module from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MIN_TIME, + RESET_BUDGET_JOB_BATCH_SIZE, RESET_BUDGET_JOB_LOCK_TTL_SECONDS, RESET_BUDGET_JOB_NAME, ) @@ -31,13 +32,36 @@ class MockTable: self.find_many_calls: List[Dict[str, Any]] = [] self.update_many_calls: List[Dict[str, Any]] = [] self._find_many_results: List[Any] = [] + self._find_many_error: Optional[tuple[int, Exception]] = None def set_find_many_results(self, results: List[Any]): self._find_many_results = results - async def find_many(self, where: Dict[str, Any]) -> List[Any]: - self.find_many_calls.append({"where": where}) - return self._find_many_results + def set_find_many_error(self, after_reads: int, error: Exception): + """Fail every read past the first ``after_reads``, the way a connection + dropping partway through a paged walk does.""" + self._find_many_error = (after_reads, error) + + async def find_many( + self, + where: Dict[str, Any], + order: Optional[Dict[str, str]] = None, + take: Optional[int] = None, + ) -> List[Any]: + """Replays canned rows, honouring the keyset cursor + ``take`` a paged + caller relies on: without that a paged walk never advances and the + test would hang instead of failing.""" + if self._find_many_error is not None and len(self.find_many_calls) >= self._find_many_error[0]: + raise self._find_many_error[1] + paging = {k: v for k, v in (("order", order), ("take", take)) if v is not None} + self.find_many_calls.append({"where": where, **paging}) + rows = list(self._find_many_results) + for field, condition in where.items(): + if isinstance(condition, dict) and "gt" in condition and field != "spend": + rows = [row for row in rows if getattr(row, field, "") > condition["gt"]] + for field, direction in (order or {}).items(): + rows.sort(key=lambda row: getattr(row, field, ""), reverse=direction == "desc") + return rows[:take] if take is not None else rows async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: self.update_many_calls.append({"where": where, "data": data}) @@ -801,10 +825,16 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock }, ] - # Verify find_many was called to fetch NULL-budget-id end users + # The post-commit invalidation walk covers both branches, so implicitly + # created customers on the default tier get their cached spend dropped too, + # and it is paged rather than reading the whole customer population. find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls assert len(find_many_calls) == 1 - assert find_many_calls[0]["where"] == {"budget_id": None, "spend": {"gt": 0}} + assert find_many_calls[0]["where"]["OR"] == [ + {"budget_id": {"in": [default_budget_id]}}, + {"budget_id": None}, + ] + assert find_many_calls[0]["take"] == RESET_BUDGET_JOB_BATCH_SIZE litellm.max_end_user_budget_id = None @@ -835,9 +865,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_configured( asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) - # Should NOT have queried for NULL-budget-id end users + # The invalidation walk must not reach for NULL-budget-id customers: they + # ride a default tier that is not expiring, so their spend stays put. find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls - assert len(find_many_calls) == 0 + assert [call["where"] for call in find_many_calls] == [ + {"budget_id": {"in": ["some-budget"]}, "user_id": {"gt": ""}} + ] litellm.max_end_user_budget_id = None @@ -872,9 +905,12 @@ def test_reset_budget_skips_null_budget_id_endusers_when_default_not_in_reset_li asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) - # Should NOT have queried for NULL-budget-id end users + # The invalidation walk must not reach for NULL-budget-id customers: they + # ride a default tier that is not expiring, so their spend stays put. find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls - assert len(find_many_calls) == 0 + assert [call["where"] for call in find_many_calls] == [ + {"budget_id": {"in": ["other-budget"]}, "user_id": {"gt": ""}} + ] litellm.max_end_user_budget_id = None @@ -1252,6 +1288,21 @@ def _make_counter_invalidation_job(monkeypatch): user_api_key_cache = MagicMock() user_api_key_cache.async_delete_cache = AsyncMock() + # Batch deletes fan out to the same per-key calls the real DualCache makes, + # so an assertion reads "this key was invalidated" whether the caller went + # one key at a time or a page at a time. + async def _delete_counter_keys(keys): + for key in keys: + spend_counter_cache.in_memory_cache.delete_cache(key=key) + await spend_counter_cache.redis_cache.async_delete_cache(key=key) + + async def _delete_management_keys(keys): + for key in keys: + await user_api_key_cache.async_delete_cache(key=key) + + spend_counter_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_counter_keys) + user_api_key_cache.async_delete_cache_keys = AsyncMock(side_effect=_delete_management_keys) + fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache fake_module.user_api_key_cache = user_api_key_cache @@ -1586,7 +1637,7 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j "user_id": "customer-42", }, ) - mock_prisma_client.data["enduser"] = [test_enduser] + mock_prisma_client.db.litellm_endusertable.set_find_many_results([test_enduser]) asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) @@ -1596,6 +1647,107 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j assert "end_user_id:customer-42" in deleted +def test_enduser_invalidation_is_paged_and_batched(reset_budget_job, mock_prisma_client, monkeypatch): + """The post-commit invalidation walk stays bounded in memory and in round trips. + + Reading every customer on an expiring tier into one result set puts a + customer-count-sized list in the proxy's heap on every tick, which is an OOM + on a large enough deployment rather than a slow tick. Awaiting one cache call + per customer makes the last customer wait out every customer ahead of it. + Both regress silently, so pin the page size, the strictly advancing cursor, + and one batched call per page. + """ + counter_cache: Final = _make_counter_invalidation_job(monkeypatch) + mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")] + population: Final = RESET_BUDGET_JOB_BATCH_SIZE * 2 + 3 + mock_prisma_client.db.litellm_endusertable.set_find_many_results( + [ + type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"}) + for i in range(population) + ] + ) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + reads: Final = mock_prisma_client.db.litellm_endusertable.find_many_calls + assert [read["take"] for read in reads] == [RESET_BUDGET_JOB_BATCH_SIZE] * 3 + assert [read["where"]["user_id"]["gt"] for read in reads] == [ + "", + f"cust-{RESET_BUDGET_JOB_BATCH_SIZE - 1:06d}", + f"cust-{RESET_BUDGET_JOB_BATCH_SIZE * 2 - 1:06d}", + ] + + assert counter_cache.async_delete_cache_keys.await_count == 3 + assert counter_cache.user_api_key_cache.async_delete_cache_keys.await_count == 3 + counter_cache.async_delete_cache.assert_not_called() + + invalidated: Final = { + key for call in counter_cache.async_delete_cache_keys.await_args_list for key in call.args[0] + } + assert invalidated == {f"spend:end_user:cust-{i:06d}" for i in range(population)} + evicted: Final = { + key for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list for key in call.args[0] + } + assert evicted == {f"end_user_id:cust-{i:06d}" for i in range(population)} + + + +def test_enduser_invalidation_reports_a_page_read_failure_instead_of_a_clean_finish( + mock_prisma_client, monkeypatch +): + """A page that fails to read is not the end of the customer list. + + The tier's window is already advanced by the time this walk runs, so no later + tick comes back for the customers past the page that failed: their cached + spend goes on rejecting requests until it expires. Returning the same empty + page normal end-of-data returns hid that behind a report of a clean pass. + """ + _make_counter_invalidation_job(monkeypatch) + mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")] + endusers: Final = mock_prisma_client.db.litellm_endusertable + endusers.set_find_many_results( + [ + type("EndUser", (), {"user_id": f"cust-{i:06d}", "spend": 5.0, "budget_id": "budget-1"}) + for i in range(RESET_BUDGET_JOB_BATCH_SIZE + 3) + ] + ) + endusers.set_find_many_error(1, RuntimeError("connection reset while paging customers")) + logging_obj: Final = RecordingProxyLogging() + job: Final = ResetBudgetJob(proxy_logging_obj=logging_obj, prisma_client=mock_prisma_client) + + _run_and_drain_hooks(job.reset_budget_for_litellm_budget_table) + + metadata: Final = logging_obj.service_logging_obj.success_calls[0]["event_metadata"] + assert metadata["enduser_invalidation_truncated"] is True + assert metadata["num_endusers_updated"] == RESET_BUDGET_JOB_BATCH_SIZE + + +def test_a_failed_counter_batch_still_evicts_the_management_cache( + reset_budget_job, mock_prisma_client, monkeypatch +): + """The spend counters and the management cache are invalidated independently. + + Sharing one handler meant a Redis failure on the counters returned before the + management cache was touched at all. The commit has already zeroed those rows + by then, so the cached objects keep authorizing against their pre-reset spend + until they expire. + """ + counter_cache: Final = _make_counter_invalidation_job(monkeypatch) + counter_cache.async_delete_cache_keys = AsyncMock(side_effect=RuntimeError("redis unavailable")) + mock_prisma_client.data["budget"] = [_budget_row(budget_id="budget-1")] + mock_prisma_client.db.litellm_endusertable.set_find_many_results( + [type("EndUser", (), {"user_id": "customer-42", "spend": 5.0, "budget_id": "budget-1"})] + ) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + evicted: Final = { + key + for call in counter_cache.user_api_key_cache.async_delete_cache_keys.await_args_list + for key in call.args[0] + } + assert "end_user_id:customer-42" in evicted + def test_budget_table_reset_commits_even_when_cache_eviction_fails(reset_budget_job, mock_prisma_client, monkeypatch): """Eviction runs after the commit, so a broken cache cannot undo the write.""" diff --git a/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py b/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py index 2d5d76ed542..f24175a1922 100644 --- a/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py +++ b/tests/test_litellm/proxy/common_utils/test_user_api_key_cache.py @@ -82,6 +82,19 @@ class FakeRedisCache(RedisCache): async def async_delete_cache(self, key: str): # type: ignore[override] self._store.pop(key, None) + async def delete_cache_keys(self, keys): # type: ignore[override] + for key in keys: + self._store.pop(key, None) + + +class PartitionFailingRedisCache(FakeRedisCache): + """Fails the batch delete for the key-object partition and no other.""" + + async def delete_cache_keys(self, keys): # type: ignore[override] + if any(is_user_key_cache_key(key) for key in keys): + raise ConnectionError("redis unavailable") + await super().delete_cache_keys(keys) + def _make_key_obj(token: str = "tok") -> UserAPIKeyAuth: # Minimal object (UserAPIKeyAuth inherits token from base view). @@ -331,6 +344,46 @@ class TestUserKeyObjectPartition: assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None assert await redis.async_get_cache(HASHED_TOKEN) is None + @pytest.mark.asyncio + async def test_batch_delete_routes_each_key_to_its_partition(self): + """A batch delete has to clear the same partition the single delete does. + + ``DualCache``'s batch delete only knows about the main in-memory cache, so + inheriting it unchanged leaves a key object sitting in ``key_object_cache`` + with its pre-reset spend, and the next request is authorized against that + stale copy until the local entry expires. + """ + redis = FakeRedisCache() + cache = UserApiKeyCache(redis_cache=redis) + await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth) + await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"}) + + await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")]) + + assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is None + assert await cache.async_get_cache(end_user_cache_key("u1")) is None + assert await redis.async_get_cache(HASHED_TOKEN) is None + assert await redis.async_get_cache(end_user_cache_key("u1")) is None + + @pytest.mark.asyncio + async def test_batch_delete_clears_the_other_partition_when_one_fails(self): + """One partition failing must not cost the other its deletions. + + A caller batching these has already committed the rows they cache, so a + partition that is skipped keeps authorizing against pre-reset spend until + the entry expires. The failure is still raised for the caller to report. + """ + redis = PartitionFailingRedisCache() + cache = UserApiKeyCache(redis_cache=redis) + await cache.async_set_cache(HASHED_TOKEN, _make_key_obj(HASHED_TOKEN), model_type=UserAPIKeyAuth) + await cache.async_set_cache(end_user_cache_key("u1"), {"user_id": "u1"}) + + with pytest.raises(ConnectionError): + await cache.async_delete_cache_keys([HASHED_TOKEN, end_user_cache_key("u1")]) + + assert await cache.async_get_cache(end_user_cache_key("u1")) is None + assert await redis.async_get_cache(end_user_cache_key("u1")) is None + @pytest.mark.asyncio async def test_pipeline_write_routes_each_entry_to_its_partition(self): cache = UserApiKeyCache(in_memory_cache=InMemoryCache(max_size_in_memory=2)) diff --git a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py index 31b87530c94..ad0901e9eee 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py @@ -95,18 +95,14 @@ async def test_image_generation_prompt_rerouting(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", {}) - monkeypatch.setattr( - "litellm.proxy.proxy_server.proxy_logging_obj", fake_proxy_logger - ) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", fake_proxy_logger) monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None) monkeypatch.setattr("litellm.proxy.proxy_server.version", "test-version") monkeypatch.setattr( "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers", classmethod(lambda *args, **kwargs: {}), ) - monkeypatch.setattr( - "litellm.proxy.image_endpoints.endpoints.route_request", fake_route_request - ) + monkeypatch.setattr("litellm.proxy.image_endpoints.endpoints.route_request", fake_route_request) result = await endpoints.image_generation( request=request, @@ -141,6 +137,60 @@ def _image_edit_client(monkeypatch, captured: Dict[str, Any]) -> TestClient: return TestClient(app) +def test_image_edit_image_array_alias_is_not_forwarded(monkeypatch): + """The documented `image[]` alias must reach the provider only as `image`.""" + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + files={"image[]": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png")}, + data={"model": "gpt-image-1", "prompt": "add a hat"}, + ) + + assert response.status_code == 200 + assert "image[]" not in captured + assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"] + assert [buffer.name for buffer in captured["image"]] == ["tree.png"] + + +def test_image_edit_mask_array_alias_is_not_forwarded(monkeypatch): + """`mask[]` has the same shape as `image[]` and must be dropped the same way.""" + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + files={ + "image": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png"), + "mask[]": ("mask.png", b"\x89PNG\r\n\x1a\nmask", "image/png"), + }, + data={"model": "gpt-image-1", "prompt": "add a hat"}, + ) + + assert response.status_code == 200 + assert "mask[]" not in captured + assert [buffer.getvalue() for buffer in captured["mask"]] == [b"\x89PNG\r\n\x1a\nmask"] + assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"] + + +def test_image_edit_canonical_file_fields_still_reach_the_provider(monkeypatch): + """Dropping the bracketed aliases must not touch the canonical fields.""" + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + files={ + "image": ("tree.png", b"\x89PNG\r\n\x1a\ntree", "image/png"), + "mask": ("mask.png", b"\x89PNG\r\n\x1a\nmask", "image/png"), + }, + data={"model": "gpt-image-1", "prompt": "add a hat"}, + ) + + assert response.status_code == 200 + assert [buffer.getvalue() for buffer in captured["image"]] == [b"\x89PNG\r\n\x1a\ntree"] + assert [buffer.getvalue() for buffer in captured["mask"]] == [b"\x89PNG\r\n\x1a\nmask"] + assert captured["prompt"] == "add a hat" + + def test_image_edit_multipart_n_reaches_the_provider_as_an_int(monkeypatch): """A multipart `n` must not arrive as the string Starlette parsed it into.""" captured: Dict[str, Any] = {} @@ -180,7 +230,9 @@ async def test_a_model_the_router_cannot_serve_answers_an_openai_typed_error(mon async def fake_add_litellm_data_to_request(**kwargs: object) -> object: return kwargs["data"] - async def fake_pre_call_hook(*, user_api_key_dict: UserAPIKeyAuth, data: dict[str, object], call_type: str) -> dict[str, object]: + async def fake_pre_call_hook( + *, user_api_key_dict: UserAPIKeyAuth, data: dict[str, object], call_type: str + ) -> dict[str, object]: return data async def fake_post_call_failure_hook(**_: object) -> None: @@ -211,7 +263,9 @@ async def test_a_model_the_router_cannot_serve_answers_an_openai_typed_error(mon request = Request({"type": "http", "method": "POST", "path": "/v1/images/generations", "headers": []}, receive) with pytest.raises(ProxyException) as raised: - await endpoints.image_generation(request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth()) + await endpoints.image_generation( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "404") diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_teams.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_teams.py new file mode 100644 index 00000000000..337d47f39da --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_teams.py @@ -0,0 +1,772 @@ +"""`POST /management/v1/teams/{team_id}/members/bulk_update`: the per-member limit writes and the +HTTP contract around them. + +The in-memory Prisma here follows the one in +`tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py`, extended with the budget +table and the membership/budget relation the bulk budget writer needs. +""" + +import copy +from collections.abc import Mapping, Sequence +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict, Field + +from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + team_membership_auth_cache_key, + team_membership_reservation_cache_key, +) +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper +from litellm.proxy.list_api.common import ManagementProblem, problem_response, request_validation_problem +from litellm.proxy.management_endpoints.management_v1 import router +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_helpers.bulk_team_member_budgets import bulk_update_team_member_budgets +from litellm.types.proxy.management_endpoints.team_endpoints import ( + MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES, + BulkTeamMemberBudgetUpdateRequest, + TeamMemberBudgetUpdateResult, +) + +ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin") +OUTSIDER: Final = UserAPIKeyAuth(user_id="outsider", user_role=LitellmUserRoles.INTERNAL_USER) +TEAM_ID: Final = "t1" + + +class _BudgetRow(BaseModel): + """A `LiteLLM_BudgetTable` row, carrying every column the merge patch reads or writes.""" + + model_config = ConfigDict(extra="allow") + + budget_id: str + max_budget: float | None = None + soft_budget: float | None = None + max_parallel_requests: int | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + model_max_budget: Mapping[str, object] | None = None + budget_duration: str | None = None + budget_reset_at: datetime | None = None + allowed_models: list[str] = Field(default_factory=list) + created_by: str | None = None + updated_by: str | None = None + + +class _MembershipRow(BaseModel): + """A `LiteLLM_TeamMembership` row; `litellm_budget_table` is only filled on an `include` read.""" + + model_config = ConfigDict(extra="allow") + + user_id: str + team_id: str + budget_id: str | None = None + litellm_budget_table: _BudgetRow | None = None + + +def _wanted(where: Mapping[str, object], field: str) -> set[str] | None: + clause: Final = where.get(field) + if isinstance(clause, dict) and "in" in clause: + return set(clause["in"]) + if isinstance(clause, str): + return {clause} + return None + + +def _matches(row: Mapping[str, object], where: Mapping[str, object]) -> bool: + return all((wanted := _wanted(where, field)) is not None and row.get(field) in wanted for field in where) + + +class _BudgetTable: + def __init__(self, budgets: Sequence[_BudgetRow]) -> None: + self.rows: dict[str, _BudgetRow] = {b.budget_id: b for b in budgets} + + async def find_unique(self, where: Mapping[str, str]) -> _BudgetRow | None: + return self.rows.get(where["budget_id"]) + + async def update(self, where: Mapping[str, str], data: Mapping[str, object]) -> _BudgetRow: + row: Final = self.rows[where["budget_id"]] + updated: Final = row.model_copy(update=dict(data)) + self.rows[row.budget_id] = updated + return updated + + async def create(self, data: Mapping[str, object], include: Mapping[str, bool] | None = None) -> _BudgetRow: + budget_id: Final = f"new-budget-{len(self.rows) + 1}" + row: Final = _BudgetRow.model_validate({**data, "budget_id": budget_id}) + self.rows[budget_id] = row + return row + + +class _MembershipTable: + def __init__(self, budgets: _BudgetTable, memberships: Sequence[_MembershipRow]) -> None: + self._budgets = budgets + self.rows: list[_MembershipRow] = list(memberships) + + def _index_of(self, user_id: str, team_id: str) -> int | None: + return next( + (i for i, r in enumerate(self.rows) if r.user_id == user_id and r.team_id == team_id), + None, + ) + + async def find_many( + self, where: Mapping[str, object], include: Mapping[str, bool] | None = None + ) -> list[_MembershipRow]: + matched: Final = [r for r in self.rows if _matches(r.model_dump(), where)] + if not include: + return matched + return [ + r.model_copy(update={"litellm_budget_table": self._budgets.rows.get(r.budget_id or "")}) for r in matched + ] + + async def update(self, where: Mapping[str, Mapping[str, str]], data: Mapping[str, object]) -> _MembershipRow: + key: Final = where["user_id_team_id"] + index: Final = self._index_of(key["user_id"], key["team_id"]) + assert index is not None, f"no membership row for {key}" + relation: Final = data.get("litellm_budget_table") + if isinstance(relation, dict) and relation.get("disconnect"): + self.rows[index] = self.rows[index].model_copy(update={"budget_id": None}) + return self.rows[index] + + async def upsert(self, where: Mapping[str, Mapping[str, str]], data: Mapping[str, object]) -> _MembershipRow: + key: Final = where["user_id_team_id"] + budget_id: Final = data["update"]["litellm_budget_table"]["connect"]["budget_id"] + index: Final = self._index_of(key["user_id"], key["team_id"]) + if index is None: + self.rows.append(_MembershipRow(user_id=key["user_id"], team_id=key["team_id"], budget_id=budget_id)) + return self.rows[-1] + self.rows[index] = self.rows[index].model_copy(update={"budget_id": budget_id}) + return self.rows[index] + + +class _TeamTable: + """`find_many` and `create` are what `RoutingPrismaWrapper` keys read routing off, so a fake + table without them would silently never route and pass a reader-staleness test on the writer.""" + + def __init__(self, teams: Sequence[LiteLLM_TeamTable]) -> None: + self.rows: dict[str, LiteLLM_TeamTable] = {t.team_id: t for t in teams} + + async def find_unique(self, where: Mapping[str, str]) -> LiteLLM_TeamTable | None: + return self.rows.get(where["team_id"]) + + async def find_many(self, where: Mapping[str, object] | None = None) -> list[LiteLLM_TeamTable]: + return [t for t in self.rows.values() if where is None or _matches(t.model_dump(), where)] + + async def create(self, data: Mapping[str, object]) -> LiteLLM_TeamTable: + row: Final = LiteLLM_TeamTable.model_validate(dict(data)) + self.rows[row.team_id] = row + return row + + +class _Db: + def __init__( + self, + teams: Sequence[LiteLLM_TeamTable], + memberships: Sequence[_MembershipRow], + budgets: Sequence[_BudgetRow], + ) -> None: + self.litellm_teamtable = _TeamTable(teams) + self.litellm_budgettable = _BudgetTable(budgets) + self.litellm_teammembership = _MembershipTable(self.litellm_budgettable, memberships) + + +class _FakePrisma: + def __init__( + self, + teams: Sequence[LiteLLM_TeamTable] = (), + memberships: Sequence[_MembershipRow] = (), + budgets: Sequence[_BudgetRow] = (), + ) -> None: + self.db = _Db(teams, memberships, budgets) + + @asynccontextmanager + async def tx(self, *, timeout: object = None): + snapshot: Final = copy.deepcopy(self.db) + try: + yield self.db + except BaseException: + self.db = snapshot + raise + + +class _ReplicatedPrisma: + """A client whose reads route to a lagging replica, as a proxy with `DATABASE_URL_READ_REPLICA` does.""" + + def __init__(self, writer: _FakePrisma, reader: _FakePrisma) -> None: + self._writer = writer + self.db = RoutingPrismaWrapper(writer=writer.db, reader=reader.db) # pyright: ignore[reportArgumentType] # fake dbs stand in for PrismaWrapper + + def tx(self, *, timeout: object = None): + return self._writer.tx(timeout=timeout) + + +class _UnreachableDb: + """A `.db` whose every table access fails, as one behind a dropped connection does.""" + + def __getattr__(self, name: str) -> object: + raise RuntimeError("connection reset by peer") + + +class _UnreachablePrisma: + def __init__(self) -> None: + self.db = _UnreachableDb() + + +def _team( + *members: str, + team_id: str = TEAM_ID, + default_budget_id: str | None = None, + admins: Sequence[str] = (), +) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + metadata={"team_member_budget_id": default_budget_id} if default_budget_id else {}, + members_with_roles=[ + Member(user_id=m, user_email=f"{m}@example.com", role="admin" if m in admins else "user") for m in members + ], + ) + + +def _membership(user_id: str, budget_id: str | None = None, team_id: str = TEAM_ID) -> _MembershipRow: + return _MembershipRow(user_id=user_id, team_id=team_id, budget_id=budget_id) + + +def _budget( + budget_id: str, + *, + max_budget: float | None = None, + tpm_limit: int | None = None, + rpm_limit: int | None = None, + budget_duration: str | None = None, +) -> _BudgetRow: + return _BudgetRow( + budget_id=budget_id, + max_budget=max_budget, + tpm_limit=tpm_limit, + rpm_limit=rpm_limit, + budget_duration=budget_duration, + ) + + +async def _bulk_update( + prisma: _FakePrisma | _ReplicatedPrisma, + members: Sequence[Mapping[str, object]], + team_id: str = TEAM_ID, + caller: UserAPIKeyAuth = ADMIN, + cache: UserApiKeyCache | None = None, +) -> tuple[TeamMemberBudgetUpdateResult, ...]: + return await bulk_update_team_member_budgets( + team_id=team_id, + data=BulkTeamMemberBudgetUpdateRequest.model_validate({"members": list(members)}), + user_api_key_dict=caller, + prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + user_api_key_cache=cache or UserApiKeyCache(), + ) + + +def _budget_id_of(prisma: _FakePrisma, user_id: str, team_id: str = TEAM_ID) -> str | None: + row: Final = next(r for r in prisma.db.litellm_teammembership.rows if r.user_id == user_id and r.team_id == team_id) + return row.budget_id + + +def _budget_of(prisma: _FakePrisma, user_id: str, team_id: str = TEAM_ID) -> _BudgetRow: + budget_id: Final = _budget_id_of(prisma, user_id, team_id) + assert budget_id is not None, f"{user_id} has no budget" + return prisma.db.litellm_budgettable.rows[budget_id] + + +def _seeded_cache(*user_ids: str, team_id: str = TEAM_ID) -> UserApiKeyCache: + cache: Final = UserApiKeyCache() + for user_id in user_ids: + cache.set_cache(key=team_membership_auth_cache_key(team_id=team_id, user_id=user_id), value={"cap": "old"}) + cache.set_cache( + key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id), value={"cap": "old"} + ) + return cache + + +def _cached_keys(cache: UserApiKeyCache, user_id: str, team_id: str = TEAM_ID) -> tuple[object, object]: + return ( + cache.get_cache(key=team_membership_auth_cache_key(team_id=team_id, user_id=user_id)), + cache.get_cache(key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id)), + ) + + +@pytest.mark.asyncio +async def test_patching_one_member_of_a_shared_budget_row_forks_it_and_leaves_the_other_member_untouched(): + prisma = _FakePrisma( + teams=[_team("m1", "m2")], + memberships=[_membership("m1", "shared-b"), _membership("m2", "shared-b")], + budgets=[_budget("shared-b", max_budget=100.0, tpm_limit=900)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 50}]) + + assert [(r.user_id, r.success, r.max_budget) for r in results] == [("m1", True, 50.0)] + assert _budget_id_of(prisma, "m1") not in (None, "shared-b") + assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (50.0, 900) + assert _budget_id_of(prisma, "m2") == "shared-b" + assert prisma.db.litellm_budgettable.rows["shared-b"].max_budget == 100.0 + assert results[0].budget_id == _budget_id_of(prisma, "m1") + + +@pytest.mark.asyncio +async def test_patching_members_of_the_team_default_budget_gives_each_their_own_row_and_leaves_the_default_alone(): + prisma = _FakePrisma( + teams=[_team("m1", "m2", "m3", default_budget_id="team-default")], + memberships=[ + _membership("m1", "team-default"), + _membership("m2", "team-default"), + _membership("m3", "team-default"), + ], + budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)], + ) + + results = await _bulk_update( + prisma, + [{"user_id": "m1", "max_budget_in_team": 5}, {"user_id": "m2", "max_budget_in_team": 7}], + ) + + assert [r.success for r in results] == [True, True] + default = prisma.db.litellm_budgettable.rows["team-default"] + assert (default.max_budget, default.tpm_limit) == (25.0, 1000) + assert _budget_id_of(prisma, "m3") == "team-default" + patched = (_budget_id_of(prisma, "m1"), _budget_id_of(prisma, "m2")) + assert len(set(patched)) == 2 and "team-default" not in patched + assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (5.0, 1000) + assert (_budget_of(prisma, "m2").max_budget, _budget_of(prisma, "m2").tpm_limit) == (7.0, 1000) + + +@pytest.mark.asyncio +async def test_the_team_default_row_is_forked_even_when_only_one_membership_points_at_it(): + prisma = _FakePrisma( + teams=[_team("m1", "m2", default_budget_id="team-default")], + memberships=[_membership("m1", "team-default")], + budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 5}]) + + assert [(r.success, r.max_budget, r.tpm_limit) for r in results] == [(True, 5.0, 1000)] + default = prisma.db.litellm_budgettable.rows["team-default"] + assert (default.max_budget, default.tpm_limit) == (25.0, 1000) + assert _budget_id_of(prisma, "m1") not in (None, "team-default") + + +@pytest.mark.asyncio +async def test_a_budget_row_only_one_member_points_at_is_updated_in_place(): + prisma = _FakePrisma( + teams=[_team("m1", "m2", default_budget_id="team-default")], + memberships=[_membership("m1", "priv-m1"), _membership("m2", "team-default")], + budgets=[_budget("team-default", max_budget=25.0), _budget("priv-m1", max_budget=10.0, tpm_limit=5)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 20}]) + + assert [(r.success, r.budget_id, r.max_budget) for r in results] == [(True, "priv-m1", 20.0)] + assert set(prisma.db.litellm_budgettable.rows) == {"team-default", "priv-m1"} + assert _budget_id_of(prisma, "m1") == "priv-m1" + assert (_budget_of(prisma, "m1").max_budget, _budget_of(prisma, "m1").tpm_limit) == (20.0, 5) + + +@pytest.mark.asyncio +async def test_an_omitted_field_is_kept_an_explicit_null_clears_it_and_clearing_the_last_limit_disconnects(): + prisma = _FakePrisma( + teams=[_team("m1")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("priv-m1", max_budget=10.0, tpm_limit=5, rpm_limit=7)], + ) + + kept = await _bulk_update(prisma, [{"user_id": "m1", "rpm_limit": 9}]) + + assert (kept[0].max_budget, kept[0].tpm_limit, kept[0].rpm_limit) == (10.0, 5, 9) + + cleared = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": None}]) + + assert (cleared[0].max_budget, cleared[0].tpm_limit, cleared[0].rpm_limit) == (10.0, None, 9) + assert _budget_id_of(prisma, "m1") == "priv-m1" + + emptied = await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": None, "rpm_limit": None}]) + + assert (emptied[0].success, emptied[0].budget_id, emptied[0].max_budget) == (True, None, None) + assert _budget_id_of(prisma, "m1") is None + + +@pytest.mark.asyncio +async def test_budget_duration_seeds_a_reset_time_derived_from_the_duration_and_clearing_it_clears_the_reset(): + prisma = _FakePrisma( + teams=[_team("m1", "m2")], + memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")], + budgets=[_budget("priv-m1", max_budget=10.0), _budget("priv-m2", max_budget=10.0)], + ) + before = datetime.now(timezone.utc) + + await _bulk_update( + prisma, + [{"user_id": "m1", "budget_duration": "2d"}, {"user_id": "m2", "budget_duration": "5d"}], + ) + + two_day = _budget_of(prisma, "m1").budget_reset_at + five_day = _budget_of(prisma, "m2").budget_reset_at + assert two_day is not None and five_day is not None + assert before < two_day <= before + timedelta(days=2) + assert before + timedelta(days=4) - timedelta(seconds=1) < five_day <= before + timedelta(days=5) + assert five_day - two_day == timedelta(days=3) + + await _bulk_update(prisma, [{"user_id": "m1", "budget_duration": None}]) + + assert _budget_of(prisma, "m1").budget_reset_at is None + assert _budget_of(prisma, "m1").budget_duration is None + assert _budget_of(prisma, "m1").max_budget == 10.0 + + +@pytest.mark.asyncio +async def test_a_member_named_twice_is_written_once_and_the_later_rows_report_the_duplicate(): + prisma = _FakePrisma( + teams=[_team("m1")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("priv-m1", max_budget=1.0)], + ) + + results = await _bulk_update( + prisma, + [ + {"user_id": "m1", "max_budget_in_team": 10}, + {"user_id": "m1", "max_budget_in_team": 20}, + {"user_email": "m1@example.com", "max_budget_in_team": 30}, + ], + ) + + assert [(r.success, r.error) for r in results] == [ + (True, None), + (False, "Duplicate member in request"), + (False, "Duplicate member in request"), + ] + assert _budget_of(prisma, "m1").max_budget == 10.0 + + +@pytest.mark.asyncio +async def test_a_row_naming_somebody_off_the_team_fails_without_writing_while_the_rest_of_the_batch_lands(): + prisma = _FakePrisma( + teams=[_team("m1")], + memberships=[_membership("m1", "priv-m1"), _membership("elsewhere", "priv-other")], + budgets=[_budget("priv-m1", max_budget=1.0), _budget("priv-other", max_budget=2.0)], + ) + + results = await _bulk_update( + prisma, + [ + {"user_id": "elsewhere", "max_budget_in_team": 99}, + {"user_email": "nobody@example.com", "max_budget_in_team": 99}, + {"user_id": "m1", "max_budget_in_team": 10}, + ], + ) + + assert [(r.success, r.error) for r in results] == [ + (False, "User not found in team"), + (False, "User not found in team"), + (True, None), + ] + assert prisma.db.litellm_budgettable.rows["priv-other"].max_budget == 2.0 + assert _budget_of(prisma, "m1").max_budget == 10.0 + assert set(prisma.db.litellm_budgettable.rows) == {"priv-m1", "priv-other"} + + +@pytest.mark.asyncio +async def test_each_result_carries_the_limits_read_back_after_the_write_in_request_order(): + prisma = _FakePrisma( + teams=[_team("m1", "m2")], + memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")], + budgets=[ + _budget("priv-m1", tpm_limit=100, budget_duration="7d"), + _budget("priv-m2", rpm_limit=3), + ], + ) + + results = await _bulk_update( + prisma, + [{"user_id": "m2", "rpm_limit": 8}, {"user_id": "m1", "max_budget_in_team": 42}], + ) + + assert [r.user_id for r in results] == ["m2", "m1"] + assert (results[1].max_budget, results[1].tpm_limit, results[1].budget_duration) == (42.0, 100, "7d") + assert (results[0].rpm_limit, results[0].max_budget) == (8, None) + + +@pytest.mark.asyncio +async def test_every_written_member_is_evicted_from_both_team_membership_cache_keys(): + prisma = _FakePrisma( + teams=[_team("m1", "m2", "m3")], + memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2"), _membership("m3", "priv-m3")], + budgets=[_budget("priv-m1", max_budget=1.0), _budget("priv-m2", max_budget=2.0), _budget("priv-m3")], + ) + cache = _seeded_cache("m1", "m2", "m3") + + await _bulk_update( + prisma, + [{"user_id": "m1", "max_budget_in_team": 10}, {"user_id": "m2", "max_budget_in_team": 20}], + cache=cache, + ) + + assert _cached_keys(cache, "m1") == (None, None) + assert _cached_keys(cache, "m2") == (None, None) + assert _cached_keys(cache, "m3") == ({"cap": "old"}, {"cap": "old"}) + + +@pytest.mark.asyncio +async def test_a_member_with_no_cap_of_their_own_reports_the_team_default_cap_but_only_their_own_rate_limits(): + prisma = _FakePrisma( + teams=[_team("m1", default_budget_id="team-default")], + memberships=[], + budgets=[_budget("team-default", max_budget=25.0, tpm_limit=1000)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": 7}]) + + assert [(r.success, r.max_budget, r.max_budget_source, r.tpm_limit) for r in results] == [ + (True, 25.0, "team_default", 7) + ] + assert _budget_of(prisma, "m1").max_budget is None + default = prisma.db.litellm_budgettable.rows["team-default"] + assert (default.max_budget, default.tpm_limit) == (25.0, 1000) + + +@pytest.mark.asyncio +async def test_an_explicit_cap_reports_as_the_members_own_while_clearing_one_falls_back_to_the_team_default(): + prisma = _FakePrisma( + teams=[_team("m1", "m2", default_budget_id="team-default")], + memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")], + budgets=[ + _budget("team-default", max_budget=25.0), + _budget("priv-m1", max_budget=5.0), + _budget("priv-m2", max_budget=9.0), + ], + ) + + results = await _bulk_update( + prisma, + [{"user_id": "m1", "max_budget_in_team": 50}, {"user_id": "m2", "max_budget_in_team": None}], + ) + + assert [(r.user_id, r.max_budget, r.max_budget_source) for r in results] == [ + ("m1", 50.0, "member"), + ("m2", 25.0, "team_default"), + ] + assert results[1].budget_id is None + assert _budget_id_of(prisma, "m2") is None + assert prisma.db.litellm_budgettable.rows["team-default"].max_budget == 25.0 + + +@pytest.mark.asyncio +async def test_a_team_with_no_default_budget_reports_no_effective_cap_for_a_member_without_one(): + prisma = _FakePrisma( + teams=[_team("m1")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("priv-m1", tpm_limit=5)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "rpm_limit": 3}]) + + assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [(True, None, None)] + assert (results[0].tpm_limit, results[0].rpm_limit) == (5, 3) + + +@pytest.mark.asyncio +async def test_a_zero_team_default_reports_no_cap_because_enforcement_reads_zero_there_as_uncapped(): + prisma = _FakePrisma( + teams=[_team("m1", default_budget_id="team-default")], + memberships=[_membership("m1", None)], + budgets=[_budget("team-default", max_budget=0.0)], + ) + + results = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": 9}]) + + assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [(True, None, None)] + assert results[0].tpm_limit == 9 + + +@pytest.mark.asyncio +async def test_a_row_that_names_nobody_on_the_team_reports_no_cap_and_no_source(): + prisma = _FakePrisma( + teams=[_team("m1", default_budget_id="team-default")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("team-default", max_budget=25.0), _budget("priv-m1", max_budget=5.0)], + ) + + results = await _bulk_update( + prisma, + [{"user_id": "ghost", "max_budget_in_team": 1}, {"user_id": "m1", "max_budget_in_team": 6}], + ) + + assert [(r.success, r.max_budget, r.max_budget_source) for r in results] == [ + (False, None, None), + (True, 6.0, "member"), + ] + + +@pytest.mark.asyncio +async def test_the_roster_authz_read_runs_on_the_writer_so_a_lagging_replica_cannot_let_a_demoted_admin_write(): + writer = _FakePrisma( + teams=[_team("lead", "m1")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("priv-m1", max_budget=1.0)], + ) + replica = _FakePrisma(teams=[_team("lead", "m1", admins=("lead",))]) + demoted = UserAPIKeyAuth(user_id="lead", user_role=LitellmUserRoles.INTERNAL_USER) + + with pytest.raises(ManagementProblem) as raised: + await _bulk_update( + _ReplicatedPrisma(writer=writer, reader=replica), + [{"user_id": "m1", "max_budget_in_team": 99}], + caller=demoted, + ) + + assert raised.value.problem.status == 403 + assert writer.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0 + + +app = FastAPI() + + +@app.exception_handler(ManagementProblem) +async def management_problem_exception_handler(request: Request, exc: ManagementProblem): + return problem_response(exc.problem) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return problem_response(request_validation_problem(exc.errors())) + + +app.include_router(router) +client = TestClient(app) + +BULK_UPDATE_PATH: Final = f"{MANAGEMENT_V1_PREFIX}/teams/{TEAM_ID}/members/bulk_update" + + +@pytest.fixture +def as_proxy_admin(): + app.dependency_overrides[user_api_key_auth] = lambda: ADMIN + yield + app.dependency_overrides.clear() + + +@pytest.fixture +def as_outsider(): + app.dependency_overrides[user_api_key_auth] = lambda: OUTSIDER + yield + app.dependency_overrides.clear() + + +@pytest.fixture +def prisma(monkeypatch): + fake = _FakePrisma( + teams=[_team("m1", "m2")], + memberships=[_membership("m1", "priv-m1")], + budgets=[_budget("priv-m1", max_budget=1.0)], + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake) + return fake + + +def _post(body: object, path: str = BULK_UPDATE_PATH): + return client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"}) + + +def test_unknown_fields_empty_and_oversized_batches_are_422_problem_documents(prisma, as_proxy_admin): + bodies = ( + {"members": [{"user_id": "m1", "max_budget": 10}]}, + {"members": [{"user_id": "m1"}], "team_id": TEAM_ID}, + {"members": []}, + {"members": [{"user_id": f"u{i}"} for i in range(MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES + 1)]}, + ) + + for body in bodies: + response = _post(body) + + assert response.status_code == 422, body + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:invalid-request-body" + assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0 + + +def test_an_unknown_team_is_a_404_problem_document(prisma, as_proxy_admin): + response = _post( + {"members": [{"user_id": "m1", "max_budget_in_team": 10}]}, + path=f"{MANAGEMENT_V1_PREFIX}/teams/nope/members/bulk_update", + ) + + assert response.status_code == 404 + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:team-not-found" + assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0 + + +def test_a_caller_who_administers_neither_the_team_nor_its_org_is_a_403_problem_document(prisma, as_outsider): + response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]}) + + assert response.status_code == 403 + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:forbidden" + assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0 + + +def test_a_team_admin_may_bulk_update_their_own_teams_members(prisma, monkeypatch): + prisma.db.litellm_teamtable.rows[TEAM_ID] = _team("lead", "m1", admins=("lead",)) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="lead", user_role=LitellmUserRoles.INTERNAL_USER + ) + try: + response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]}) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + assert [(r["user_id"], r["success"], r["max_budget"]) for r in response.json()["data"]] == [("m1", True, 10.0)] + + +@pytest.mark.parametrize("duration", ("0d", "nonsense")) +def test_a_budget_duration_no_reset_can_be_scheduled_from_is_a_422_naming_its_row_and_writes_nothing( + prisma, as_proxy_admin, duration +): + response = _post( + { + "members": [ + {"user_id": "m1", "max_budget_in_team": 10}, + {"user_id": "m2", "budget_duration": duration}, + ] + } + ) + + assert response.status_code == 422 + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:invalid-request-body" + assert "members.1.budget_duration" in response.json()["detail"] + assert prisma.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0 + + +def test_an_unconnected_database_is_a_503_problem_document(monkeypatch, as_proxy_admin): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + + response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]}) + + assert response.status_code == 503 + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:database-not-connected" + + +def test_a_driver_error_answers_as_a_problem_document_without_leaking_the_exception(monkeypatch, as_proxy_admin): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _UnreachablePrisma()) + + response = _post({"members": [{"user_id": "m1", "max_budget_in_team": 10}]}) + + assert response.status_code == 500 + assert response.headers["content-type"] == "application/problem+json" + assert response.json()["type"] == "urn:litellm:error:internal-server-error" + assert "connection reset by peer" not in response.text diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 60f9a1a55e2..364ec4aad61 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -7,7 +7,8 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, call import pytest -from fastapi import HTTPException +from fastapi import FastAPI, HTTPException +from httpx import ASGITransport, AsyncClient from pytest_mock import MockerFixture from litellm.proxy._types import ( @@ -31,6 +32,7 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( _handle_group_membership_changes, _handle_team_membership_changes, _parse_member_entries, + _premium_user_check, _process_group_patch_operations, _recompute_scim_member_roles, _resolve_group_member_ids, @@ -45,8 +47,10 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( patch_group, patch_team_membership, patch_user, + scim_router, update_group, update_user, + user_api_key_auth, ) from litellm.types.proxy.management_endpoints.scim_v2 import ( SCIM_ENTERPRISE_USER_SCHEMA, @@ -484,6 +488,48 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp ) +@pytest.fixture +def scim_test_client(): + """An in-process SCIM application with authorization dependencies bypassed.""" + app = FastAPI() + app.dependency_overrides[_premium_user_check] = lambda: None + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + app.include_router(scim_router) + return AsyncClient(transport=ASGITransport(app=app), base_url="http://test") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["Users", "Groups"]) +@pytest.mark.parametrize(("requested_count", "effective_count"), [(0, 0), (200, 100), (1000, 100)]) +async def test_scim_collection_endpoints_clamp_requested_page_size( + scim_test_client, endpoint, requested_count, effective_count, mocker +): + """SCIM list endpoints accept zero and cap larger client page requests.""" + mock_prisma_client = MagicMock() + mock_prisma_client.db = MagicMock() + table = MagicMock() + table.find_many = AsyncMock(return_value=[]) + table.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_usertable = table + mock_prisma_client.db.litellm_teamtable = table + mocker.patch( # test-quality-ok: HTTP validation requires an in-memory database boundary. + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + + async with scim_test_client as client: + response = await client.get(f"/scim/v2/{endpoint}?startIndex=1&count={requested_count}") + + assert response.status_code == 200 + table.find_many.assert_awaited_once_with( + where={}, + skip=0, + take=effective_count, + order={"created_at": "desc"}, + ) + assert response.json()["itemsPerPage"] == 0 + + @pytest.mark.asyncio async def test_get_users_filters_username_by_exposed_scim_username_for_okta(mocker): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a7748220e44..4b3adcfff4e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -15424,8 +15424,8 @@ async def test_team_info_reports_what_the_caller_may_edit(caller, org_admin, ena assert response["team_info"].caller_edit_access.model_dump(mode="json") == expected -def test_build_member_budget_patch_maps_temp_budget_fields() -> None: - from litellm.proxy.management_endpoints.team_endpoints import _build_member_budget_patch +def test_member_budget_patch_maps_temp_budget_fields() -> None: + from litellm.proxy.management_endpoints.common_utils import member_budget_patch expiry: Final = datetime(2030, 1, 1, tzinfo=timezone.utc) request: Final = TeamMemberUpdateRequest( @@ -15434,7 +15434,7 @@ def test_build_member_budget_patch_maps_temp_budget_fields() -> None: temp_budget_increase=50.0, temp_budget_expiry=expiry, ) - assert _build_member_budget_patch(request) == { + assert member_budget_patch(request) == { "temp_budget_increase": 50.0, "temp_budget_expiry": expiry, } diff --git a/tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py b/tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py index 44572aed08e..e157c982105 100644 --- a/tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py +++ b/tests/test_litellm/test_dockerfile_bedrock_realtime_extra.py @@ -4,15 +4,23 @@ Static checks that every proxy Docker image installs the `bedrock-realtime` extr Bedrock Nova Sonic speech-to-speech (`/v1/realtime`) needs `aws-sdk-bedrock-runtime`, which only ships in the `bedrock-realtime` extra. An image whose `uv sync` stages omit the extra fails every Nova Sonic realtime session with -"Missing aws_sdk_bedrock_runtime. Install with: pip install aws-sdk-bedrock-runtime". +"Missing aws_sdk_bedrock_runtime: pip install 'litellm[bedrock-realtime]' ...". """ import os import re +import sys from typing import Final import pytest +from litellm.constants import BEDROCK_REALTIME_SDK_DISTRIBUTION, BEDROCK_REALTIME_SDK_SUPPORTED_RANGE + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "..", "..") PROXY_DOCKERFILES: Final = ( @@ -54,3 +62,16 @@ def test_every_uv_sync_installs_bedrock_realtime_extra(relative_path: str): "`--extra bedrock-realtime`, so aws-sdk-bedrock-runtime is absent and Bedrock Nova Sonic " "/v1/realtime sessions fail with 'Missing aws_sdk_bedrock_runtime'" ) + + +def test_bedrock_realtime_extra_pins_the_range_named_in_the_runtime_error(): + with open(os.path.join(REPO_ROOT, "pyproject.toml"), "rb") as f: + extra_specs: Final = tomllib.load(f)["project"]["optional-dependencies"]["bedrock-realtime"] + + sdk_specs: Final = tuple(spec for spec in extra_specs if spec.startswith(BEDROCK_REALTIME_SDK_DISTRIBUTION)) + assert len(sdk_specs) == 1, f"expected exactly one {BEDROCK_REALTIME_SDK_DISTRIBUTION} spec, got {extra_specs}" + requirement: Final = sdk_specs[0].split(";")[0].strip() + assert requirement == f"{BEDROCK_REALTIME_SDK_DISTRIBUTION}[awscrt]{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE}", ( + f"pyproject pins {requirement!r} but the handler's install hint names " + f"{BEDROCK_REALTIME_SDK_SUPPORTED_RANGE!r} with the awscrt extra; keep them in sync" + ) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 7f343211596..ff5e736306a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -1,7 +1,7 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; -import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils"; +import { renderWithProviders, screen, waitFor, within } from "../../../../tests/test-utils"; import { GuardrailInformation, makeBedrockResponse, @@ -24,6 +24,42 @@ const skippedPreCall: Partial = { duration: null, }; +const untimedPreCall: Partial = { + guardrail_name: "conduct", + guardrail_status: "success", + guardrail_mode: "pre_call", + start_time: null, + end_time: null, + duration: null, +}; + +const timedPreCall: Partial = { + guardrail_name: "timed-pre-rail", + guardrail_status: "success", + guardrail_mode: "pre_call", + start_time: 1_700_000_000, + end_time: 1_700_000_000.1, + duration: 0.1, +}; + +const latePreCall: Partial = { + guardrail_name: "late-pre-rail", + guardrail_status: "success", + guardrail_mode: "pre_call", + start_time: 1_700_000_500, + end_time: 1_700_000_500.1, + duration: 0.1, +}; + +const untimedPostCall: Partial = { + guardrail_name: "untimed-post-rail", + guardrail_status: "success", + guardrail_mode: "post_call", + start_time: null, + end_time: null, + duration: null, +}; + const ranPostCall: Partial = { guardrail_name: "ran-rail", guardrail_status: "success", @@ -98,6 +134,67 @@ describe("GuardrailViewer", () => { expect(screen.getByText("ā€”")).toBeInTheDocument(); }); + it("keeps a guardrail that ran without any timing on the lifecycle", () => { + renderWithProviders(); + + expect(screen.getByText("Request received")).toBeInTheDocument(); + expect(screen.getByText(/Pre-call guardrail: conduct/)).toBeInTheDocument(); + expect(screen.getByText("LLM call")).toBeInTheDocument(); + expect(screen.getByText("Response returned")).toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + }); + + it("keeps an untimed guardrail ahead of a timed one recorded after it in the same phase", () => { + const untimed = makeGuardrailInformation(untimedPreCall); + const timedPre = makeGuardrailInformation(timedPreCall); + renderWithProviders(); + + const rows = screen.getAllByTestId("lifecycle-row"); + const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null); + const untimedIndex = rowIndex(/Pre-call guardrail: conduct/); + const timedIndex = rowIndex(/Pre-call guardrail: timed-pre-rail/); + + expect(untimedIndex).toBeGreaterThanOrEqual(0); + expect(timedIndex).toBeGreaterThanOrEqual(0); + expect(untimedIndex).toBeLessThan(timedIndex); + }); + + it("orders each phase on its own clock when a later pre-call outlives an earlier post-call", () => { + const latePre = makeGuardrailInformation(latePreCall); + const untimedPost = makeGuardrailInformation(untimedPostCall); + const earlyPost = makeGuardrailInformation(ranPostCall); + renderWithProviders(); + + const rows = screen.getAllByTestId("lifecycle-row"); + const rowIndex = (label: RegExp): number => rows.findIndex((r) => within(r).queryByText(label) !== null); + const untimedIndex = rowIndex(/Post-call guardrail: untimed-post-rail/); + const earlyIndex = rowIndex(/Post-call guardrail: ran-rail/); + + expect(untimedIndex).toBeGreaterThanOrEqual(0); + expect(earlyIndex).toBeGreaterThanOrEqual(0); + expect(untimedIndex).toBeLessThan(earlyIndex); + }); + + it("anchors offsets on the timed entries and gives the untimed one no fabricated offset", () => { + const untimed = makeGuardrailInformation(untimedPreCall); + const ran = makeGuardrailInformation(ranPostCall); + renderWithProviders(); + + const lifecycleRow = (label: string | RegExp): HTMLElement => { + const row = screen.getAllByTestId("lifecycle-row").find((r) => within(r).queryByText(label) !== null); + if (row === undefined) throw new Error(`no lifecycle row labelled ${label}`); + return row; + }; + + expect(within(lifecycleRow("Request received")).getByText("T+0ms")).toBeInTheDocument(); + expect(within(lifecycleRow(/Post-call guardrail: ran-rail/)).getByText("T+250ms")).toBeInTheDocument(); + expect(within(lifecycleRow("Response returned")).getByText("T+251ms")).toBeInTheDocument(); + + const untimedRow = within(lifecycleRow(/Pre-call guardrail: conduct/)); + expect(untimedRow.getByText("ā€”")).toBeInTheDocument(); + expect(untimedRow.queryByText(/^T\+/)).not.toBeInTheDocument(); + }); + it("calculates and displays masked entity totals", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation({ diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1de0e3878b2..58076ef6c00 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -361,7 +361,7 @@ const GenericGuardrailResponse = ({ response }: { response: any }) => { interface TimelineEntry { type: "request" | "guardrail" | "llm" | "response"; label: string; - offsetMs: number; + offsetMs: number | null; outcome?: EntryOutcome; } @@ -370,73 +370,85 @@ type TimedGuardrailInformation = GuardrailInformation & { start_time: number; en const isTimed = (e: GuardrailInformation): e is TimedGuardrailInformation => typeof e.start_time === "number" && typeof e.end_time === "number"; +const belongsOnLifecycle = (e: GuardrailInformation): boolean => isTimed(e) || getEntryOutcome(e) !== "not_run"; + +// Sorts a phase's timed entries by start time while leaving its untimed entries in the +// slots they were recorded in. Applied per phase, never globally: an entry can land in +// more than one phase bucket, so a global pass can reorder one phase by another's clock. +const orderWithinPhase = (group: GuardrailInformation[]): GuardrailInformation[] => { + const byStart = group.filter(isTimed).sort((a, b) => a.start_time - b.start_time); + const timedSlots = new Map(group.flatMap((e, i) => (isTimed(e) ? [i] : [])).map((slot, k) => [slot, byStart[k]])); + return group.map((e, i) => timedSlots.get(i) ?? e); +}; + const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { - const sorted = useMemo(() => entries.filter(isTimed).sort((a, b) => a.start_time - b.start_time), [entries]); + const sorted = useMemo(() => entries.filter(belongsOnLifecycle), [entries]); const timeline = useMemo(() => { if (sorted.length === 0) return []; - const baseTime = sorted[0].start_time; + const timed = sorted.filter(isTimed); + const baseTime = timed.length > 0 ? Math.min(...timed.map((e) => e.start_time)) : null; + const offsetOf = (e: GuardrailInformation): number | null => + baseTime === null || !isTimed(e) ? null : Math.round((e.end_time - baseTime) * 1000); const items: TimelineEntry[] = []; // Request received - items.push({ type: "request", label: "Request received", offsetMs: 0 }); + items.push({ type: "request", label: "Request received", offsetMs: baseTime === null ? null : 0 }); // Pre-call guardrails ā€” use modeMatches so array modes (e.g. ["pre_call", "post_call"]) // place the entry in every matching bucket. - const preCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call")); - const postCalls = sorted.filter( - (e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only"), + const preCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call"))); + const postCalls = orderWithinPhase( + sorted.filter((e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only")), ); - const duringCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call")); + const duringCalls = orderWithinPhase(sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call"))); for (const e of preCalls) { - const offsetMs = Math.round((e.end_time - baseTime) * 1000); items.push({ type: "guardrail", label: `Pre-call guardrail: ${getDisplayName(e)}`, - offsetMs, + offsetMs: offsetOf(e), outcome: getEntryOutcome(e), }); } // LLM call ā€” infer from gap between pre-call end and post-call start - const lastPreEnd = preCalls.length > 0 ? Math.max(...preCalls.map((e) => e.end_time)) : baseTime; - const firstPostStart = postCalls.length > 0 ? Math.min(...postCalls.map((e) => e.start_time)) : undefined; - const llmEndTime = firstPostStart ?? lastPreEnd + 1; - const llmOffsetMs = Math.round((llmEndTime - baseTime) * 1000); + const timedPre = preCalls.filter(isTimed); + const timedPost = postCalls.filter(isTimed); + const lastPreEnd = timedPre.length > 0 ? Math.max(...timedPre.map((e) => e.end_time)) : baseTime; + const firstPostStart = timedPost.length > 0 ? Math.min(...timedPost.map((e) => e.start_time)) : undefined; + const llmEndTime = firstPostStart ?? (lastPreEnd === null ? null : lastPreEnd + 1); items.push({ type: "llm", label: "LLM call", - offsetMs: llmOffsetMs, + offsetMs: llmEndTime === null || baseTime === null ? null : Math.round((llmEndTime - baseTime) * 1000), }); // During-call guardrails (rare) for (const e of duringCalls) { - const offsetMs = Math.round((e.end_time - baseTime) * 1000); items.push({ type: "guardrail", label: `During-call guardrail: ${getDisplayName(e)}`, - offsetMs, + offsetMs: offsetOf(e), outcome: getEntryOutcome(e), }); } // Post-call guardrails for (const e of postCalls) { - const offsetMs = Math.round((e.end_time - baseTime) * 1000); items.push({ type: "guardrail", label: `Post-call guardrail: ${getDisplayName(e)}`, - offsetMs, + offsetMs: offsetOf(e), outcome: getEntryOutcome(e), }); } // Response returned - const maxEnd = Math.max(...sorted.map((e) => e.end_time)); - const responseOffsetMs = Math.round((maxEnd - baseTime) * 1000) + 1; + const maxEnd = timed.length > 0 ? Math.max(...timed.map((e) => e.end_time)) : null; + const responseOffsetMs = maxEnd === null || baseTime === null ? null : Math.round((maxEnd - baseTime) * 1000) + 1; items.push({ type: "response", label: "Response returned", offsetMs: responseOffsetMs }); return items; @@ -447,7 +459,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => {

Request Lifecycle

{timeline.map((item, idx) => ( -
+
{/* Vertical line */}
@@ -475,7 +487,9 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { {OUTCOME_LABEL[item.outcome]} )} - T+{item.offsetMs}ms + + {item.offsetMs === null ? "ā€”" : `T+${item.offsetMs}ms`} +
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 27ffb905237..5138c456ff2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8544,6 +8544,45 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/teams/{team_id}/members/bulk_update": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Bulk Update Team Member Budgets Action + * @description Set per-member limits for up to 500 members of one team in one call. Same + * authorization and member addressing as `/team/member_update`: proxy admins, the team's + * admins, and admins of the team's organization, with each member named by exactly one of + * `user_id` or `user_email`. Unknown body fields are a 422 and an unknown team is a 404. + * + * Each row is a merge patch of that member's limits: a field left out is untouched, a + * field sent as null is cleared, and clearing the last limit drops the member back to the + * team default. A budget row shared by several memberships, the team default included, is + * copied for the member being patched rather than written in place, so one member's new + * cap never lands on anybody else. + * + * `data` holds one result per requested member, in request order, carrying the limits in + * force after the write. A row is `success: false` with an `error` when it names nobody on + * the team or repeats an earlier row. Roles are not part of this route; `/team/member_update` + * still owns them. + * + * Example curl: + * ``` + * curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_update' --header 'Authorization: Bearer sk-1234' --header 'Content-Type: application/json' --data '{"members": [{"user_id": "user-1", "max_budget_in_team": 10}, {"user_email": "user-2@example.com", "max_budget_in_team": 10, "budget_duration": "30d"}]}' + * ``` + */ + post: operations["bulk_update_team_member_budgets_action_management_v1_teams__team_id__members_bulk_update_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/users/bulk": { parameters: { query?: never; @@ -24930,6 +24969,22 @@ export interface components { [key: string]: unknown; } | null; }; + /** + * BulkTeamMemberBudgetUpdateRequest + * @description Body of `POST /management/v1/teams/{team_id}/members/bulk_update`. + */ + BulkTeamMemberBudgetUpdateRequest: { + /** Members */ + members: components["schemas"]["TeamMemberBudgetPatch"][]; + }; + /** + * BulkTeamMemberBudgetUpdateResponse + * @description `{data: [...]}` with one `TeamMemberBudgetUpdateResult` per requested member, in request order. + */ + BulkTeamMemberBudgetUpdateResponse: { + /** Data */ + data: components["schemas"]["TeamMemberBudgetUpdateResult"][]; + }; /** * BulkTeamMemberDeleteRequest * @description Body of `POST /management/v1/teams/{team_id}/members/bulk_delete`. @@ -37945,6 +38000,57 @@ export interface components { /** User Id */ user_id?: string | null; }; + /** + * TeamMemberBudgetPatch + * @description One member's per-member limits, merge-patch style: a field left out of the row is + * untouched, a field sent as null is cleared, and clearing the last limit drops the + * member back to the team default. + */ + TeamMemberBudgetPatch: { + /** Allowed Models */ + allowed_models?: string[] | null; + /** Budget Duration */ + budget_duration?: string | null; + /** Max Budget In Team */ + max_budget_in_team?: number | null; + /** Rpm Limit */ + rpm_limit?: number | null; + /** Tpm Limit */ + tpm_limit?: number | null; + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id?: string | null; + }; + /** + * TeamMemberBudgetUpdateResult + * @description Outcome for one requested member, in request order, carrying the limits in force + * after the write rather than the ones that were asked for. + */ + TeamMemberBudgetUpdateResult: { + /** Allowed Models */ + allowed_models?: string[] | null; + /** Budget Duration */ + budget_duration?: string | null; + /** Budget Id */ + budget_id?: string | null; + /** Error */ + error?: string | null; + /** Max Budget */ + max_budget?: number | null; + /** Max Budget Source */ + max_budget_source?: ("member" | "team_default") | null; + /** Rpm Limit */ + rpm_limit?: number | null; + /** Success */ + success: boolean; + /** Tpm Limit */ + tpm_limit?: number | null; + /** User Email */ + user_email?: string | null; + /** User Id */ + user_id?: string | null; + }; /** TeamMemberDeleteRequest */ TeamMemberDeleteRequest: { /** Team Id */ @@ -38000,7 +38106,7 @@ export interface components { }; /** * TeamMemberRef - * @description One member to remove, named by exactly one of `user_id` or `user_email`. + * @description One member, named by exactly one of `user_id` or `user_email`. */ TeamMemberRef: { /** User Email */ @@ -52113,6 +52219,41 @@ export interface operations { }; }; }; + bulk_update_team_member_budgets_action_management_v1_teams__team_id__members_bulk_update_post: { + parameters: { + query?: never; + header?: never; + path: { + team_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BulkTeamMemberBudgetUpdateRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkTeamMemberBudgetUpdateResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; bulk_create_users_route_management_v1_users_bulk_post: { parameters: { query?: never; diff --git a/uv.lock b/uv.lock index f8c7a0d7e83..35eaa20c39e 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-12T22:48:38.53978Z" +exclude-newer = "2026-09-14T20:32:38.482736111Z" exclude-newer-span = "P3D" [manifest] @@ -535,16 +535,21 @@ wheels = [ [[package]] name = "aws-sdk-bedrock-runtime" -version = "0.7.0" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-aws-core", extra = ["eventstream", "json"], marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, - { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, + { name = "smithy-http", extra = ["aiohttp"], marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/67/8a/ed3fd98775273b0b7f6006b4970aa876d506668b7fe29145f54fcb941c3b/aws_sdk_bedrock_runtime-0.7.0.tar.gz", hash = "sha256:0cb172cbc03ff060e5c1d6f9cfa9a8ac5e71d9e0d58d3117006ebf614cbb4677", size = 170304, upload-time = "2026-06-23T04:04:52.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/b3/9c225cbfe9f17ea2e3d75a0fdd0b325ef79839b9c09a376bda63a7bf3bb3/aws_sdk_bedrock_runtime-0.11.0.tar.gz", hash = "sha256:f2c45d34625bf6a7b56375e29a53a16b376880bda771e4bbf7d84491622eb193", size = 173854, upload-time = "2026-08-24T21:17:16.304Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/e1/f86d50f0ad9c8200645f315c524d285e86b30b94bb65118e1108597714e6/aws_sdk_bedrock_runtime-0.7.0-py3-none-any.whl", hash = "sha256:de67ede6f441bbb77ef61c237945d559513843fc827abe1af12535c2519650c5", size = 94948, upload-time = "2026-06-23T04:04:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/29/0c/9512304ed017ce49992df6661eac2b914550247e13bccb55be6ca594170d/aws_sdk_bedrock_runtime-0.11.0-py3-none-any.whl", hash = "sha256:ef01c26ddfd83a5d3e438ab72ebb3c13b41fc0ef11d81095b22c8016f97e9795", size = 97112, upload-time = "2026-08-24T21:17:17.396Z" }, +] + +[package.optional-dependencies] +awscrt = [ + { name = "smithy-http", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] [[package]] @@ -4483,7 +4488,7 @@ dependencies = [ [package.optional-dependencies] bedrock-realtime = [ - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12'" }, + { name = "aws-sdk-bedrock-runtime", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] caching = [ { name = "diskcache" }, @@ -4693,7 +4698,7 @@ requires-dist = [ { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, { name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.0.19,<1.0" }, - { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.7.0,<0.8.0" }, + { name = "aws-sdk-bedrock-runtime", extras = ["awscrt"], marker = "python_full_version >= '3.12' and extra == 'bedrock-realtime'", specifier = ">=0.10.0,<0.12.0" }, { name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = ">=1.0.0,<2.0" }, { name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = ">=1.25.2,<2.0" }, { name = "azure-identity", marker = "extra == 'proxy'", specifier = ">=1.25.2,<2.0" }, @@ -4884,7 +4889,7 @@ source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.98" +version = "0.4.99" source = { editable = "litellm-proxy-extras" } [[package]] @@ -9126,16 +9131,16 @@ wheels = [ [[package]] name = "smithy-aws-core" -version = "0.7.0" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sdk-signers", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, { name = "smithy-http", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/a8/37bfde59519f45d2047d0033b791aca6574d867aaf57bb56a6de42ab5c26/smithy_aws_core-0.7.0.tar.gz", hash = "sha256:34e82d09fc808acd5ffc80f03828d0609c6a211f49f0884dc6ee7ca095a1b6af", size = 15670, upload-time = "2026-06-23T04:04:50.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/d3/501c0023548173416109ac42298ca33b708469dc922005770811a597949f/smithy_aws_core-0.11.0.tar.gz", hash = "sha256:29ee89976a520a87e3db557e03e115fdc21a0a60b81161e95174395a1b064da1", size = 38791, upload-time = "2026-08-24T21:16:59.631Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/54/2d06dd9a3972a380d71bb8c3312e317aa8f1ea68dd28cffc06955ccf0220/smithy_aws_core-0.7.0-py3-none-any.whl", hash = "sha256:6c60c8fbb9431c60e80ea7f2d37e7ae48409cc1541f587fe073f202eca067e92", size = 24894, upload-time = "2026-06-23T04:04:49.349Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f6/fefda9aab809fa1a62bf7073bd6d8ab427bd9989f39b13c0d6e29d4d1045/smithy_aws_core-0.11.0-py3-none-any.whl", hash = "sha256:77cf130c22deac14a8cbeb8ccc4bcfe5a91798f4b38cb53a987080ec58c89f23", size = 58855, upload-time = "2026-08-24T21:16:58.657Z" }, ] [package.optional-dependencies] @@ -9160,41 +9165,45 @@ wheels = [ [[package]] name = "smithy-core" -version = "0.6.0" +version = "0.8.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/45/688d52c61cd4d843bb230694259e91d4c7d6954eeecbadf452a168001d45/smithy_core-0.6.0.tar.gz", hash = "sha256:ba2e5d860d716aff75004a23f53e09dfaca3e2b94f8a00c1f76dcb355b769ce0", size = 52095, upload-time = "2026-06-23T04:04:44.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/c6/93e9eea3c6163228dfe972c3e989e0553047858805ab7aa4a59f074ba129/smithy_core-0.8.1.tar.gz", hash = "sha256:3d2f8fca5960d74bd7ef380f70901c7bcdebe53f929d2d3d2fa6cb790b3f5214", size = 54259, upload-time = "2026-08-20T17:55:30.354Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/b6/06795faa9844b9667ae492e6293370393e19e7f0c2df8da1b4bf7e5f6ed9/smithy_core-0.6.0-py3-none-any.whl", hash = "sha256:51e347ed309d60ab9d36b783dbf88de614c460d51bec79d39cd403956b00f063", size = 66879, upload-time = "2026-06-23T04:04:43.596Z" }, + { url = "https://files.pythonhosted.org/packages/0c/23/c6430bbf406477fc7d16254b9908a723b299a4a21a94c99db9d12c84a8bf/smithy_core-0.8.1-py3-none-any.whl", hash = "sha256:44bd9bdf702f76919af58e44a6a1bb3dc136a745b2f955281743022ce767e347", size = 68805, upload-time = "2026-08-20T17:55:29.366Z" }, ] [[package]] name = "smithy-http" -version = "0.4.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/58/5a772d212e066d6fc1398946c4aae19bcdaa75209879d776f641b6a06b5b/smithy_http-0.4.2.tar.gz", hash = "sha256:50d11b6a55e42448450a01e3d0f605ccee65a72abf52d02eed82862a15be5937", size = 29616, upload-time = "2026-06-23T04:04:45.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/78/b5f3113d6c8f0bc1f9777a7f5ca84b892d29efac05850e14f7d4f7e645b5/smithy_http-0.5.0.tar.gz", hash = "sha256:bb4a19672f7c7eeb872a308f777eb505281a5bafb1ee3d1ea9c760c06c352510", size = 31122, upload-time = "2026-08-24T21:16:56.488Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/3e/7b2464d40893bec0b5d1f479d25116d4aa09f9f66536b4c4b3126202215d/smithy_http-0.4.2-py3-none-any.whl", hash = "sha256:a158f107e9fab925289d20772c2e38b0bba94e55c05d0edc9290310f22a60454", size = 41025, upload-time = "2026-06-23T04:04:46.764Z" }, + { url = "https://files.pythonhosted.org/packages/27/27/e414082643028846b73afa52a1a8f934548196ee12b187a06803f02a3e66/smithy_http-0.5.0-py3-none-any.whl", hash = "sha256:af273d5f42e7733ce7a6e9bd6fdd6a59ef1b61f6cd1f4a89dd53dfce99da7bef", size = 42198, upload-time = "2026-08-24T21:16:57.52Z" }, ] [package.optional-dependencies] +aiohttp = [ + { name = "aiohttp", marker = "python_full_version >= '3.12'" }, + { name = "yarl", marker = "python_full_version >= '3.12'" }, +] awscrt = [ { name = "awscrt", marker = "python_full_version >= '3.12'" }, ] [[package]] name = "smithy-json" -version = "0.2.3" +version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ijson", marker = "python_full_version >= '3.12'" }, { name = "smithy-core", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/6c/418b5687d8933b7a135d5e1a98c61fe814b98f72517dbae0e666860cb876/smithy_json-0.2.3.tar.gz", hash = "sha256:686e9b55a36dacb08e472732b358573ef78009055e05e9fce2e806d61490b2b3", size = 7805, upload-time = "2026-06-23T04:04:47.71Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/ac/04164eefb3da7479f52f6535b4b39cc8384c292cb2bb74279f2acc4f4b4d/smithy_json-0.3.0.tar.gz", hash = "sha256:c81c7034587e01bc64767cbbecb05a7d65ca9070612fd94e8a03e80540290a22", size = 7956, upload-time = "2026-08-20T17:55:32.177Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/14/eabb26b355415bcd9feef27fb5b18f1dad3fabd4208cfcbaf152025fa9ae/smithy_json-0.2.3-py3-none-any.whl", hash = "sha256:594e1bbe3d480963237f8fd0fc648dbd4e988b4503fea90157b5f07706796327", size = 10252, upload-time = "2026-06-23T04:04:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/9d/cf/0104c40a0e18fa307ea3da4310eba949f474a5bc1df3cc2b5851a72e8486/smithy_json-0.3.0-py3-none-any.whl", hash = "sha256:ffb73d2e60cf5e616e5d0a1019e7b9f518edba076cb423f10981457725dcddc4", size = 10252, upload-time = "2026-08-20T17:55:31.204Z" }, ] [[package]]