From 4630793fb0d839050b2e849703da90512f841723 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 19 Feb 2026 21:51:00 +0100 Subject: [PATCH 01/49] fix(websearch_interception): preserve thinking blocks in agentic loop follow-up messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When extended thinking is enabled, the websearch interception agentic loop builds a follow-up assistant message with only tool_use blocks. Anthropic's API requires assistant messages to start with thinking/redacted_thinking blocks when thinking is enabled, causing a 400 Bad Request. Extract thinking blocks from the model's initial response, thread them through the agentic loop, and prepend them to the follow-up assistant message — matching the pattern used by anthropic_messages_pt in factory.py. Fixes the error: "Expected 'thinking' or 'redacted_thinking', but found 'tool_use'" --- .../websearch_interception/handler.py | 48 ++- .../websearch_interception/transformation.py | 34 +- .../test_websearch_interception_thinking.py | 327 ++++++++++++++++++ 3 files changed, 401 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_interception_thinking.py diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index d7858d71eb3..bef8925e8e9 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -299,12 +299,54 @@ class WebSearchInterceptionLogger(CustomLogger): f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" ) - # Return tools dict with tool calls + # Extract thinking blocks from response content. + # When extended thinking is enabled, the model response includes + # thinking/redacted_thinking blocks that must be preserved and + # prepended to the follow-up assistant message. + thinking_blocks: List[Dict] = [] + if isinstance(response, dict): + content = response.get("content", []) + else: + content = getattr(response, "content", []) or [] + + for block in content: + if isinstance(block, dict): + block_type = block.get("type") + else: + block_type = getattr(block, "type", None) + + if block_type in ("thinking", "redacted_thinking"): + if isinstance(block, dict): + thinking_blocks.append(block) + else: + # Convert object to dict using getattr, matching the + # pattern in _detect_from_non_streaming_response + thinking_block_dict: Dict = {"type": block_type} + if block_type == "thinking": + thinking_block_dict["thinking"] = getattr( + block, "thinking", "" + ) + thinking_block_dict["signature"] = getattr( + block, "signature", "" + ) + else: # redacted_thinking + thinking_block_dict["data"] = getattr( + block, "data", "" + ) + thinking_blocks.append(thinking_block_dict) + + if thinking_blocks: + verbose_logger.debug( + f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response" + ) + + # Return tools dict with tool calls and thinking blocks tools_dict = { "tool_calls": tool_calls, "tool_type": "websearch", "provider": custom_llm_provider, "response_format": "anthropic", + "thinking_blocks": thinking_blocks, } return True, tools_dict @@ -387,6 +429,7 @@ class WebSearchInterceptionLogger(CustomLogger): """ tool_calls = tools["tool_calls"] + thinking_blocks = tools.get("thinking_blocks", []) verbose_logger.debug( f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)" @@ -396,6 +439,7 @@ class WebSearchInterceptionLogger(CustomLogger): model=model, messages=messages, tool_calls=tool_calls, + thinking_blocks=thinking_blocks, anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, logging_obj=logging_obj, stream=stream, @@ -442,6 +486,7 @@ class WebSearchInterceptionLogger(CustomLogger): model: str, messages: List[Dict], tool_calls: List[Dict], + thinking_blocks: List[Dict], anthropic_messages_optional_request_params: Dict, logging_obj: Any, stream: bool, @@ -495,6 +540,7 @@ class WebSearchInterceptionLogger(CustomLogger): assistant_message, user_message = WebSearchTransformation.transform_response( tool_calls=tool_calls, search_results=final_search_results, + thinking_blocks=thinking_blocks, ) # Make follow-up request with search results diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index e44ec35c3a2..e016899e0c3 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -4,7 +4,7 @@ WebSearch Tool Transformation Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. """ import json -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from litellm._logging import verbose_logger from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME @@ -224,6 +224,7 @@ class WebSearchTransformation: tool_calls: List[Dict], search_results: List[str], response_format: str = "anthropic", + thinking_blocks: Optional[List[Dict]] = None, ) -> Tuple[Dict, Union[Dict, List[Dict]]]: """ Transform LiteLLM search results to Anthropic/OpenAI tool_result format. @@ -235,6 +236,10 @@ class WebSearchTransformation: tool_calls: List of tool_use/tool_calls dicts from transform_request search_results: List of search result strings (one per tool_call) response_format: Response format - "anthropic" or "openai" (default: "anthropic") + thinking_blocks: Optional list of thinking/redacted_thinking blocks + from the model's response. When present, prepended to the + assistant message content (required by Anthropic API when + thinking is enabled). Returns: (assistant_message, user_or_tool_messages): @@ -247,19 +252,29 @@ class WebSearchTransformation: ) else: return WebSearchTransformation._transform_response_anthropic( - tool_calls, search_results + tool_calls, search_results, thinking_blocks=thinking_blocks ) @staticmethod def _transform_response_anthropic( tool_calls: List[Dict], search_results: List[str], + thinking_blocks: Optional[List[Dict]] = None, ) -> Tuple[Dict, Dict]: """Transform to Anthropic format (single user message with tool_result blocks)""" - # Build assistant message with tool_use blocks - assistant_message = { - "role": "assistant", - "content": [ + # Build assistant message content + assistant_content: List[Dict] = [] + + # Prepend thinking blocks if present. + # When extended thinking is enabled, Anthropic requires the assistant + # message to start with thinking/redacted_thinking blocks before any + # tool_use blocks. Same pattern as anthropic_messages_pt in factory.py. + if thinking_blocks: + assistant_content.extend(thinking_blocks) + + # Add tool_use blocks + assistant_content.extend( + [ { "type": "tool_use", "id": tc["id"], @@ -267,7 +282,12 @@ class WebSearchTransformation: "input": tc["input"], } for tc in tool_calls - ], + ] + ) + + assistant_message = { + "role": "assistant", + "content": assistant_content, } # Build user message with tool_result blocks diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_thinking.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_thinking.py new file mode 100644 index 00000000000..8093ce6fc12 --- /dev/null +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_thinking.py @@ -0,0 +1,327 @@ +""" +Unit tests for WebSearch Interception with Extended Thinking + +Tests that the websearch interception agentic loop correctly handles +thinking/redacted_thinking blocks when extended thinking is enabled. +""" + +from unittest.mock import Mock + +import pytest + +from litellm.integrations.websearch_interception.handler import ( + WebSearchInterceptionLogger, +) +from litellm.integrations.websearch_interception.transformation import ( + WebSearchTransformation, +) + + +class TestTransformResponseWithThinking: + """Tests for _transform_response_anthropic with thinking blocks.""" + + def test_thinking_blocks_prepended_to_assistant_message(self): + """Test that thinking blocks are prepended before tool_use blocks.""" + tool_calls = [ + { + "id": "toolu_01", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "latest news"}, + } + ] + search_results = [ + "Title: News\nURL: https://example.com\nSnippet: Latest news" + ] + thinking_blocks = [ + { + "type": "thinking", + "thinking": "Let me search for that.", + "signature": "sig123", + }, + {"type": "redacted_thinking", "data": "abc123"}, + ] + + assistant_msg, user_msg = ( + WebSearchTransformation._transform_response_anthropic( + tool_calls=tool_calls, + search_results=search_results, + thinking_blocks=thinking_blocks, + ) + ) + + # Verify thinking blocks come first + content = assistant_msg["content"] + assert len(content) == 3 # 2 thinking + 1 tool_use + assert content[0]["type"] == "thinking" + assert content[0]["thinking"] == "Let me search for that." + assert content[1]["type"] == "redacted_thinking" + assert content[1]["data"] == "abc123" + assert content[2]["type"] == "tool_use" + assert content[2]["id"] == "toolu_01" + + def test_no_thinking_blocks_backward_compat(self): + """Test that transform works without thinking blocks (backward compat).""" + tool_calls = [ + { + "id": "toolu_01", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "test"}, + } + ] + search_results = ["Search result text"] + + # No thinking_blocks param (default None) + assistant_msg, _ = ( + WebSearchTransformation._transform_response_anthropic( + tool_calls=tool_calls, + search_results=search_results, + ) + ) + + content = assistant_msg["content"] + assert len(content) == 1 + assert content[0]["type"] == "tool_use" + + def test_empty_thinking_blocks_list(self): + """Test that an empty thinking_blocks list behaves like None.""" + tool_calls = [ + { + "id": "toolu_01", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "test"}, + } + ] + search_results = ["Search result text"] + + assistant_msg, _ = ( + WebSearchTransformation._transform_response_anthropic( + tool_calls=tool_calls, + search_results=search_results, + thinking_blocks=[], + ) + ) + + content = assistant_msg["content"] + assert len(content) == 1 + assert content[0]["type"] == "tool_use" + + def test_transform_response_passes_thinking_to_anthropic(self): + """Test that transform_response routes thinking_blocks correctly.""" + tool_calls = [ + { + "id": "toolu_01", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "test"}, + } + ] + search_results = ["Search result"] + thinking_blocks = [ + { + "type": "thinking", + "thinking": "Reasoning here.", + "signature": "sig", + }, + ] + + assistant_msg, _ = WebSearchTransformation.transform_response( + tool_calls=tool_calls, + search_results=search_results, + response_format="anthropic", + thinking_blocks=thinking_blocks, + ) + + content = assistant_msg["content"] + assert content[0]["type"] == "thinking" + assert content[1]["type"] == "tool_use" + + def test_transform_response_openai_ignores_thinking(self): + """Test that OpenAI format is unaffected by thinking_blocks param.""" + tool_calls = [ + { + "id": "call_01", + "type": "function", + "name": "litellm_web_search", + "function": { + "name": "litellm_web_search", + "arguments": {"query": "test"}, + }, + "input": {"query": "test"}, + } + ] + search_results = ["Search result"] + thinking_blocks = [ + { + "type": "thinking", + "thinking": "Should not appear.", + "signature": "sig", + }, + ] + + assistant_msg, _ = WebSearchTransformation.transform_response( + tool_calls=tool_calls, + search_results=search_results, + response_format="openai", + thinking_blocks=thinking_blocks, + ) + + # OpenAI format uses tool_calls key, not content — thinking is irrelevant + assert "tool_calls" in assistant_msg + assert "content" not in assistant_msg + + +class TestAgenticLoopThinkingExtraction: + """Tests for thinking block extraction in async_should_run_agentic_loop.""" + + @pytest.mark.asyncio + async def test_extracts_thinking_blocks_from_dict_response(self): + """Test extraction of thinking blocks from dict-style response.""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + response = { + "content": [ + { + "type": "thinking", + "thinking": "Let me think...", + "signature": "sig1", + }, + {"type": "redacted_thinking", "data": "redacted_data"}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "litellm_web_search", + "input": {"query": "latest news"}, + }, + ] + } + + should_run, tools_dict = await logger.async_should_run_agentic_loop( + response=response, + model="bedrock/claude", + messages=[], + tools=[{"name": "WebSearch"}], + stream=False, + custom_llm_provider="bedrock", + kwargs={}, + ) + + assert should_run is True + assert len(tools_dict["tool_calls"]) == 1 + assert len(tools_dict["thinking_blocks"]) == 2 + assert tools_dict["thinking_blocks"][0]["type"] == "thinking" + assert tools_dict["thinking_blocks"][0]["thinking"] == "Let me think..." + assert tools_dict["thinking_blocks"][1]["type"] == "redacted_thinking" + assert tools_dict["thinking_blocks"][1]["data"] == "redacted_data" + + @pytest.mark.asyncio + async def test_extracts_thinking_blocks_from_object_response(self): + """Test extraction of thinking blocks from non-dict response objects. + + In practice, the Anthropic pass-through always returns plain dicts + (TypedDict(**raw_json) produces a dict). This test covers the safety + branch for non-dict response objects. + """ + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + # Simulate object-style response blocks + thinking_block = Mock() + thinking_block.type = "thinking" + thinking_block.thinking = "Reasoning..." + thinking_block.signature = "sig" + + redacted_block = Mock() + redacted_block.type = "redacted_thinking" + redacted_block.data = "abc" + + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "litellm_web_search" + tool_block.id = "toolu_01" + tool_block.input = {"query": "test"} + + response = Mock() + response.content = [thinking_block, redacted_block, tool_block] + + should_run, tools_dict = await logger.async_should_run_agentic_loop( + response=response, + model="bedrock/claude", + messages=[], + tools=[{"name": "WebSearch"}], + stream=False, + custom_llm_provider="bedrock", + kwargs={}, + ) + + assert should_run is True + assert len(tools_dict["thinking_blocks"]) == 2 + # Verify getattr-based conversion produced correct dicts + assert tools_dict["thinking_blocks"][0] == { + "type": "thinking", + "thinking": "Reasoning...", + "signature": "sig", + } + assert tools_dict["thinking_blocks"][1] == { + "type": "redacted_thinking", + "data": "abc", + } + + @pytest.mark.asyncio + async def test_no_thinking_blocks_when_thinking_disabled(self): + """Test that thinking_blocks is empty when response has no thinking.""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + response = { + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "litellm_web_search", + "input": {"query": "test"}, + }, + ] + } + + should_run, tools_dict = await logger.async_should_run_agentic_loop( + response=response, + model="bedrock/claude", + messages=[], + tools=[{"name": "WebSearch"}], + stream=False, + custom_llm_provider="bedrock", + kwargs={}, + ) + + assert should_run is True + assert tools_dict["thinking_blocks"] == [] + + @pytest.mark.asyncio + async def test_thinking_blocks_not_extracted_when_no_tool_calls(self): + """Test that no extraction happens when no websearch tool calls found.""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + response = { + "content": [ + { + "type": "thinking", + "thinking": "Just thinking...", + "signature": "sig", + }, + {"type": "text", "text": "Here is my response."}, + ] + } + + should_run, tools_dict = await logger.async_should_run_agentic_loop( + response=response, + model="bedrock/claude", + messages=[], + tools=[{"name": "WebSearch"}], + stream=False, + custom_llm_provider="bedrock", + kwargs={}, + ) + + assert should_run is False + assert tools_dict == {} From 6e850805a33caf1a8a982482d0e87e6123064e80 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 17:32:13 +0530 Subject: [PATCH 02/49] Add audio as supported openai param --- .../llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d248d2862e8..7bcefc1dd87 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -269,6 +269,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "logprobs", "top_logprobs", "modalities", + "audio", "parallel_tool_calls", "web_search_options", ] From 4643685e7838ce0c8fd6c0ed1f12a2a8a34f9e86 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 17:55:26 -0800 Subject: [PATCH 03/49] [Fix] /key/aliases: Add pagination and search to prevent OOMs The /key/aliases endpoint previously fetched all key aliases from the database without limit, causing OOM crashes with large key sets. Added page, size, and search query parameters with database-level filtering to enable paginated and searchable key alias retrieval. Updated the response to include pagination metadata (total_count, current_page, total_pages, size) matching the /v2/model/info pattern. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 59 ++++++++++++------- .../test_key_generate_prisma.py | 34 +++++++---- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c1165ab26d0..8a5145bf793 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4107,13 +4107,23 @@ async def list_keys( dependencies=[Depends(user_api_key_auth)], ) @management_endpoint_wrapper -async def key_aliases() -> Dict[str, List[str]]: +async def key_aliases( + page: int = Query(1, ge=1, description="Page number"), + size: int = Query(50, ge=1, le=100, description="Page size"), + search: Optional[str] = Query( + None, description="Search key aliases (case-insensitive partial match)" + ), +) -> Dict[str, Any]: """ - Lists all key aliases + Lists key aliases with pagination and optional search. Returns: { - "aliases": List[str] + "aliases": List[str], + "total_count": int, + "current_page": int, + "total_pages": int, + "size": int, } """ try: @@ -4125,36 +4135,43 @@ async def key_aliases() -> Dict[str, List[str]]: verbose_proxy_logger.error("Database not connected") raise Exception("Database not connected") - where: Dict[str, Any] = {} + conditions: List[Dict[str, Any]] = [{"key_alias": {"not": None}}] try: - where.update(_get_condition_to_filter_out_ui_session_tokens()) + conditions.append(_get_condition_to_filter_out_ui_session_tokens()) except NameError: # Helper may not exist in some builds; ignore if missing pass + if search: + conditions.append( + {"key_alias": {"contains": search, "mode": "insensitive"}} + ) + where: Dict[str, Any] = {"AND": conditions} + total_count = await prisma_client.db.litellm_verificationtoken.count( + where=where, + ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, order=[{"key_alias": "asc"}], + skip=(page - 1) * size, + take=size, ) - seen = set() - aliases: List[str] = [] - for row in rows: - alias = getattr(row, "key_alias", None) - if alias is None and isinstance(row, dict): - alias = row.get("key_alias") + aliases: List[str] = [row.key_alias for row in rows if row.key_alias] # type: ignore[misc] - if not alias: - continue + total_pages = -(-total_count // size) if total_count > 0 else 0 + verbose_proxy_logger.debug( + f"key_aliases: page={page}, size={size}, search={search!r}, " + f"total_count={total_count}, total_pages={total_pages}" + ) - alias_str = str(alias).strip() - if alias_str and alias_str not in seen: - seen.add(alias_str) - aliases.append(alias_str) - - verbose_proxy_logger.debug(f"Returning {len(aliases)} key aliases") - - return {"aliases": aliases} + return { + "aliases": aliases, + "total_count": total_count, + "current_page": page, + "total_pages": total_pages, + "size": size, + } except Exception as e: verbose_proxy_logger.exception(f"Error in key_aliases: {e}") diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index c3f68762810..ed528f21e0d 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3668,9 +3668,10 @@ async def test_list_keys(prisma_client): async def test_key_aliases(prisma_client): """ Test the key_aliases function: - - Returns a list + - Returns a paginated response - Includes alias from a newly created key - - Aliases are unique and sorted + - Aliases are sorted + - Pagination and search params work correctly """ import asyncio import uuid @@ -3682,10 +3683,16 @@ async def test_key_aliases(prisma_client): setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") await litellm.proxy.proxy_server.prisma_client.connect() - # Basic call - response = await key_aliases() + # Basic call - check pagination response shape + response = await key_aliases(page=1, size=50) assert "aliases" in response assert isinstance(response["aliases"], list) + assert "total_count" in response + assert "current_page" in response + assert "total_pages" in response + assert "size" in response + assert response["current_page"] == 1 + assert response["size"] == 50 # Create a new user (and key) with a unique alias unique_id = str(uuid.uuid4()) @@ -3704,17 +3711,22 @@ async def test_key_aliases(prisma_client): # Allow async DB writes to settle await asyncio.sleep(2) - # Call again and validate - response_after = await key_aliases() + # Call again and validate alias is present + response_after = await key_aliases(page=1, size=50) aliases = response_after["aliases"] - - # Contains the new alias assert test_alias in aliases - - # Unique & sorted (endpoint dedupes and orders ascending) - assert len(aliases) == len(set(aliases)) assert aliases == sorted(aliases) + # Search by partial alias + partial = test_alias[:10] + search_response = await key_aliases(page=1, size=50, search=partial) + assert test_alias in search_response["aliases"] + + # Search with no match + no_match_response = await key_aliases(page=1, size=50, search="__no_match_xyz__") + assert len(no_match_response["aliases"]) == 0 + assert no_match_response["total_count"] == 0 + @pytest.mark.skip(reason="Requires reliable external DB connection (prisma).") @pytest.mark.asyncio From 48f549b30bd096cfecbd66bf7349f4366aeec9b8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 09:56:44 +0530 Subject: [PATCH 04/49] Return Clear error message why no tools are available / IP Filtering occured --- .../mcp_server/mcp_server_manager.py | 30 +++++++--- .../mcp_server/rest_endpoints.py | 41 +++++++++++++- .../proxy/_experimental/mcp_server/server.py | 14 ++++- .../proxy/auth/test_mcp_ip_filtering.py | 55 +++++++++++++++++++ 4 files changed, 128 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7e90c64efdc..7484de33ce4 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -756,14 +756,30 @@ class MCPServerManager: Returns server_ids unchanged when client_ip is None (no filtering). """ + filtered, _ = self.filter_server_ids_by_ip_with_info(server_ids, client_ip) + return filtered + + def filter_server_ids_by_ip_with_info( + self, server_ids: List[str], client_ip: Optional[str] + ) -> Tuple[List[str], int]: + """ + Filter server IDs by client IP — external callers only see public servers. + + Returns (filtered_ids, ip_blocked_count) where ip_blocked_count is the number + of servers that were blocked because the client IP is not allowed to access them. + Returns server_ids unchanged (with 0 blocked) when client_ip is None. + """ if client_ip is None: - return server_ids - return [ - sid - for sid in server_ids - if (s := self.get_mcp_server_by_id(sid)) is not None - and self._is_server_accessible_from_ip(s, client_ip) - ] + return server_ids, 0 + allowed = [] + blocked = 0 + for sid in server_ids: + s = self.get_mcp_server_by_id(sid) + if s is not None and self._is_server_accessible_from_ip(s, client_ip): + allowed.append(sid) + elif s is not None: + blocked += 1 + return allowed, blocked async def get_tools_for_server(self, server_id: str) -> List[MCPTool]: """ diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 6e1a252be73..16f8f835430 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -10,9 +10,9 @@ from litellm.proxy._experimental.mcp_server.ui_session_utils import ( ) from litellm.proxy._experimental.mcp_server.utils import merge_mcp_headers from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers from litellm.types.mcp import MCPAuth from litellm.types.utils import CallTypes @@ -283,8 +283,10 @@ if MCP_AVAILABLE: ) allowed_server_ids_set.update(servers) - allowed_server_ids = global_mcp_server_manager.filter_server_ids_by_ip( - list(allowed_server_ids_set), _rest_client_ip + allowed_server_ids, _ip_blocked_count = ( + global_mcp_server_manager.filter_server_ids_by_ip_with_info( + list(allowed_server_ids_set), _rest_client_ip + ) ) list_tools_result = [] @@ -293,6 +295,26 @@ if MCP_AVAILABLE: # If server_id is specified, only query that specific server if server_id: if server_id not in allowed_server_ids: + _server = global_mcp_server_manager.get_mcp_server_by_id(server_id) + if ( + _server is not None + and _rest_client_ip is not None + and not global_mcp_server_manager._is_server_accessible_from_ip( + _server, _rest_client_ip + ) + ): + raise HTTPException( + status_code=403, + detail={ + "error": "ip_filtering", + "message": ( + f"MCP server '{server_id}' is not accessible from your IP address " + f"({_rest_client_ip}). This server is restricted to internal " + "networks only. To make it externally accessible, set " + "'available_on_public_internet: true' in the server configuration." + ), + }, + ) raise HTTPException( status_code=403, detail={ @@ -330,6 +352,19 @@ if MCP_AVAILABLE: } else: if not allowed_server_ids: + if _ip_blocked_count > 0: + raise HTTPException( + status_code=403, + detail={ + "error": "ip_filtering", + "message": ( + f"No MCP tools are available for your IP address ({_rest_client_ip}). " + f"{_ip_blocked_count} server(s) are restricted to internal networks only. " + "To make servers externally accessible, set " + "'available_on_public_internet: true' in the server configuration." + ), + }, + ) raise HTTPException( status_code=403, detail={ diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index e8877b4fff7..48c837c1e4f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -771,8 +771,8 @@ if MCP_AVAILABLE: user_api_key_auth ) ) - allowed_mcp_server_ids = ( - global_mcp_server_manager.filter_server_ids_by_ip( + allowed_mcp_server_ids, _ip_blocked = ( + global_mcp_server_manager.filter_server_ids_by_ip_with_info( allowed_mcp_server_ids, client_ip ) ) @@ -780,6 +780,16 @@ if MCP_AVAILABLE: "MCP IP filter: client_ip=%s, allowed_server_ids=%s", client_ip, allowed_mcp_server_ids, ) + if _ip_blocked > 0: + verbose_logger.debug( + "MCP IP filtering: %d server(s) are not accessible from client IP %s " + "because they are restricted to internal networks. " + "No tools from those servers will be returned. " + "To expose a server externally, set 'available_on_public_internet: true' " + "in its configuration.", + _ip_blocked, + client_ip, + ) allowed_mcp_servers: List[MCPServer] = [] for allowed_mcp_server_id in allowed_mcp_server_ids: mcp_server = global_mcp_server_manager.get_mcp_server_by_id( diff --git a/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py b/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py index 50e51fbe035..3b13ef3641f 100644 --- a/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py +++ b/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py @@ -91,3 +91,58 @@ class TestMCPServerIPFiltering: result = manager.filter_server_ids_by_ip(["priv"], client_ip=None) assert result == ["priv"] + + +class TestFilterServerIdsByIpWithInfo: + """Tests that filter_server_ids_by_ip_with_info returns accurate block counts.""" + + @patch("litellm.public_mcp_servers", []) + @patch("litellm.proxy.proxy_server.general_settings", {}) + def test_external_ip_reports_blocked_count(self): + pub = _make_server("pub", available_on_public_internet=True) + priv = _make_server("priv", available_on_public_internet=False) + manager = _make_manager([pub, priv]) + + allowed, blocked = manager.filter_server_ids_by_ip_with_info( + ["pub", "priv"], client_ip="8.8.8.8" + ) + assert allowed == ["pub"] + assert blocked == 1 + + @patch("litellm.public_mcp_servers", []) + @patch("litellm.proxy.proxy_server.general_settings", {}) + def test_internal_ip_reports_zero_blocked(self): + pub = _make_server("pub", available_on_public_internet=True) + priv = _make_server("priv", available_on_public_internet=False) + manager = _make_manager([pub, priv]) + + allowed, blocked = manager.filter_server_ids_by_ip_with_info( + ["pub", "priv"], client_ip="192.168.1.1" + ) + assert allowed == ["pub", "priv"] + assert blocked == 0 + + @patch("litellm.public_mcp_servers", []) + @patch("litellm.proxy.proxy_server.general_settings", {}) + def test_no_ip_returns_all_with_zero_blocked(self): + priv = _make_server("priv", available_on_public_internet=False) + manager = _make_manager([priv]) + + allowed, blocked = manager.filter_server_ids_by_ip_with_info( + ["priv"], client_ip=None + ) + assert allowed == ["priv"] + assert blocked == 0 + + @patch("litellm.public_mcp_servers", []) + @patch("litellm.proxy.proxy_server.general_settings", {}) + def test_all_private_external_ip_reports_all_blocked(self): + priv1 = _make_server("priv1", available_on_public_internet=False) + priv2 = _make_server("priv2", available_on_public_internet=False) + manager = _make_manager([priv1, priv2]) + + allowed, blocked = manager.filter_server_ids_by_ip_with_info( + ["priv1", "priv2"], client_ip="1.2.3.4" + ) + assert allowed == [] + assert blocked == 2 From 691927f9c92b44351c87b01db832bdd78f7b2c74 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 09:59:37 +0530 Subject: [PATCH 05/49] fix(embeddings): allow dimensions param passthrough via allowed_openai_params for non-text-embedding-3 OpenAI models When calling non-text-embedding-3 models routed through the openai provider (e.g. nvidia/llama-3.2-nv-embedqa-1b-v2), passing `dimensions` previously raised an UnsupportedParamsError unconditionally. This fix threads `allowed_openai_params` through the embedding call stack so that providers can opt-in to passing `dimensions` by including it in the list. Co-Authored-By: Claude Sonnet 4.6 --- litellm/main.py | 4 ++ litellm/utils.py | 5 ++- .../test_get_optional_params_embeddings.py | 37 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 8b239c454f4..cb3ddc2f401 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4680,12 +4680,16 @@ def embedding( # noqa: PLR0915 if dynamic_api_key is not None: api_key = dynamic_api_key + allowed_openai_params: Optional[List[str]] = kwargs.get( + "allowed_openai_params", None + ) optional_params = get_optional_params_embeddings( model=model, user=user, dimensions=dimensions, encoding_format=encoding_format, custom_llm_provider=custom_llm_provider, + allowed_openai_params=allowed_openai_params, **non_default_params, ) diff --git a/litellm/utils.py b/litellm/utils.py index 7ac828aefc1..68ca29d9403 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3117,6 +3117,7 @@ def get_optional_params_embeddings( # noqa: PLR0915 custom_llm_provider="", drop_params: Optional[bool] = None, additional_drop_params: Optional[List[str]] = None, + allowed_openai_params: Optional[List[str]] = None, **kwargs, ): # Lazy load get_supported_openai_params @@ -3131,6 +3132,7 @@ def get_optional_params_embeddings( # noqa: PLR0915 drop_params = passed_params.pop("drop_params", None) additional_drop_params = passed_params.pop("additional_drop_params", None) + allowed_openai_params = passed_params.pop("allowed_openai_params", None) or [] # Remove function objects from passed_params to avoid JSON serialization errors passed_params.pop("get_supported_openai_params", None) @@ -3188,11 +3190,12 @@ def get_optional_params_embeddings( # noqa: PLR0915 ## raise exception if non-default value passed for non-openai/azure embedding calls elif custom_llm_provider == "openai": # 'dimensions` is only supported in `text-embedding-3` and later models - + verbose_logger.debug(f"allowed_openai_params: {allowed_openai_params}") if ( model is not None and "text-embedding-3" not in model and "dimensions" in non_default_params.keys() + and "dimensions" not in (allowed_openai_params or []) ): raise UnsupportedParamsError( status_code=500, diff --git a/tests/local_testing/test_get_optional_params_embeddings.py b/tests/local_testing/test_get_optional_params_embeddings.py index 055be487551..8a94c8f4682 100644 --- a/tests/local_testing/test_get_optional_params_embeddings.py +++ b/tests/local_testing/test_get_optional_params_embeddings.py @@ -69,3 +69,40 @@ def test_bedrock_embed_v2_with_drop_params(): ) print(f"received optional_params: {optional_params}") assert optional_params == {"dimensions": 512, "embeddingTypes": ["binary"]} + + +def test_openai_non_text_embedding_3_with_allowed_openai_params(): + """ + Test that `dimensions` is allowed for non-text-embedding-3 OpenAI models + when `allowed_openai_params=["dimensions"]` is passed. Without this flag, + an UnsupportedParamsError would be raised. + """ + model, custom_llm_provider, _, _ = get_llm_provider( + model="openai/nvidia/llama-3.2-nv-embedqa-1b-v2" + ) + optional_params = get_optional_params_embeddings( + model=model, + dimensions=1024, + custom_llm_provider=custom_llm_provider, + allowed_openai_params=["dimensions"], + ) + print(f"received optional_params: {optional_params}") + assert optional_params.get("dimensions") == 1024 + + +def test_openai_non_text_embedding_3_without_allowed_openai_params_raises(): + """ + Test that passing `dimensions` to a non-text-embedding-3 OpenAI model + without `allowed_openai_params` still raises UnsupportedParamsError. + """ + from litellm.exceptions import UnsupportedParamsError + + model, custom_llm_provider, _, _ = get_llm_provider( + model="openai/nvidia/llama-3.2-nv-embedqa-1b-v2" + ) + with pytest.raises(UnsupportedParamsError): + get_optional_params_embeddings( + model=model, + dimensions=1024, + custom_llm_provider=custom_llm_provider, + ) From 3634b5fda09c18681a229929f5c0b77529cdbd9b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:02:29 +0530 Subject: [PATCH 06/49] Remove logger --- litellm/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 68ca29d9403..81e772b1765 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3190,7 +3190,6 @@ def get_optional_params_embeddings( # noqa: PLR0915 ## raise exception if non-default value passed for non-openai/azure embedding calls elif custom_llm_provider == "openai": # 'dimensions` is only supported in `text-embedding-3` and later models - verbose_logger.debug(f"allowed_openai_params: {allowed_openai_params}") if ( model is not None and "text-embedding-3" not in model From 0b0809a3d571ed2b1f713dcf5340fe826d9ec76c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:03:15 -0800 Subject: [PATCH 07/49] optimize key_aliases to select only key_alias column and add unit tests Add select={"key_alias": True} to the find_many call so only the alias column is fetched from the database instead of full token rows. Add five unit tests in test_key_management_endpoints.py covering response shape, pagination skip/take computation, search filter injection, absence of contains filter when no search term is given, and the select-only-alias optimization. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 1 + .../test_key_management_endpoints.py | 98 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8a5145bf793..8ac6f282719 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4152,6 +4152,7 @@ async def key_aliases( ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, + select={"key_alias": True}, order=[{"key_alias": "asc"}], skip=(page - 1) * size, take=size, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 05df3c2dcbb..8580c89fdaa 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -45,6 +45,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( check_team_key_model_specific_limits, delete_verification_tokens, generate_key_helper_fn, + key_aliases, list_keys, prepare_key_update_data, reset_key_spend_fn, @@ -6211,3 +6212,100 @@ async def test_generate_key_helper_fn_agent_id(): assert key_data.get("agent_id") == "test-agent-456", ( f"Expected agent_id='test-agent-456' in key_data, got: {key_data.get('agent_id')}" ) + + +@pytest.mark.asyncio +async def test_key_aliases_response_shape(): + """Test that key_aliases returns the correct paginated response shape.""" + mock_row1 = MagicMock() + mock_row1.key_alias = "alias-alpha" + mock_row2 = MagicMock() + mock_row2.key_alias = "alias-beta" + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=2) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[mock_row1, mock_row2] + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + result = await key_aliases(page=1, size=50, search=None) + + assert result["aliases"] == ["alias-alpha", "alias-beta"] + assert result["total_count"] == 2 + assert result["current_page"] == 1 + assert result["total_pages"] == 1 + assert result["size"] == 50 + + # Both count and find_many must use the same where clause + count_where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + find_where = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] + assert count_where == find_where + + # Non-null alias filter must be present + assert json.dumps({"key_alias": {"not": None}}) in json.dumps(count_where) + + +@pytest.mark.asyncio +async def test_key_aliases_pagination_skip_take(): + """Test that skip and take are correctly computed from page and size.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=120) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + result = await key_aliases(page=3, size=25, search=None) + + assert result["current_page"] == 3 + assert result["size"] == 25 + assert result["total_count"] == 120 + assert result["total_pages"] == 5 # ceil(120 / 25) + + find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs + assert find_many_kwargs["skip"] == 50 # (3 - 1) * 25 + assert find_many_kwargs["take"] == 25 + + +@pytest.mark.asyncio +async def test_key_aliases_search_filter(): + """Test that the search param adds a case-insensitive contains condition.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search="my-key") + + where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + assert ( + json.dumps({"key_alias": {"contains": "my-key", "mode": "insensitive"}}) + in json.dumps(where) + ) + + +@pytest.mark.asyncio +async def test_key_aliases_no_search_omits_contains_filter(): + """Test that without a search term no contains condition is added.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search=None) + + where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + assert "contains" not in json.dumps(where) + + +@pytest.mark.asyncio +async def test_key_aliases_select_only_key_alias(): + """Test that find_many is called with select={key_alias: True} to avoid fetching full rows.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search=None) + + find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs + assert find_many_kwargs.get("select") == {"key_alias": True} From 33268934200d998010f92ed571ce5c48796316e6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:42:01 +0530 Subject: [PATCH 08/49] Fix mypy issues --- litellm/litellm_core_utils/realtime_streaming.py | 8 ++++---- .../amazon_qwen2_transformation.py | 15 +++++++++------ .../amazon_qwen3_transformation.py | 15 +++++++++------ .../mcp_server/auth/user_api_key_auth_mcp.py | 4 ++-- litellm/proxy/db/db_spend_update_writer.py | 2 +- .../usage_endpoints/ai_usage_chat.py | 8 ++++---- litellm/router.py | 2 +- 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d15d23f8eea..ee028a74f48 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,7 +1,7 @@ import asyncio import concurrent.futures import json -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast import litellm from litellm._logging import verbose_logger @@ -88,7 +88,7 @@ class RealTimeStreaming: message_obj = message else: message_obj = json.loads(message) - self._collect_tool_calls_from_response_done(message_obj) + self._collect_tool_calls_from_response_done(cast(dict, message_obj)) try: if ( not isinstance(message, dict) @@ -355,11 +355,11 @@ class RealTimeStreaming: == "conversation.item.input_audio_transcription.completed" ): transcript = event.get("transcript", "") - self._collect_user_input_from_backend_event(event) + self._collect_user_input_from_backend_event(cast(dict, event)) self.store_message(event_str) await self.websocket.send_text(event_str) blocked = await self.run_realtime_guardrails( - transcript, item_id=event.get("item_id") + cast(str, transcript), item_id=cast(Optional[str], event.get("item_id")) ) if not blocked: await self.backend_ws.send( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index 2abcc679eef..0260eeafe63 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -11,7 +11,6 @@ from typing import Any, List, Optional import httpx -from litellm.types.utils import Usage from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( AmazonQwen3Config, ) @@ -19,7 +18,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation LiteLLMLoggingObj, ) from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class AmazonQwen2Config(AmazonQwen3Config): @@ -80,10 +79,14 @@ class AmazonQwen2Config(AmazonQwen3Config): # Set usage information if available in response if "usage" in response_data: usage_data = response_data["usage"] - model_response.usage = Usage( - prompt_tokens=usage_data.get("prompt_tokens", 0), - completion_tokens=usage_data.get("completion_tokens", 0), - total_tokens=usage_data.get("total_tokens", 0), + setattr( + model_response, + "usage", + Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ), ) return model_response diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index 12333623f51..6eddcccd631 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -10,14 +10,13 @@ from typing import Any, List, Optional import httpx -from litellm.types.utils import Usage from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, LiteLLMLoggingObj, ) from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): @@ -202,10 +201,14 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): # Set usage information if available in response if "usage" in response_data: usage_data = response_data["usage"] - model_response.usage = Usage( - prompt_tokens=usage_data.get("prompt_tokens", 0), - completion_tokens=usage_data.get("completion_tokens", 0), - total_tokens=usage_data.get("total_tokens", 0), + setattr( + model_response, + "usage", + Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ), ) return model_response diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 860569d24cb..6e78458cc0e 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple, cast from fastapi import HTTPException from starlette.datastructures import Headers @@ -539,7 +539,7 @@ class MCPRequestHandler: allowed_tools = team_tools else: # No team restrictions → use key restrictions - allowed_tools = key_tools + allowed_tools = cast(List[str], key_tools) # Intersect with agent's tool permissions if agent_id is set if user_api_key_auth.agent_id: diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 37fac8b56ed..0c25424ceaa 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -312,7 +312,7 @@ class DBSpendUpdateWriter: prisma_client: Optional[PrismaClient], user_api_key_cache: DualCache, litellm_proxy_budget_name: Optional[str], - payload_copy: dict, + payload_copy: SpendLogsPayload, request_tags: Optional[Any], ): """ diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index f156be7d2cc..56c3c5f0477 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -5,7 +5,7 @@ usage/spend data by querying the aggregated daily activity endpoints. import json from datetime import date -from typing import Any, AsyncIterator, Callable, Dict, List, Literal, Optional +from typing import Any, AsyncIterator, Callable, Dict, List, Literal, Optional, cast import litellm from litellm._logging import verbose_proxy_logger @@ -492,17 +492,17 @@ async def _process_tool_call( "tool_label": handler["label"], "arguments": fn_args, } - yield _sse({**tool_event_base, "status": "running"}) + yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"})) try: tool_result = await _execute_tool_call( handler, fn_name, fn_args, user_id, is_admin ) - yield _sse({**tool_event_base, "status": "complete"}) + yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"})) except Exception as e: verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e) tool_result = f"Error fetching {handler['label']}. Please try again." - yield _sse({**tool_event_base, "status": "error"}) + yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"})) chat_messages.append( {"role": "tool", "tool_call_id": tc.id, "content": tool_result} diff --git a/litellm/router.py b/litellm/router.py index 3a6c514989d..cbe5b414040 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7053,7 +7053,7 @@ class Router: user_model_info = deployment.get("model_info") or {} if model_info is not None: - model_info.update(user_model_info) + model_info.update(cast(ModelInfo, user_model_info)) return model_info From f68cbc4c9588bc04cbff4f2ee4e04f23c82dd690 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:43:05 +0530 Subject: [PATCH 09/49] Fix test_perform_health_check_filters_by_model_id --- tests/litellm_utils_tests/test_health_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index b459c3cfc99..963fe4f5b9f 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -497,7 +497,7 @@ async def test_perform_health_check_filters_by_model_id(): captured_list = [] - async def mock_perform_health_check(m_list, details=True): + async def mock_perform_health_check(m_list, details=True, **kwargs): captured_list.append(m_list) return [{"model": "gpt-4", "api_key": m_list[0]["litellm_params"]["api_key"]}], [] From 552d9aa7929206b9e071385ce6a10f577890b252 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:43:39 +0530 Subject: [PATCH 10/49] Fix code qa for _types.py --- litellm/proxy/_types.py | 74 +++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 88cfa5f6c11..de1609baf62 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1,40 +1,59 @@ import enum import json from datetime import datetime -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, - Optional, Union) +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union import httpx -from pydantic import (BaseModel, ConfigDict, Field, Json, field_validator, - model_validator) +from pydantic import ( + BaseModel, + ConfigDict, + Field, + Json, + field_validator, + model_validator, +) from typing_extensions import Required, TypedDict from litellm._uuid import uuid from litellm.types.integrations.slack_alerting import AlertType -from litellm.types.llms.openai import (AllMessageValues, OpenAIFileObject, - ResponsesAPIResponse) -from litellm.types.mcp import (MCPAuth, MCPAuthType, MCPCredentials, - MCPTransport, MCPTransportType) +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIFileObject, + ResponsesAPIResponse, +) +from litellm.types.mcp import ( + MCPAuthType, + MCPCredentials, + MCPTransport, + MCPTransportType, +) from litellm.types.mcp_server.mcp_server_manager import MCPInfo from litellm.types.router import RouterErrors, UpdateRouterConfig from litellm.types.secret_managers.main import KeyManagementSystem -from litellm.types.utils import (CallTypes, CostBreakdown, EmbeddingResponse, - GenericBudgetConfigType, ImageResponse, - LiteLLMBatch, LiteLLMFineTuningJob, - LiteLLMPydanticObjectBase, ModelResponse, - ProviderField, StandardCallbackDynamicParams, - StandardLoggingGuardrailInformation, - StandardLoggingMCPToolCall, - StandardLoggingModelInformation, - StandardLoggingPayloadErrorInformation, - StandardLoggingPayloadStatus, - StandardLoggingVectorStoreRequest, - StandardPassThroughResponseObject, - TextCompletionResponse) +from litellm.types.utils import ( + CallTypes, + CostBreakdown, + EmbeddingResponse, + GenericBudgetConfigType, + ImageResponse, + LiteLLMBatch, + LiteLLMFineTuningJob, + LiteLLMPydanticObjectBase, + ModelResponse, + ProviderField, + StandardCallbackDynamicParams, + StandardLoggingGuardrailInformation, + StandardLoggingMCPToolCall, + StandardLoggingModelInformation, + StandardLoggingPayloadErrorInformation, + StandardLoggingPayloadStatus, + StandardLoggingVectorStoreRequest, + StandardPassThroughResponseObject, + TextCompletionResponse, +) from litellm.types.videos.main import VideoObject -from .types_utils.utils import (get_instance_fn, - validate_custom_validate_return_type) +from .types_utils.utils import get_instance_fn, validate_custom_validate_return_type if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -2349,8 +2368,7 @@ class UserAPIKeyAuth( This is used to track number of requests/spend for health check calls. """ - from litellm.constants import \ - LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME return cls( api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, @@ -2382,8 +2400,7 @@ class UserAPIKeyAuth( This is used to track actions performed by automated system jobs. """ - from litellm.constants import \ - LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME return cls( api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, @@ -2774,8 +2791,7 @@ class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): @model_validator(mode="after") def mask_api_keys(self): - from litellm.litellm_core_utils.sensitive_data_masker import \ - SensitiveDataMasker + from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker masker = SensitiveDataMasker(sensitive_patterns={"key"}) From 8f439c96eede12e01d868255def8a1c0f5615e9f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:43:59 +0530 Subject: [PATCH 11/49] Fix UI build --- .../src/components/ToolPolicies.tsx | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/ui/litellm-dashboard/src/components/ToolPolicies.tsx b/ui/litellm-dashboard/src/components/ToolPolicies.tsx index 82785496eae..860093ceadb 100644 --- a/ui/litellm-dashboard/src/components/ToolPolicies.tsx +++ b/ui/litellm-dashboard/src/components/ToolPolicies.tsx @@ -2,19 +2,7 @@ import React, { useCallback, useDeferredValue, useEffect, useState } from "react"; import { Select, Switch, Tooltip } from "antd"; -<<<<<<< cursor/development-environment-setup-13a7 -// @ts-ignore - duplicate import removed -import { - Table, - TableHead, - TableHeaderCell, - TableBody, - TableRow, - TableCell, -} from "@tremor/react"; -======= import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; ->>>>>>> main import { TimeCell } from "./view_logs/time_cell"; import { TableHeaderSortDropdown } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; import type { SortState } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; @@ -60,21 +48,6 @@ const PolicySelect: React.FC<{ minWidth: 110, fontWeight: 500, }} -<<<<<<< cursor/development-environment-setup-13a7 - {...{styles: { - selector: { - backgroundColor: style.bg, - borderColor: style.border, - color: style.color, - borderRadius: 999, - fontSize: 11, - fontWeight: 600, - paddingLeft: 8, - paddingRight: 4, - }, - }} as any} -======= ->>>>>>> main popupMatchSelectWidth={false} options={POLICY_OPTIONS.map((o) => ({ value: o.value, From 828ce40eacba21e1f1d3693c3f88d5c075966a9e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:46:01 +0530 Subject: [PATCH 12/49] Fix test_async_gcs_pub_sub_v1 --- .../management_endpoints/usage_endpoints/ai_usage_chat.py | 4 ++-- tests/logging_callback_tests/test_gcs_pub_sub.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index 56c3c5f0477..4de29e04092 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -7,6 +7,8 @@ import json from datetime import date from typing import Any, AsyncIterator, Callable, Dict, List, Literal, Optional, cast +from typing_extensions import TypedDict + import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL @@ -14,8 +16,6 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) -from typing_extensions import TypedDict - # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 8ffbc8eedd5..540fb59ab01 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -36,6 +36,7 @@ ignored_keys = [ "endTime", "completionStartTime", "endTime", + "request_duration_ms", "metadata.model_map_information", "metadata.usage_object", "metadata.cold_storage_object_key", From 827bb2f535d7339a8e233c9dcc8d42a66c107059 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:48:52 +0530 Subject: [PATCH 13/49] Fix: test_sentence[te_6] --- .../sg_mas_transparency_explainability.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml index 0a08abd86e8..4b3555b49df 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/sg_mas_transparency_explainability.yaml @@ -69,10 +69,12 @@ always_block_keywords: severity: "high" exceptions: - - "explainability" + - "improve explainability" + - "add explainability" - "interpretability" - "model card" - - "audit trail" + - "with audit trail" + - "add audit trail" - "explain what" - "explain how" - "what is" From 2349b51d80ebc1fdeeebac38eea2b7323d835cfc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:50:05 +0530 Subject: [PATCH 14/49] Fix test_pass_through_request_logging_failure --- tests/pass_through_unit_tests/test_pass_through_unit_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 0e681cb1e02..f529fe85ab8 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -68,6 +68,8 @@ def mock_request(): self.request_body = request_body or {} # Add url attribute that the actual code expects self.url = "http://localhost:8000/test" + # Add state attribute that FastAPI requests have + self.state = type("State", (), {})() async def body(self) -> bytes: return bytes(json.dumps(self.request_body), "utf-8") From f8e0b377691a118721df5cc3f24449305ae2a341 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 10:54:17 +0530 Subject: [PATCH 15/49] FIx test_read_request* --- litellm/proxy/common_utils/http_parsing_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 8d179a9caed..04d46ecaeb8 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -143,7 +143,8 @@ def _safe_get_request_headers(request: Optional[Request]) -> dict: """ if request is None: return {} - cached = getattr(request.state, "_cached_headers", None) + state = getattr(request, "state", None) + cached = getattr(state, "_cached_headers", None) if cached is not None: return cached try: @@ -154,7 +155,8 @@ def _safe_get_request_headers(request: Optional[Request]) -> dict: ) headers = {} try: - request.state._cached_headers = headers + if state is not None: + state._cached_headers = headers except Exception: pass # request.state may not be available in all contexts return headers From 06e87eea877f866af9aa3d32839cac9ffd491595 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:40:56 -0800 Subject: [PATCH 16/49] remove unsupported select param from find_many call LiteLLM_VerificationTokenActions.find_many() does not support the select keyword argument. Remove it and drop the corresponding test. Co-Authored-By: Claude Haiku 4.5 --- .../management_endpoints/key_management_endpoints.py | 1 - .../test_key_management_endpoints.py | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8ac6f282719..8a5145bf793 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4152,7 +4152,6 @@ async def key_aliases( ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, - select={"key_alias": True}, order=[{"key_alias": "asc"}], skip=(page - 1) * size, take=size, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8580c89fdaa..bdfc6734664 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6297,15 +6297,3 @@ async def test_key_aliases_no_search_omits_contains_filter(): assert "contains" not in json.dumps(where) -@pytest.mark.asyncio -async def test_key_aliases_select_only_key_alias(): - """Test that find_many is called with select={key_alias: True} to avoid fetching full rows.""" - mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - - with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): - await key_aliases(page=1, size=50, search=None) - - find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs - assert find_many_kwargs.get("select") == {"key_alias": True} From 386c148b8db369a5c2661a17dca1e4065fb661b8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:48:53 -0800 Subject: [PATCH 17/49] use raw SQL in key_aliases to avoid loading full rows into memory Replace Prisma ORM count/find_many calls with two query_raw calls that only project the key_alias column. The Prisma client wrapper does not support SELECT projection via find_many, so raw SQL is used to keep memory usage proportional to the page size rather than total key count. Update tests to mock query_raw instead of count/find_many. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 52 +++++++----- .../test_key_management_endpoints.py | 79 +++++++++++-------- 2 files changed, 76 insertions(+), 55 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8a5145bf793..a0442a476f9 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4135,29 +4135,41 @@ async def key_aliases( verbose_proxy_logger.error("Database not connected") raise Exception("Database not connected") - conditions: List[Dict[str, Any]] = [{"key_alias": {"not": None}}] - try: - conditions.append(_get_condition_to_filter_out_ui_session_tokens()) - except NameError: - # Helper may not exist in some builds; ignore if missing - pass + # Build a parameterized WHERE clause to avoid loading full rows into + # memory. Raw SQL is used because the Prisma client wrapper does not + # support column-level SELECT projection on find_many. + # + # $1 is always UI_SESSION_TOKEN_TEAM_ID (filters out UI session tokens). + query_params: List[Any] = [UI_SESSION_TOKEN_TEAM_ID] + where_parts = [ + "key_alias IS NOT NULL", + "key_alias != ''", + "(team_id IS NULL OR team_id != $1)", + ] if search: - conditions.append( - {"key_alias": {"contains": search, "mode": "insensitive"}} - ) - where: Dict[str, Any] = {"AND": conditions} + query_params.append(f"%{search}%") + where_parts.append(f"key_alias ILIKE ${len(query_params)}") - total_count = await prisma_client.db.litellm_verificationtoken.count( - where=where, - ) - rows = await prisma_client.db.litellm_verificationtoken.find_many( - where=where, - order=[{"key_alias": "asc"}], - skip=(page - 1) * size, - take=size, - ) + where_sql = " AND ".join(where_parts) - aliases: List[str] = [row.key_alias for row in rows if row.key_alias] # type: ignore[misc] + count_sql = ( + f'SELECT COUNT(*) AS count FROM "LiteLLM_VerificationToken" WHERE {where_sql}' + ) + count_rows = await prisma_client.db.query_raw(count_sql, *query_params) + total_count = int(count_rows[0]["count"]) if count_rows else 0 + + aliases_params = query_params + [size, (page - 1) * size] + limit_idx = len(aliases_params) - 1 + offset_idx = len(aliases_params) + aliases_sql = ( + f"SELECT key_alias" + f' FROM "LiteLLM_VerificationToken"' + f" WHERE {where_sql}" + f" ORDER BY key_alias ASC" + f" LIMIT ${limit_idx} OFFSET ${offset_idx}" + ) + alias_rows = await prisma_client.db.query_raw(aliases_sql, *aliases_params) + aliases: List[str] = [row["key_alias"] for row in alias_rows if row.get("key_alias")] total_pages = -(-total_count // size) if total_count > 0 else 0 verbose_proxy_logger.debug( diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index bdfc6734664..75325508b01 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6217,15 +6217,12 @@ async def test_generate_key_helper_fn_agent_id(): @pytest.mark.asyncio async def test_key_aliases_response_shape(): """Test that key_aliases returns the correct paginated response shape.""" - mock_row1 = MagicMock() - mock_row1.key_alias = "alias-alpha" - mock_row2 = MagicMock() - mock_row2.key_alias = "alias-beta" - mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=2) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[mock_row1, mock_row2] + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 2}], + [{"key_alias": "alias-alpha"}, {"key_alias": "alias-beta"}], + ] ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -6237,21 +6234,23 @@ async def test_key_aliases_response_shape(): assert result["total_pages"] == 1 assert result["size"] == 50 - # Both count and find_many must use the same where clause - count_where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - find_where = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] - assert count_where == find_where - - # Non-null alias filter must be present - assert json.dumps({"key_alias": {"not": None}}) in json.dumps(count_where) + # Both SQL calls must filter out null/empty aliases + count_sql = mock_prisma_client.db.query_raw.call_args_list[0].args[0] + aliases_sql = mock_prisma_client.db.query_raw.call_args_list[1].args[0] + assert "key_alias IS NOT NULL" in count_sql + assert "key_alias IS NOT NULL" in aliases_sql @pytest.mark.asyncio async def test_key_aliases_pagination_skip_take(): - """Test that skip and take are correctly computed from page and size.""" + """Test that LIMIT and OFFSET are correctly derived from page and size.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=120) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 120}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): result = await key_aliases(page=3, size=25, search=None) @@ -6261,39 +6260,49 @@ async def test_key_aliases_pagination_skip_take(): assert result["total_count"] == 120 assert result["total_pages"] == 5 # ceil(120 / 25) - find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs - assert find_many_kwargs["skip"] == 50 # (3 - 1) * 25 - assert find_many_kwargs["take"] == 25 + # aliases query params: [UI_SESSION_TOKEN_TEAM_ID, size=25, offset=50] + aliases_call_args = mock_prisma_client.db.query_raw.call_args_list[1].args + assert aliases_call_args[-2] == 25 # LIMIT = size + assert aliases_call_args[-1] == 50 # OFFSET = (3 - 1) * 25 @pytest.mark.asyncio async def test_key_aliases_search_filter(): - """Test that the search param adds a case-insensitive contains condition.""" + """Test that the search param adds a case-insensitive ILIKE condition.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 0}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await key_aliases(page=1, size=50, search="my-key") - where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - assert ( - json.dumps({"key_alias": {"contains": "my-key", "mode": "insensitive"}}) - in json.dumps(where) - ) + count_call = mock_prisma_client.db.query_raw.call_args_list[0] + count_sql = count_call.args[0] + count_params = count_call.args[1:] + + assert "ILIKE" in count_sql + assert "%my-key%" in count_params @pytest.mark.asyncio -async def test_key_aliases_no_search_omits_contains_filter(): - """Test that without a search term no contains condition is added.""" +async def test_key_aliases_no_search_omits_ilike_filter(): + """Test that without a search term no ILIKE condition is added.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 0}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await key_aliases(page=1, size=50, search=None) - where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - assert "contains" not in json.dumps(where) + count_sql = mock_prisma_client.db.query_raw.call_args_list[0].args[0] + assert "ILIKE" not in count_sql From 2d231c2f1af3ac4f322537221804ae8721d19e46 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:08:40 +0530 Subject: [PATCH 18/49] Fix code qa --- tests/code_coverage_tests/liccheck.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index e6e9d761ad5..376d2859ffa 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -105,6 +105,7 @@ google-cloud-aiplatform: >=1.47.0 # Unknown license mcp: >=1.5.0 # Unknown license google-generativeai: >=0.5.0 # Unknown license async_generator: >=1.10.0 # Unknown license +wheel: >=0.40.0 # MIT License - https://github.com/pypa/wheel/blob/main/LICENSE.txt langfuse: >=2.45.0 # Unknown license prometheus_client: >=0.20.0 # Unknown license ddtrace: >=2.19.0 # Unknown license From af3b6f333442d396cbde10230c010408acf5404b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:09:42 +0530 Subject: [PATCH 19/49] Fix: litellm/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py --- .../proxy/common_utils/test_http_parsing_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index d236f46e5c5..a1484bc263b 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -75,6 +75,7 @@ async def test_form_data_parsing(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) @@ -123,6 +124,7 @@ async def test_form_data_with_json_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) @@ -163,6 +165,7 @@ async def test_form_data_with_invalid_json_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} + mock_request.state._cached_headers = None # Should raise JSONDecodeError when trying to parse invalid JSON metadata with pytest.raises(json.JSONDecodeError): @@ -189,6 +192,7 @@ async def test_form_data_without_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) @@ -219,6 +223,7 @@ async def test_form_data_with_empty_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) @@ -256,6 +261,7 @@ async def test_form_data_with_dict_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) @@ -286,6 +292,7 @@ async def test_form_data_with_none_metadata(): mock_request.form = AsyncMock(return_value=test_data) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} + mock_request.state._cached_headers = None # Parse the form data result = await _read_request_body(mock_request) From f34b0366a31d9f8a917000aa5dee64bb5d43291b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:10:49 +0530 Subject: [PATCH 20/49] Fix: test_gaurdrails* --- .../guardrails/test_guardrail_endpoints.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index c0f16c8b953..0ac3637b380 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -13,6 +13,7 @@ sys.path.insert( from fastapi import HTTPException +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_endpoints import ( CreateGuardrailRequest, PatchGuardrailRequest, @@ -25,6 +26,8 @@ from litellm.proxy.guardrails.guardrail_endpoints import ( patch_guardrail, update_guardrail, ) + +MOCK_ADMIN_USER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) from litellm.proxy.guardrails.guardrail_registry import ( IN_MEMORY_GUARDRAIL_HANDLER, InMemoryGuardrailHandler, @@ -700,15 +703,15 @@ async def test_create_guardrail_endpoint( # Run the test if expected_exception: with pytest.raises(expected_exception) as exc_info: - await create_guardrail(MOCK_CREATE_REQUEST) - + await create_guardrail(MOCK_CREATE_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) + if scenario == "database_failure": assert "Database error" in str(exc_info.value.detail) elif scenario == "no_prisma_client": assert "Prisma client not initialized" in str(exc_info.value.detail) - + else: - result = await create_guardrail(MOCK_CREATE_REQUEST) + result = await create_guardrail(MOCK_CREATE_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) assert result["guardrail_id"] == expected_result assert result["guardrail_name"] == "Test DB Guardrail" @@ -789,15 +792,15 @@ async def test_update_guardrail_endpoint( # Run the test if expected_exception: with pytest.raises(expected_exception) as exc_info: - await update_guardrail("test-guardrail-id", MOCK_UPDATE_REQUEST) - + await update_guardrail("test-guardrail-id", MOCK_UPDATE_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) + if scenario == "database_failure": assert "Database error" in str(exc_info.value.detail) elif scenario == "no_prisma_client": assert "Prisma client not initialized" in str(exc_info.value.detail) - + else: - result = await update_guardrail("test-guardrail-id", MOCK_UPDATE_REQUEST) + result = await update_guardrail("test-guardrail-id", MOCK_UPDATE_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) assert result["guardrail_id"] == expected_result assert result["guardrail_name"] == "Test DB Guardrail" @@ -883,15 +886,15 @@ async def test_patch_guardrail_endpoint( # Run the test if expected_exception: with pytest.raises(expected_exception) as exc_info: - await patch_guardrail("test-guardrail-id", MOCK_PATCH_REQUEST) - + await patch_guardrail("test-guardrail-id", MOCK_PATCH_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) + if scenario == "database_failure": assert "Database error" in str(exc_info.value.detail) elif scenario == "no_prisma_client": assert "Prisma client not initialized" in str(exc_info.value.detail) - + else: - result = await patch_guardrail("test-guardrail-id", MOCK_PATCH_REQUEST) + result = await patch_guardrail("test-guardrail-id", MOCK_PATCH_REQUEST, user_api_key_dict=MOCK_ADMIN_USER) assert result["guardrail_id"] == expected_result assert result["guardrail_name"] == "Test DB Guardrail" @@ -947,9 +950,9 @@ async def test_delete_guardrail_endpoint( if expected_exception: with pytest.raises(expected_exception): - await delete_guardrail(guardrail_id=expected_result) + await delete_guardrail(guardrail_id=expected_result, user_api_key_dict=MOCK_ADMIN_USER) else: - result = await delete_guardrail(guardrail_id=expected_result) + result = await delete_guardrail(guardrail_id=expected_result, user_api_key_dict=MOCK_ADMIN_USER) assert result == MOCK_DB_GUARDRAIL From a54cf53ffb554a0afa5fea2e61e0895c7cecbdfa Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:13:26 +0530 Subject: [PATCH 21/49] Fix test_standard_logging_payload_includes_guardrail_information --- .../test_tracing_guardrails.py | 88 ++++++++++--------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/tests/guardrails_tests/test_tracing_guardrails.py b/tests/guardrails_tests/test_tracing_guardrails.py index 8e7ce27bc28..f119d6df3db 100644 --- a/tests/guardrails_tests/test_tracing_guardrails.py +++ b/tests/guardrails_tests/test_tracing_guardrails.py @@ -12,6 +12,8 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import _OPTIONAL_Presidio from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import StandardLoggingPayload, StandardLoggingGuardrailInformation from litellm.types.guardrails import GuardrailEventHooks +from litellm.proxy._types import UserAPIKeyAuth +from litellm.caching.caching import DualCache from typing import Optional @@ -64,9 +66,13 @@ async def test_standard_logging_payload_includes_guardrail_information(): # Create mock response objects mock_analyze_resp = MagicMock() + mock_analyze_resp.status = 200 + mock_analyze_resp.content_type = "application/json" mock_analyze_resp.json = AsyncMock(return_value=mock_analyze_response) - + mock_anonymize_resp = MagicMock() + mock_anonymize_resp.status = 200 + mock_anonymize_resp.content_type = "application/json" mock_anonymize_resp.json = AsyncMock(return_value=mock_anonymize_response) # Mock the aiohttp ClientSession with global call tracking @@ -85,7 +91,7 @@ async def test_standard_logging_payload_includes_guardrail_information(): async def close(self): self.closed = True - def post(self, url, json=None): + def post(self, url, json=None, **kwargs): class MockResponse: def __init__(self, response_obj): self.response_obj = response_obj @@ -116,8 +122,8 @@ async def test_standard_logging_payload_includes_guardrail_information(): with patch("aiohttp.ClientSession", MockClientSession): await presidio_guard.async_pre_call_hook( - user_api_key_dict={}, - cache=None, + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), data=request_data, call_type="acompletion" ) @@ -136,11 +142,11 @@ async def test_standard_logging_payload_includes_guardrail_information(): assert len(test_custom_logger.standard_logging_payload["guardrail_information"]) > 0 guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_name"] == "presidio_guard" - assert guardrail_info["guardrail_mode"] == GuardrailEventHooks.pre_call + assert guardrail_info.get("guardrail_name") == "presidio_guard" + assert guardrail_info.get("guardrail_mode") == GuardrailEventHooks.pre_call # assert that the guardrail_response is a response from presidio analyze - presidio_response = guardrail_info["guardrail_response"] + presidio_response = guardrail_info.get("guardrail_response") assert isinstance(presidio_response, list) for response_item in presidio_response: assert "analysis_explanation" in response_item @@ -150,12 +156,14 @@ async def test_standard_logging_payload_includes_guardrail_information(): assert "entity_type" in response_item # assert that the duration is not None - assert guardrail_info["duration"] is not None - assert guardrail_info["duration"] > 0 + duration = guardrail_info.get("duration") + assert duration is not None + assert duration > 0 # assert that we get the count of masked entities - assert guardrail_info["masked_entity_count"] is not None - assert guardrail_info["masked_entity_count"]["PHONE_NUMBER"] == 1 + masked_entity_count = guardrail_info.get("masked_entity_count") + assert masked_entity_count is not None + assert masked_entity_count["PHONE_NUMBER"] == 1 @@ -201,8 +209,8 @@ async def test_langfuse_trace_includes_guardrail_information(): "metadata": {}, } await presidio_guard.async_pre_call_hook( - user_api_key_dict={}, - cache=None, + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), data=request_data, call_type="acompletion" ) @@ -310,7 +318,7 @@ async def test_bedrock_guardrail_status_blocked(): try: await bedrock_guard.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), - cache=None, + cache=DualCache(), data=request_data, call_type="completion" ) @@ -331,8 +339,8 @@ async def test_bedrock_guardrail_status_blocked(): # Verify guardrail information fields (guardrail_information is now a list) guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_status"] == "guardrail_intervened" - assert guardrail_info["guardrail_provider"] == "bedrock" + assert guardrail_info.get("guardrail_status") == "guardrail_intervened" + assert guardrail_info.get("guardrail_provider") == "bedrock" # Verify the new typed status fields # guardrail_status should be "guardrail_intervened" when content is blocked @@ -395,7 +403,7 @@ async def test_bedrock_guardrail_status_success(): with patch.object(bedrock_guard, 'should_run_guardrail', return_value=True): await bedrock_guard.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), - cache=None, + cache=DualCache(), data=request_data, call_type="completion" ) @@ -411,8 +419,8 @@ async def test_bedrock_guardrail_status_success(): assert len(test_custom_logger.standard_logging_payload["guardrail_information"]) > 0 guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_status"] == "success" - assert guardrail_info["guardrail_provider"] == "bedrock" + assert guardrail_info.get("guardrail_status") == "success" + assert guardrail_info.get("guardrail_provider") == "bedrock" # Check status fields status_fields = test_custom_logger.standard_logging_payload.get("status_fields", {}) @@ -469,7 +477,7 @@ async def test_bedrock_guardrail_status_failure(): try: await bedrock_guard.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), - cache=None, + cache=DualCache(), data=request_data, call_type="completion" ) @@ -488,8 +496,8 @@ async def test_bedrock_guardrail_status_failure(): assert len(test_custom_logger.standard_logging_payload["guardrail_information"]) > 0 guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_status"] == "guardrail_failed_to_respond" - assert guardrail_info["guardrail_provider"] == "bedrock" + assert guardrail_info.get("guardrail_status") == "guardrail_failed_to_respond" + assert guardrail_info.get("guardrail_provider") == "bedrock" # Check status fields status_fields = test_custom_logger.standard_logging_payload.get("status_fields", {}) @@ -554,7 +562,7 @@ async def test_noma_guardrail_status_blocked(): try: await noma_guard.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), - cache=None, + cache=DualCache(), data=request_data, call_type="completion" ) @@ -572,8 +580,8 @@ async def test_noma_guardrail_status_blocked(): assert len(test_custom_logger.standard_logging_payload["guardrail_information"]) > 0 guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_status"] == "guardrail_intervened" - assert guardrail_info["guardrail_provider"] == "noma" + assert guardrail_info.get("guardrail_status") == "guardrail_intervened" + assert guardrail_info.get("guardrail_provider") == "noma" # Check status fields status_fields = test_custom_logger.standard_logging_payload.get("status_fields", {}) @@ -632,7 +640,7 @@ async def test_noma_guardrail_status_success(): with patch.object(noma_guard, 'should_run_guardrail', return_value=True): await noma_guard.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), - cache=None, + cache=DualCache(), data=request_data, call_type="completion" ) @@ -648,8 +656,8 @@ async def test_noma_guardrail_status_success(): assert len(test_custom_logger.standard_logging_payload["guardrail_information"]) > 0 guardrail_info = test_custom_logger.standard_logging_payload["guardrail_information"][0] - assert guardrail_info["guardrail_status"] == "success" - assert guardrail_info["guardrail_provider"] == "noma" + assert guardrail_info.get("guardrail_status") == "success" + assert guardrail_info.get("guardrail_provider") == "noma" # Check status fields status_fields = test_custom_logger.standard_logging_payload.get("status_fields", {}) @@ -679,8 +687,8 @@ def test_guardrail_status_fields_computation(): guardrail_information=intervened_info, error_str=None ) - assert status_fields_intervened["llm_api_status"] == "success" - assert status_fields_intervened["guardrail_status"] == "guardrail_intervened" + assert status_fields_intervened.get("llm_api_status") == "success" + assert status_fields_intervened.get("guardrail_status") == "guardrail_intervened" # Test legacy blocked status (for backward compatibility) blocked_info = [{"guardrail_status": "blocked"}] @@ -689,8 +697,8 @@ def test_guardrail_status_fields_computation(): guardrail_information=blocked_info, error_str=None ) - assert status_fields_blocked["llm_api_status"] == "success" - assert status_fields_blocked["guardrail_status"] == "guardrail_intervened" + assert status_fields_blocked.get("llm_api_status") == "success" + assert status_fields_blocked.get("guardrail_status") == "guardrail_intervened" # Test success status success_info = [{"guardrail_status": "success"}] @@ -699,8 +707,8 @@ def test_guardrail_status_fields_computation(): guardrail_information=success_info, error_str=None ) - assert status_fields_success["llm_api_status"] == "success" - assert status_fields_success["guardrail_status"] == "success" + assert status_fields_success.get("llm_api_status") == "success" + assert status_fields_success.get("guardrail_status") == "success" # Test guardrail_failed_to_respond status failed_info = [{"guardrail_status": "guardrail_failed_to_respond"}] @@ -709,8 +717,8 @@ def test_guardrail_status_fields_computation(): guardrail_information=failed_info, error_str=None ) - assert status_fields_failed["llm_api_status"] == "failure" - assert status_fields_failed["guardrail_status"] == "guardrail_failed_to_respond" + assert status_fields_failed.get("llm_api_status") == "failure" + assert status_fields_failed.get("guardrail_status") == "guardrail_failed_to_respond" # Test legacy failure status (for backward compatibility) failure_info = [{"guardrail_status": "failure"}] @@ -719,8 +727,8 @@ def test_guardrail_status_fields_computation(): guardrail_information=failure_info, error_str=None ) - assert status_fields_failure["llm_api_status"] == "failure" - assert status_fields_failure["guardrail_status"] == "guardrail_failed_to_respond" + assert status_fields_failure.get("llm_api_status") == "failure" + assert status_fields_failure.get("guardrail_status") == "guardrail_failed_to_respond" # Test no guardrail run no_guardrail = None @@ -729,5 +737,5 @@ def test_guardrail_status_fields_computation(): guardrail_information=no_guardrail, error_str=None ) - assert status_fields_no_guardrail["llm_api_status"] == "success" - assert status_fields_no_guardrail["guardrail_status"] == "not_run" \ No newline at end of file + assert status_fields_no_guardrail.get("llm_api_status") == "success" + assert status_fields_no_guardrail.get("guardrail_status") == "not_run" \ No newline at end of file From 9bf369d53b54f5fb0df8f378db13c9be34c93994 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:18:01 +0530 Subject: [PATCH 22/49] Fix : enterprise tests --- litellm/types/integrations/prometheus.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 482b87085dd..2c75276d9ca 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from typing_extensions import Annotated import litellm @@ -721,6 +721,13 @@ class UserAPIKeyLabelValues(BaseModel): Optional[str], Field(..., alias=UserAPIKeyLabelNames.STREAM.value) ] = None + @field_validator("stream", mode="before") + @classmethod + def coerce_stream_to_str(cls, v: Any) -> Optional[str]: + if v is None: + return None + return str(v) + class PrometheusMetricsConfig(BaseModel): """Configuration for filtering Prometheus metrics""" From 7cda0e4edde439ff978ee807d089668bfc0d0ea9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:24:59 +0530 Subject: [PATCH 23/49] Fix code qa --- .../test_router_utils_common_utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index 8ff1ba45cc2..587b6a97b56 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -3,6 +3,7 @@ from unittest.mock import Mock import pytest +from litellm import Router from litellm.router_utils.common_utils import ( _deployment_supports_web_search, filter_team_based_models, @@ -340,3 +341,22 @@ class TestFilterWebSearchDeployments: result = filter_web_search_deployments(deployment, request_kwargs) # Should return the dict unchanged, not filter it assert result == deployment + + +def test_invalidate_model_group_info_cache(): + """Test that _invalidate_model_group_info_cache clears the LRU cache.""" + router = Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake-key"}, + } + ] + ) + # Populate the cache + router._cached_get_model_group_info("gpt-4") + assert router._cached_get_model_group_info.cache_info().currsize > 0 + + # Invalidate and verify cache is cleared + router._invalidate_model_group_info_cache() + assert router._cached_get_model_group_info.cache_info().currsize == 0 From 9c7f8138e1d2e72b0876a673b3f3c1fda5d43698 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:29:07 +0530 Subject: [PATCH 24/49] FIx : litellm/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py --- .../test_key_management_endpoints.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 05df3c2dcbb..fb71adc1085 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6194,15 +6194,14 @@ async def test_generate_key_helper_fn_agent_id(): ) mock_prisma_client.insert_data = mock_insert - with patch.object(km, "prisma_client", mock_prisma_client): - with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): - await generate_key_helper_fn( - request_type="key", - agent_id="test-agent-456", - key_alias="test-agent-key", - models=[], - table_name="key", - ) + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await generate_key_helper_fn( + request_type="key", + agent_id="test-agent-456", + key_alias="test-agent-key", + models=[], + table_name="key", + ) assert mock_insert.called, "insert_data was never called" # insert_data is called as insert_data(data=key_data, ...) From 0debe926059c049501b3d33de31dff94a756ef1f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:42:27 +0530 Subject: [PATCH 25/49] Fix_mapped tests part 2 --- .../test_llm_pass_through_endpoints.py | 3 +- .../test_spend_management_endpoints.py | 98 ++++++++++--------- .../proxy/test_health_check_functions.py | 2 +- .../test_model_dump_with_preserved_fields.py | 9 +- tests/test_litellm/proxy/test_proxy_cli.py | 8 +- .../proxy/test_shared_health_check.py | 11 ++- 6 files changed, 69 insertions(+), 62 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 98161402c45..cac8f98cbb7 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1479,10 +1479,11 @@ class TestForwardHeaders: # Create a mock request with custom headers mock_request = MagicMock(spec=Request) + mock_request.state = None # Prevent MagicMock from returning a truthy _cached_headers mock_request.method = "POST" mock_request.url = MagicMock() mock_request.url.path = "/test/endpoint" - + # User headers that should be forwarded user_headers = { "x-custom-header": "custom-value", diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index e439dfd693c..2aecc2ec2e5 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1650,58 +1650,64 @@ async def test_global_spend_keys_endpoint_limit_validation(client, monkeypatch): # Create a simple mock for prisma client with empty response mock_prisma_client = MagicMock() mock_db = MagicMock() - mock_query_raw = MagicMock() - mock_query_raw.return_value = asyncio.Future() - mock_query_raw.return_value.set_result([]) + mock_query_raw = AsyncMock(return_value=[]) mock_db.query_raw = mock_query_raw mock_prisma_client.db = mock_db # Apply the mock to the prisma_client module monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - # Call the endpoint without specifying a limit - no_limit_response = client.get("/global/spend/keys") - assert no_limit_response.status_code == 200 - mock_query_raw.assert_called_once_with('SELECT * FROM "Last30dKeysBySpend";') - # Reset the mock for the next test - mock_query_raw.reset_mock() - # Test with valid input - normal_limit = "10" - good_input_response = client.get(f"/global/spend/keys?limit={normal_limit}") - assert good_input_response.status_code == 200 - # Verify the mock was called with the correct parameters - mock_query_raw.assert_called_once_with( - 'SELECT * FROM "Last30dKeysBySpend" LIMIT $1 ;', 10 + # Override auth to bypass API key validation + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" ) - # Reset the mock for the next test - mock_query_raw.reset_mock() - # Test with SQL injection payload - sql_injection_limit = "10; DROP TABLE spend_logs; --" - response = client.get(f"/global/spend/keys?limit={sql_injection_limit}") - # Verify the response is a validation error (422) - assert response.status_code == 422 - # Verify the mock was not called with the SQL injection payload - # This confirms that the validation happens before the database query - mock_query_raw.assert_not_called() - # Reset the mock for the next test - mock_query_raw.reset_mock() - # Test with non-numeric input - non_numeric_limit = "abc" - response = client.get(f"/global/spend/keys?limit={non_numeric_limit}") - assert response.status_code == 422 - mock_query_raw.assert_not_called() - mock_query_raw.reset_mock() - # Test with negative number - negative_limit = "-5" - response = client.get(f"/global/spend/keys?limit={negative_limit}") - assert response.status_code == 422 - mock_query_raw.assert_not_called() - mock_query_raw.reset_mock() - # Test with zero - zero_limit = "0" - response = client.get(f"/global/spend/keys?limit={zero_limit}") - assert response.status_code == 422 - mock_query_raw.assert_not_called() - mock_query_raw.reset_mock() + + try: + # Call the endpoint without specifying a limit + no_limit_response = client.get("/global/spend/keys") + assert no_limit_response.status_code == 200 + mock_query_raw.assert_called_once_with('SELECT * FROM "Last30dKeysBySpend";') + # Reset the mock for the next test + mock_query_raw.reset_mock() + # Test with valid input + normal_limit = "10" + good_input_response = client.get(f"/global/spend/keys?limit={normal_limit}") + assert good_input_response.status_code == 200 + # Verify the mock was called with the correct parameters + mock_query_raw.assert_called_once_with( + 'SELECT * FROM "Last30dKeysBySpend" LIMIT $1 ;', 10 + ) + # Reset the mock for the next test + mock_query_raw.reset_mock() + # Test with SQL injection payload + sql_injection_limit = "10; DROP TABLE spend_logs; --" + response = client.get(f"/global/spend/keys?limit={sql_injection_limit}") + # Verify the response is a validation error (422) + assert response.status_code == 422 + # Verify the mock was not called with the SQL injection payload + # This confirms that the validation happens before the database query + mock_query_raw.assert_not_called() + # Reset the mock for the next test + mock_query_raw.reset_mock() + # Test with non-numeric input + non_numeric_limit = "abc" + response = client.get(f"/global/spend/keys?limit={non_numeric_limit}") + assert response.status_code == 422 + mock_query_raw.assert_not_called() + mock_query_raw.reset_mock() + # Test with negative number + negative_limit = "-5" + response = client.get(f"/global/spend/keys?limit={negative_limit}") + assert response.status_code == 422 + mock_query_raw.assert_not_called() + mock_query_raw.reset_mock() + # Test with zero + zero_limit = "0" + response = client.get(f"/global/spend/keys?limit={zero_limit}") + assert response.status_code == 422 + mock_query_raw.assert_not_called() + mock_query_raw.reset_mock() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index 4c91d0ae91e..354698b02fe 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -480,7 +480,7 @@ async def test_perform_health_check_and_save_passes_model_id_to_perform_health_c healthy = [{"model": "gpt-4"}] unhealthy = [] - async def mock_perform_health_check(model_list, model=None, cli_model=None, details=True, model_id=None): + async def mock_perform_health_check(model_list, model=None, cli_model=None, details=True, model_id=None, max_concurrency=None): return healthy, unhealthy with patch( diff --git a/tests/test_litellm/proxy/test_model_dump_with_preserved_fields.py b/tests/test_litellm/proxy/test_model_dump_with_preserved_fields.py index 3001c87ebed..316c1e879cc 100644 --- a/tests/test_litellm/proxy/test_model_dump_with_preserved_fields.py +++ b/tests/test_litellm/proxy/test_model_dump_with_preserved_fields.py @@ -242,7 +242,7 @@ def test_full_output_structure_non_streaming(): ) result = model_dump_with_preserved_fields(response, exclude_unset=True) - # Top-level keys + # Top-level keys (usage is None when not explicitly set and excluded by exclude_unset=True) assert set(result.keys()) == { "id", "choices", @@ -250,7 +250,6 @@ def test_full_output_structure_non_streaming(): "model", "object", "system_fingerprint", - "usage", } assert result["object"] == "chat.completion" assert result["model"] == "gpt-4.1" @@ -270,12 +269,6 @@ def test_full_output_structure_non_streaming(): assert msg["content"] == "Hello!" assert msg["role"] == "assistant" - # Usage structure - usage = result["usage"] - assert "prompt_tokens" in usage - assert "completion_tokens" in usage - assert "total_tokens" in usage - def test_full_output_structure_tool_calls(): """ diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index c839c22de5f..c6b2015984e 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -331,7 +331,8 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") - def test_max_requests_before_restart_flag(self, mock_print, mock_uvicorn_run): + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + def test_max_requests_before_restart_flag(self, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the max_requests_before_restart flag is passed to uvicorn as limit_max_requests""" from click.testing import CliRunner @@ -344,7 +345,10 @@ class TestProxyInitializationHelpers: mock_key_mgmt = MagicMock() mock_save_worker_config = MagicMock() + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with patch.dict( + os.environ, clean_env, clear=True, + ), patch.dict( "sys.modules", { "proxy_server": MagicMock( @@ -367,7 +371,7 @@ class TestProxyInitializationHelpers: run_server, ["--local", "--max_requests_before_restart", "123"] ) - assert result.exit_code == 0 + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() # Check that uvicorn.run was called with limit_max_requests parameter diff --git a/tests/test_litellm/proxy/test_shared_health_check.py b/tests/test_litellm/proxy/test_shared_health_check.py index 82deebc424a..0212d87baab 100644 --- a/tests/test_litellm/proxy/test_shared_health_check.py +++ b/tests/test_litellm/proxy/test_shared_health_check.py @@ -1,10 +1,13 @@ import asyncio import json -import pytest import time from unittest.mock import AsyncMock, MagicMock, patch -from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager +import pytest + +from litellm.proxy.health_check_utils.shared_health_check_manager import ( + SharedHealthCheckManager, +) class TestSharedHealthCheckManager: @@ -272,7 +275,7 @@ class TestSharedHealthCheckManager: ) # Should call perform_health_check and cache results - mock_perform.assert_called_once_with(model_list=model_list, details=True) + mock_perform.assert_called_once_with(model_list=model_list, details=True, max_concurrency=None) assert healthy == expected_healthy assert unhealthy == expected_unhealthy @@ -329,7 +332,7 @@ class TestSharedHealthCheckManager: # Should fall back to local health check mock_sleep.assert_called_once_with(2) - mock_perform.assert_called_once_with(model_list=model_list, details=True) + mock_perform.assert_called_once_with(model_list=model_list, details=True, max_concurrency=None) assert healthy == expected_healthy assert unhealthy == expected_unhealthy From 82025bff6e4b57574363ec7999bc9ce9841c28b3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:47:52 +0530 Subject: [PATCH 26/49] Fix code qa --- .../block_code_execution/block_code_execution.py | 2 -- litellm/realtime_api/main.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py index 77f2eaa27d7..99abdf28211 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py @@ -11,7 +11,6 @@ from datetime import datetime from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, Dict, List, Literal, @@ -37,7 +36,6 @@ from litellm.types.utils import ( GenericGuardrailAPIInputs, GuardrailStatus, GuardrailTracingDetail, - ModelResponseStream, ) if TYPE_CHECKING: diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e4c8f648190..75fb6f35add 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -33,7 +33,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() @wrapper_client -async def _arealtime( +async def _arealtime( # noqa: PLR0915 model: str, websocket: Any, # fastapi websocket api_base: Optional[str] = None, From 3545584a00c951a81d0f82c90fa360a899ccf4a7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 25 Feb 2026 23:20:03 -0800 Subject: [PATCH 27/49] Development environment setup (#22160) * feat: add pretty view for realtime API logs in dashboard - Create RealtimePrettyView component that renders structured session config, conversation turns with transcripts, and token breakdowns - Update PrettyMessagesView to detect realtime responses (via isRealtimeResponse helper) and delegate to the new component - Session card shows model, voice, modalities, temperature, instructions in a collapsible panel - Conversation turns show status, per-turn token usage, and audio/text transcripts with appropriate icons - Add 24 tests for RealtimePrettyView and 3 tests for PrettyMessagesView - All 75 LogDetailsDrawer tests pass Co-authored-by: Ishaan Jaff * chore: remove dev_config.yaml from tracked files Co-authored-by: Ishaan Jaff * feat: show turn count in realtime pretty view session header and output header - Add purple 'N turns' tag to Session card header for at-a-glance turn count - Add 'Turns: N' to the Output section header next to tokens/cost - Extend SectionHeader to accept optional turnCount prop - Add 3 new tests for turn count display (singular, plural, output header) Co-authored-by: Ishaan Jaff * fix: address Greptile review feedback - Remove response.audio.done and conversation.item.created from isRealtimeResponse() detection since the view doesn't render them; prevents misleading fallback for responses with only those events - Remove dead code: index >= 0 is always true in .map() callback Co-authored-by: Ishaan Jaff --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff --- ui/litellm-dashboard/package-lock.json | 97 +-- .../PrettyMessagesView.test.tsx | 81 +++ .../LogDetailsDrawer/PrettyMessagesView.tsx | 8 +- .../RealtimePrettyView.test.tsx | 392 ++++++++++++ .../LogDetailsDrawer/RealtimePrettyView.tsx | 570 ++++++++++++++++++ .../LogDetailsDrawer/SectionHeader.tsx | 10 +- 6 files changed, 1064 insertions(+), 94 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index cc04e674003..503ed4a62a8 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -90,7 +90,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -1772,7 +1771,6 @@ "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", @@ -1783,7 +1781,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -1793,14 +1790,12 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -1978,7 +1973,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -1992,7 +1986,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -2002,7 +1995,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -2326,7 +2318,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", "integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright": "1.58.1" @@ -3431,14 +3423,12 @@ "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "dev": true, "license": "MIT" }, "node_modules/@types/react": { "version": "18.2.48", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", - "dev": true, "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -3480,7 +3470,6 @@ "version": "0.26.0", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", - "dev": true, "license": "MIT" }, "node_modules/@types/unist": { @@ -4341,14 +4330,12 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true, "license": "MIT" }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", @@ -4362,7 +4349,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -4375,7 +4361,6 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", - "dev": true, "license": "MIT" }, "node_modules/argparse": { @@ -4747,7 +4732,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4773,7 +4757,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -4889,7 +4872,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5013,7 +4995,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - "dev": true, "license": "MIT", "dependencies": { "anymatch": "~3.1.2", @@ -5038,7 +5019,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -5114,7 +5094,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5175,7 +5154,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", - "dev": true, "license": "MIT", "bin": { "cssesc": "bin/cssesc" @@ -5589,14 +5567,12 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", - "dev": true, "license": "MIT" }, "node_modules/doctrine": { @@ -6510,7 +6486,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -6543,7 +6518,6 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -6581,7 +6555,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -6742,7 +6715,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -6893,7 +6865,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.3" @@ -7391,7 +7362,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "license": "MIT", "dependencies": { "binary-extensions": "^2.0.0" @@ -7444,7 +7414,6 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -7505,7 +7474,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7551,7 +7519,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -7600,7 +7567,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -7877,7 +7843,6 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -8163,7 +8128,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, "license": "MIT", "engines": { "node": ">=14" @@ -8176,7 +8140,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, "license": "MIT" }, "node_modules/locate-path": { @@ -8491,7 +8454,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -8943,7 +8905,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -8957,7 +8918,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -9072,7 +9032,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0", @@ -9284,7 +9243,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9303,7 +9261,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9648,7 +9605,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -9695,7 +9651,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -9708,7 +9663,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9718,7 +9672,6 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9728,7 +9681,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", "integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright-core": "1.58.1" @@ -9747,7 +9700,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz", "integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -9770,7 +9723,6 @@ "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9799,7 +9751,6 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", - "dev": true, "license": "MIT", "dependencies": { "postcss-value-parser": "^4.0.0", @@ -9817,7 +9768,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9843,7 +9793,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9886,7 +9835,6 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9912,7 +9860,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", - "dev": true, "license": "MIT", "dependencies": { "cssesc": "^3.0.0", @@ -9926,7 +9873,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true, "license": "MIT" }, "node_modules/prelude-ls": { @@ -10040,7 +9986,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -10829,7 +10774,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, "license": "MIT", "dependencies": { "pify": "^2.3.0" @@ -10839,7 +10783,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "license": "MIT", "dependencies": { "picomatch": "^2.2.1" @@ -10852,7 +10795,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -11117,7 +11059,6 @@ "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", @@ -11158,7 +11099,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -11214,7 +11154,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -11855,7 +11794,6 @@ "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", @@ -11891,7 +11829,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11927,7 +11864,6 @@ "version": "3.4.19", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", - "dev": true, "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", @@ -11965,7 +11901,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -11982,7 +11917,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -12010,7 +11944,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0" @@ -12020,7 +11953,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, "license": "MIT", "dependencies": { "thenify": ">= 3.1.0 < 4" @@ -12062,7 +11994,6 @@ "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -12129,7 +12060,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -12217,7 +12147,6 @@ "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true, "license": "Apache-2.0" }, "node_modules/tsconfig-paths": { @@ -12334,7 +12263,7 @@ "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -12536,7 +12465,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -12990,7 +12918,7 @@ "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -13056,21 +12984,6 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.33", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", - "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } } } } diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx new file mode 100644 index 00000000000..b73dcafcdc3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx @@ -0,0 +1,81 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import { PrettyMessagesView } from "./PrettyMessagesView"; + +vi.mock("antd", async () => { + const actual = await vi.importActual("antd"); + return { + ...actual, + message: { + success: vi.fn(), + }, + }; +}); + +describe("PrettyMessagesView", () => { + it("should render the component for standard chat completions", () => { + const request = { + messages: [{ role: "user", content: "Hello" }], + }; + const response = { + choices: [{ message: { role: "assistant", content: "Hi there!" } }], + }; + + render(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + expect(screen.getByText("Hi there!")).toBeInTheDocument(); + }); + + it("should render the realtime pretty view for realtime API responses", () => { + const request = {}; + const response = { + results: [ + { + type: "session.created", + session: { + id: "sess_123", + model: "gpt-4o-mini-realtime-preview", + voice: "alloy", + modalities: ["audio", "text"], + }, + }, + { + type: "response.done", + response: { + id: "resp_1", + status: "completed", + output: [ + { + id: "item_1", + role: "assistant", + type: "message", + content: [{ type: "audio", transcript: "Hello from realtime!" }], + }, + ], + }, + }, + ], + }; + + render(); + expect(screen.getByText("Session")).toBeInTheDocument(); + expect(screen.getByText("Hello from realtime!")).toBeInTheDocument(); + const modelElements = screen.getAllByText("gpt-4o-mini-realtime-preview"); + expect(modelElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should render standard view when response has results but no realtime events", () => { + const request = { + messages: [{ role: "user", content: "Test" }], + }; + const response = { + results: [{ type: "some.other.type" }], + choices: [{ message: { role: "assistant", content: "Reply" } }], + }; + + render(); + expect(screen.getByText("Test")).toBeInTheDocument(); + expect(screen.getByText("Reply")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx index 2d4a14d6f5d..06a5bbb5f9b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx @@ -1,11 +1,13 @@ /** * PrettyMessagesView - Datadog-style view with Input/Output cards - * Two main cards showing request and response with token counts and costs + * Two main cards showing request and response with token counts and costs. + * Detects realtime API responses and renders a specialized view. */ import { parseMessages } from './prettyMessagesUtils'; import { InputCard } from './InputCard'; import { OutputCard } from './OutputCard'; +import { isRealtimeResponse, RealtimePrettyView } from './RealtimePrettyView'; interface PrettyMessagesViewProps { request: any; @@ -19,6 +21,10 @@ interface PrettyMessagesViewProps { } export function PrettyMessagesView({ request, response, metrics }: PrettyMessagesViewProps) { + if (isRealtimeResponse(response)) { + return ; + } + const { requestMessages, responseMessage } = parseMessages(request, response); return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx new file mode 100644 index 00000000000..4a0adecc967 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx @@ -0,0 +1,392 @@ +import React from "react"; +import { render, screen, fireEvent, act, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { RealtimePrettyView, isRealtimeResponse } from "./RealtimePrettyView"; + +vi.mock("antd", async () => { + const actual = await vi.importActual("antd"); + return { + ...actual, + message: { + success: vi.fn(), + }, + }; +}); + +const sampleRealtimeResponse = { + usage: { + total_tokens: 587, + prompt_tokens: 294, + completion_tokens: 293, + }, + results: [ + { + type: "session.created", + session: { + id: "sess_DDNQlPKHjLsokSJPAOWY0", + model: "gpt-4o-mini-realtime-preview", + tools: [], + voice: "alloy", + modalities: ["audio", "text"], + temperature: 0.8, + tool_choice: "auto", + instructions: "You are a helpful assistant.", + turn_detection: { + type: "server_vad", + threshold: 0.5, + }, + input_audio_format: "pcm16", + output_audio_format: "pcm16", + max_response_output_tokens: "inf", + }, + event_id: "event_DDNQlB4VNUlpqTVIjBbm3", + }, + { + type: "response.done", + event_id: "event_DDNQnagYJCZyZATdJCn0L", + response: { + id: "resp_DDNQnlXGHZJB46D5JhJ95", + usage: { + input_tokens: 116, + total_tokens: 162, + output_tokens: 46, + input_token_details: { + text_tokens: 116, + audio_tokens: 0, + }, + output_token_details: { + text_tokens: 16, + audio_tokens: 30, + }, + }, + voice: "alloy", + object: "realtime.response", + output: [ + { + id: "item_DDNQnz5uN1b8NEvPOPZOM", + role: "assistant", + type: "message", + status: "completed", + content: [ + { + type: "audio", + transcript: "Hello! How's your day going?", + }, + ], + }, + ], + status: "completed", + conversation_id: "conv_DDNQlpNllPYhCCfXCtT8X", + max_output_tokens: "inf", + }, + }, + { + type: "response.done", + event_id: "event_DDNR0VmrRTVU69RxGC29U", + response: { + id: "resp_DDNQy6S4PBZxW4qsKq6Ah", + usage: { + input_tokens: 178, + total_tokens: 425, + output_tokens: 247, + }, + voice: "alloy", + object: "realtime.response", + output: [ + { + id: "item_DDNQywctWVnYmujg4FSTZ", + role: "assistant", + type: "message", + status: "completed", + content: [ + { + type: "audio", + transcript: + "I'm here to help with information and general questions.", + }, + ], + }, + ], + status: "completed", + conversation_id: "conv_DDNQlpNllPYhCCfXCtT8X", + max_output_tokens: "inf", + }, + }, + ], +}; + +describe("isRealtimeResponse", () => { + it("should return true for a valid realtime response with session.created", () => { + expect(isRealtimeResponse(sampleRealtimeResponse)).toBe(true); + }); + + it("should return true for response with only response.done events", () => { + const resp = { + results: [{ type: "response.done", response: { id: "r1" } }], + }; + expect(isRealtimeResponse(resp)).toBe(true); + }); + + it("should return false for a standard chat completion response", () => { + const chatResponse = { + choices: [{ message: { role: "assistant", content: "Hello" } }], + }; + expect(isRealtimeResponse(chatResponse)).toBe(false); + }); + + it("should return false for null/undefined", () => { + expect(isRealtimeResponse(null)).toBe(false); + expect(isRealtimeResponse(undefined)).toBe(false); + }); + + it("should return false for empty results array", () => { + expect(isRealtimeResponse({ results: [] })).toBe(false); + }); + + it("should return false for results with unrecognized event types", () => { + const resp = { + results: [{ type: "some.unknown.event" }], + }; + expect(isRealtimeResponse(resp)).toBe(false); + }); +}); + +describe("RealtimePrettyView", () => { + const mockWriteText = vi.fn().mockResolvedValue(undefined); + + beforeEach(() => { + vi.clearAllMocks(); + Object.defineProperty(navigator, "clipboard", { + value: { writeText: mockWriteText }, + writable: true, + configurable: true, + }); + }); + + it("should render the component successfully", () => { + render(); + expect(screen.getByText("Session")).toBeInTheDocument(); + }); + + it("should display the session model name", () => { + render(); + const modelElements = screen.getAllByText("gpt-4o-mini-realtime-preview"); + expect(modelElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should display the session voice tag", () => { + render(); + const voiceElements = screen.getAllByText("alloy"); + expect(voiceElements.length).toBeGreaterThanOrEqual(1); + }); + + it("should display modality tags", () => { + render(); + expect(screen.getByText("audio")).toBeInTheDocument(); + expect(screen.getByText("text")).toBeInTheDocument(); + }); + + it("should display the turn count in session header", () => { + render(); + expect(screen.getByText("2 turns")).toBeInTheDocument(); + }); + + it("should display singular 'turn' for a single response event", () => { + const singleTurnResponse = { + results: [ + { + type: "session.created", + session: { + id: "sess_1", + model: "gpt-4o-mini-realtime-preview", + voice: "alloy", + modalities: ["audio"], + }, + }, + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [ + { + id: "item1", + role: "assistant", + type: "message", + content: [{ type: "audio", transcript: "Hi!" }], + }, + ], + }, + }, + ], + }; + render(); + expect(screen.getByText("1 turn")).toBeInTheDocument(); + }); + + it("should display the turn count in the output section header", () => { + render(); + expect(screen.getByText("Turns: 2")).toBeInTheDocument(); + }); + + it("should display the Output section header", () => { + render(); + expect(screen.getByText("Output")).toBeInTheDocument(); + }); + + it("should display transcript text from response turns", () => { + render(); + expect( + screen.getByText("Hello! How's your day going?") + ).toBeInTheDocument(); + expect( + screen.getByText( + "I'm here to help with information and general questions." + ) + ).toBeInTheDocument(); + }); + + it("should display completed status tags for response turns", () => { + render(); + const completedTags = screen.getAllByText("completed"); + expect(completedTags.length).toBe(2); + }); + + it("should display token usage per turn", () => { + render(); + expect(screen.getByText("116 in / 46 out tokens")).toBeInTheDocument(); + expect(screen.getByText("178 in / 247 out tokens")).toBeInTheDocument(); + }); + + it("should expand session details when session header is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Temperature")).toBeInTheDocument(); + }); + }); + + it("should display session instructions when expanded", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Instructions")).toBeInTheDocument(); + expect( + screen.getByText("You are a helpful assistant.") + ).toBeInTheDocument(); + }); + }); + + it("should display session audio format when expanded", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Session")); + + await waitFor(() => { + expect(screen.getByText("Input Audio Format")).toBeInTheDocument(); + expect(screen.getAllByText("pcm16").length).toBeGreaterThanOrEqual(1); + }); + }); + + it("should display ASSISTANT label for output messages", () => { + render(); + const assistantLabels = screen.getAllByText("ASSISTANT"); + expect(assistantLabels.length).toBe(2); + }); + + it("should display fallback message when no recognized events exist", () => { + const emptyResponse = { + results: [{ type: "unknown.event" }], + }; + render(); + expect( + screen.getByText("No recognized realtime events found") + ).toBeInTheDocument(); + }); + + it("should handle response with no output items gracefully", () => { + const noOutputResponse = { + results: [ + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [], + }, + }, + ], + }; + render(); + expect(screen.getByText("completed")).toBeInTheDocument(); + }); + + it("should display metrics tokens when provided", () => { + render( + + ); + expect(screen.getByText(/Tokens: 500/)).toBeInTheDocument(); + expect(screen.getByText(/Cost: \$0\.005000/)).toBeInTheDocument(); + }); + + it("should toggle output section collapse when header is clicked", async () => { + const user = userEvent.setup(); + render(); + + const transcript = screen.getByText("Hello! How's your day going?"); + expect(transcript).toBeVisible(); + + const outputHeader = screen.getByText("Output").closest("div"); + if (outputHeader) { + await user.click(outputHeader); + await waitFor(() => { + expect(transcript).not.toBeVisible(); + }); + } + }); + + it("should display token breakdown tags when input_token_details are present", async () => { + render(); + expect(screen.getByText(/Text Tokens: 116/)).toBeInTheDocument(); + }); + + it("should handle text content type in addition to audio", () => { + const textResponse = { + results: [ + { + type: "response.done", + response: { + id: "r1", + status: "completed", + output: [ + { + id: "item1", + role: "assistant", + type: "message", + content: [ + { + type: "text", + text: "This is a text response", + }, + ], + }, + ], + }, + }, + ], + }; + render(); + expect(screen.getByText("This is a text response")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx new file mode 100644 index 00000000000..4029fbdb835 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.tsx @@ -0,0 +1,570 @@ +/** + * RealtimePrettyView - Structured pretty view for OpenAI Realtime API logs + * Displays session config, conversation turns, and token usage + * in a readable format instead of raw JSON. + */ + +import { useState } from 'react'; +import { Typography, Tag, Tooltip } from 'antd'; +import { + SoundOutlined, + MessageOutlined, + SettingOutlined, + AudioOutlined, + DownOutlined, + UpOutlined, +} from '@ant-design/icons'; +import { SectionHeader } from './SectionHeader'; + +const { Text } = Typography; + +interface RealtimeEvent { + type: string; + event_id?: string; + session?: RealtimeSession; + response?: RealtimeResponse; +} + +interface RealtimeSession { + id: string; + model: string; + voice?: string; + modalities?: string[]; + temperature?: number; + tools?: any[]; + instructions?: string; + turn_detection?: Record; + input_audio_format?: string; + output_audio_format?: string; + max_response_output_tokens?: string | number; + [key: string]: any; +} + +interface RealtimeResponse { + id: string; + status: string; + usage?: { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + input_token_details?: Record; + output_token_details?: Record; + }; + output?: RealtimeOutputItem[]; + modalities?: string[]; + voice?: string; + conversation_id?: string; + [key: string]: any; +} + +interface RealtimeOutputItem { + id: string; + role: string; + type: string; + status?: string; + content?: Array<{ + type: string; + transcript?: string; + text?: string; + }>; +} + +interface RealtimePrettyViewProps { + response: any; + metrics?: { + prompt_tokens?: number; + completion_tokens?: number; + input_cost?: number; + output_cost?: number; + }; +} + +export function isRealtimeResponse(response: any): boolean { + if (!response || !response.results || !Array.isArray(response.results) || response.results.length === 0) { + return false; + } + + return response.results.some( + (r: any) => + r.type === 'session.created' || + r.type === 'session.updated' || + r.type === 'response.done' + ); +} + +export function RealtimePrettyView({ response, metrics }: RealtimePrettyViewProps) { + const events: RealtimeEvent[] = response?.results || []; + const usage = response?.usage; + + const sessionEvent = events.find( + (e) => e.type === 'session.created' || e.type === 'session.updated' + ); + const responseEvents = events.filter((e) => e.type === 'response.done'); + + return ( +
+ {/* Session Configuration Card */} + {sessionEvent?.session && ( + + )} + + {/* Conversation Turns */} + {responseEvents.length > 0 && ( + e.response!).filter(Boolean)} + totalUsage={usage} + metrics={metrics} + /> + )} + + {/* Fallback if no recognized events */} + {!sessionEvent && responseEvents.length === 0 && ( +
+ No recognized realtime events found +
+ )} +
+ ); +} + +function SessionCard({ session, turnCount }: { session: RealtimeSession; turnCount: number }) { + const [isCollapsed, setIsCollapsed] = useState(true); + + return ( +
+
setIsCollapsed(!isCollapsed)} + style={{ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + padding: '10px 16px', + borderBottom: isCollapsed ? 'none' : '1px solid #f0f0f0', + background: '#fafafa', + cursor: 'pointer', + transition: 'background 0.15s ease', + }} + onMouseEnter={(e) => { + e.currentTarget.style.background = '#f5f5f5'; + }} + onMouseLeave={(e) => { + e.currentTarget.style.background = '#fafafa'; + }} + > +
+
+ {isCollapsed ? ( + + ) : ( + + )} +
+
+ + Session +
+ + {session.model} + + {turnCount > 0 && ( + + {turnCount} {turnCount === 1 ? 'turn' : 'turns'} + + )} + {session.voice && ( + + {session.voice} + + )} + {session.modalities && ( +
+ {session.modalities.map((m) => ( + + {m === 'audio' ? : } {m} + + ))} +
+ )} +
+
+ +
+
+
+ + + + + + + {session.turn_detection && ( + + )} + {session.tools && session.tools.length > 0 && ( + + )} +
+ + {session.instructions && ( +
+ + Instructions + +
+ {session.instructions} +
+
+ )} +
+
+
+ ); +} + +function ConversationCard({ + responses, + totalUsage, + metrics, +}: { + responses: RealtimeResponse[]; + totalUsage?: any; + metrics?: RealtimePrettyViewProps['metrics']; +}) { + const [isCollapsed, setIsCollapsed] = useState(false); + + const totalTokens = totalUsage?.total_tokens; + const turnCount = responses.length; + const handleCopy = () => { + const transcripts = responses + .flatMap((r) => + (r.output || []).flatMap((o) => + (o.content || []).map( + (c) => `${o.role}: ${c.transcript || c.text || ''}` + ) + ) + ) + .join('\n'); + navigator.clipboard.writeText(transcripts); + }; + + return ( +
+ setIsCollapsed(!isCollapsed)} + turnCount={turnCount} + /> + +
+
+ {responses.map((resp, idx) => ( + + ))} +
+
+
+ ); +} + +function ResponseTurn({ + response, + index, +}: { + response: RealtimeResponse; + index: number; +}) { + const outputs = response.output || []; + const usage = response.usage; + + return ( +
+ {/* Turn header */} +
+ + {response.status || 'unknown'} + + {usage && ( + + {usage.input_tokens ?? 0} in / {usage.output_tokens ?? 0} out tokens + + )} + {response.conversation_id && ( + + + conv: {response.conversation_id.slice(0, 12)}... + + + )} +
+ + {/* Output messages / transcripts */} + {outputs.map((output, oIdx) => ( + + ))} + + {/* Token breakdown if available */} + {usage?.input_token_details && ( + + )} + {usage?.output_token_details && ( + + )} +
+ ); +} + +function OutputMessage({ output }: { output: RealtimeOutputItem }) { + const contents = output.content || []; + const hasTranscripts = contents.some((c) => c.transcript || c.text); + + if (!hasTranscripts) return null; + + return ( +
+ + {output.role?.toUpperCase() || 'ASSISTANT'} + + {contents.map((c, cIdx) => { + const text = c.transcript || c.text; + if (!text) return null; + return ( +
+ {c.type === 'audio' && ( + + )} + {c.type === 'text' && ( + + )} +
+ {text} +
+
+ ); + })} +
+ ); +} + +function TokenBreakdown({ + label, + details, +}: { + label: string; + details: Record; +}) { + const entries = Object.entries(details).filter( + ([, v]) => + typeof v === 'number' || + (typeof v === 'object' && v !== null) + ); + + if (entries.length === 0) return null; + + return ( +
+ + {label} Token Breakdown + +
+ {entries.map(([key, value]) => { + if (typeof value === 'number') { + return ( + + {formatTokenLabel(key)}: {value.toLocaleString()} + + ); + } + return null; + })} +
+
+ ); +} + +function ConfigRow({ + label, + value, +}: { + label: string; + value: any; +}) { + if (value === undefined || value === null) return null; + return ( +
+ + {label} + +
+ {String(value)} +
+
+ ); +} + +function formatTokenLabel(key: string): string { + return key + .replace(/_/g, ' ') + .replace(/\b\w/g, (c) => c.toUpperCase()); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx index e667e125149..632ea961853 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx @@ -19,9 +19,10 @@ interface SectionHeaderProps { onCopy: () => void; isCollapsed?: boolean; onToggleCollapse?: () => void; + turnCount?: number; } -export function SectionHeader({ type, tokens, cost, onCopy, isCollapsed, onToggleCollapse }: SectionHeaderProps) { +export function SectionHeader({ type, tokens, cost, onCopy, isCollapsed, onToggleCollapse, turnCount }: SectionHeaderProps) { return (
)} + + {/* Turn count */} + {turnCount !== undefined && turnCount > 0 && ( + + Turns: {turnCount} + + )}
{/* Copy Button */} From 143e8dfe276c9db5d55c3b97db1e5e1c5f5c6b66 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 12:55:59 +0530 Subject: [PATCH 28/49] Fix pass through tests --- litellm/litellm_core_utils/realtime_streaming.py | 4 ++-- .../pass_through_endpoints/test_llm_pass_through_endpoints.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c5e0d1f26ae..f146b20cf21 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -155,7 +155,7 @@ class RealTimeStreaming: event_type == "conversation.item.input_audio_transcription.completed" ): - transcript = event_obj.get("transcript", "") + transcript = cast(str, event_obj.get("transcript", "")) if transcript: self.input_messages.append( {"role": "user", "content": transcript} @@ -170,7 +170,7 @@ class RealTimeStreaming: try: if event_obj.get("type") != "response.done": return - response = event_obj.get("response", {}) + response = cast(Dict[str, Any], event_obj.get("response", {})) for item in response.get("output", []): if item.get("type") == "function_call": self.tool_calls.append( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index cac8f98cbb7..fdc821ef8bb 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -224,6 +224,7 @@ class TestVertexAIPassThroughHandler: # Mock request mock_request = Mock() + mock_request.state = None # Prevent Mock from returning a truthy _cached_headers mock_request.method = "POST" mock_request.headers = { "Authorization": "Bearer test-creds", @@ -323,6 +324,7 @@ class TestVertexAIPassThroughHandler: # Mock request mock_request = Mock() + mock_request.state = None # Prevent Mock from returning a truthy _cached_headers mock_request.method = "POST" mock_request.headers = { "Authorization": "Bearer test-creds", @@ -905,6 +907,7 @@ class TestVertexAIDiscoveryPassThroughHandler: # Mock request mock_request = Mock() + mock_request.state = None # Prevent Mock from returning a truthy _cached_headers mock_request.method = "POST" mock_request.headers = { "Authorization": "Bearer test-key", From 8bbbd1e46571d006ab2a7bdc728f79c11fd76103 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 13:00:59 +0530 Subject: [PATCH 29/49] Fix gaurdrail code qa --- .../block_code_execution/block_code_execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py index 99abdf28211..e76a02a6e4d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py @@ -25,6 +25,7 @@ from fastapi import HTTPException from litellm.integrations.custom_guardrail import ( CustomGuardrail, ModifyResponseException, + log_guardrail_information, ) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -536,6 +537,7 @@ class BlockCodeExecutionGuardrail(CustomGuardrail): detection_info={"language": language}, ) + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, From 14badde13c89e0e7f1feb761c2e6e791b77ce17a Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Thu, 26 Feb 2026 13:03:01 +0530 Subject: [PATCH 30/49] fix: custom auth budget issue --- litellm/proxy/_types.py | 44 +-- litellm/proxy/auth/user_api_key_auth.py | 276 +++++++++++++++++- .../proxy/hooks/model_max_budget_limiter.py | 153 ++++++++-- litellm/proxy/litellm_pre_call_utils.py | 47 +-- ...test_unit_test_max_model_budget_limiter.py | 106 +++++++ .../auth/test_custom_auth_end_user_budget.py | 59 ++++ 6 files changed, 615 insertions(+), 70 deletions(-) create mode 100644 tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4053d9d077b..ca424b46d4a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1107,7 +1107,9 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase): raise ValueError("args is required for stdio transport") elif transport in [MCPTransport.http, MCPTransport.sse]: if not values.get("url") and not values.get("spec_path"): - raise ValueError("url or spec_path is required for HTTP/SSE transport") + raise ValueError( + "url or spec_path is required for HTTP/SSE transport" + ) return values @model_validator(mode="before") @@ -1170,7 +1172,9 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase): raise ValueError("args is required for stdio transport") elif transport in [MCPTransport.http, MCPTransport.sse]: if not values.get("url") and not values.get("spec_path"): - raise ValueError("url or spec_path is required for HTTP/SSE transport") + raise ValueError( + "url or spec_path is required for HTTP/SSE transport" + ) return values @@ -1421,12 +1425,12 @@ class NewCustomerRequest(BudgetNewRequest): blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget spend: Optional[float] = None - allowed_model_region: Optional[AllowedModelRegion] = ( - None # require all user requests to use models in this specific region - ) - default_model: Optional[str] = ( - None # if no equivalent model in allowed region - default all requests to this model - ) + allowed_model_region: Optional[ + AllowedModelRegion + ] = None # require all user requests to use models in this specific region + default_model: Optional[ + str + ] = None # if no equivalent model in allowed region - default all requests to this model object_permission: Optional[LiteLLM_ObjectPermissionBase] = None @model_validator(mode="before") @@ -1449,12 +1453,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase): blocked: bool = False # allow/disallow requests for this end-user max_budget: Optional[float] = None budget_id: Optional[str] = None # give either a budget_id or max_budget - allowed_model_region: Optional[AllowedModelRegion] = ( - None # require all user requests to use models in this specific region - ) - default_model: Optional[str] = ( - None # if no equivalent model in allowed region - default all requests to this model - ) + allowed_model_region: Optional[ + AllowedModelRegion + ] = None # require all user requests to use models in this specific region + default_model: Optional[ + str + ] = None # if no equivalent model in allowed region - default all requests to this model object_permission: Optional[LiteLLM_ObjectPermissionBase] = None @@ -2279,6 +2283,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): end_user_tpm_limit: Optional[int] = None end_user_rpm_limit: Optional[int] = None end_user_max_budget: Optional[float] = None + end_user_model_max_budget: Optional[dict] = None # Organization Params organization_max_budget: Optional[float] = None @@ -3067,7 +3072,9 @@ class SpendLogsMetadata(TypedDict): str ] # S3/GCS object key for cold storage retrieval litellm_overhead_time_ms: Optional[float] # LiteLLM overhead time in milliseconds - attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded) + attempted_retries: Optional[ + int + ] # Number of retries attempted (0 = first attempt succeeded) max_retries: Optional[int] # Max retries configured for this request cost_breakdown: Optional[ CostBreakdown @@ -4127,10 +4134,10 @@ class SpendUpdateQueueItem(TypedDict, total=False): class ToolDiscoveryQueueItem(TypedDict, total=False): tool_name: str - origin: Optional[str] # MCP server name or "user_defined" + origin: Optional[str] # MCP server name or "user_defined" created_by: Optional[str] - key_hash: Optional[str] # hash of virtual key that triggered discovery - team_id: Optional[str] # team that triggered discovery + key_hash: Optional[str] # hash of virtual key that triggered discovery + team_id: Optional[str] # team that triggered discovery key_alias: Optional[str] # human-readable key alias @@ -4154,6 +4161,7 @@ class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): """Table for managing vector stores with target_model_names support.""" + unified_resource_id: str resource_object: Optional[Any] = None # VectorStoreCreateResponse model_mappings: Dict[str, str] diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 8f17440773a..5c48ea5eb9f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -183,6 +183,9 @@ def _apply_budget_limits_to_end_user_params( if budget_info.max_budget is not None: end_user_params["end_user_max_budget"] = budget_info.max_budget + if budget_info.model_max_budget is not None: + end_user_params["end_user_model_max_budget"] = budget_info.model_max_budget + verbose_proxy_logger.debug(f"Applied budget limits to end user {end_user_id}") @@ -237,6 +240,9 @@ def update_valid_token_with_end_user_params( valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit") valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit") valid_token.allowed_model_region = end_user_params.get("allowed_model_region") + valid_token.end_user_model_max_budget = end_user_params.get( + "end_user_model_max_budget" + ) return valid_token @@ -493,13 +499,29 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 request=request, api_key=api_key, user_custom_auth=user_custom_auth ) if response is not None and isinstance(response, UserAPIKeyAuth): - return UserAPIKeyAuth.model_validate(response) + validated = UserAPIKeyAuth.model_validate(response) + validated = await _run_post_custom_auth_checks( + valid_token=validated, + request=request, + request_data=request_data, + route=route, + parent_otel_span=parent_otel_span, + ) + return validated elif response is not None and isinstance(response, str): api_key = response custom_auth_api_key = True elif user_custom_auth is not None: response = await user_custom_auth(request=request, api_key=api_key) # type: ignore - return UserAPIKeyAuth.model_validate(response) + validated = UserAPIKeyAuth.model_validate(response) + validated = await _run_post_custom_auth_checks( + valid_token=validated, + request=request, + request_data=request_data, + route=route, + parent_otel_span=parent_otel_span, + ) + return validated ### LITELLM-DEFINED AUTH FUNCTION ### #### IF JWT #### @@ -593,9 +615,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_id=user_id, team_id=team_id, team_alias=( - team_object.team_alias - if team_object is not None - else None + team_object.team_alias if team_object is not None else None ), team_metadata=team_object.metadata if team_object is not None @@ -846,7 +866,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) valid_token.parent_otel_span = parent_otel_span if _end_user_object is not None: - valid_token.end_user_object_permission = _end_user_object.object_permission + valid_token.end_user_object_permission = ( + _end_user_object.object_permission + ) return valid_token @@ -954,7 +976,11 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if isinstance( api_key, str ): # if generated token, make sure it starts with sk-. - _masked_key = "{}****{}".format(api_key[:4], api_key[-4:]) if len(api_key) > 8 else "****" + _masked_key = ( + "{}****{}".format(api_key[:4], api_key[-4:]) + if len(api_key) > 8 + else "****" + ) assert api_key.startswith( "sk-" ), "LiteLLM Virtual Key expected. Received={}, expected to start with 'sk-'.".format( @@ -1201,6 +1227,21 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 model=current_model, ) + # Check 5b. End-user model max budget + end_user_mmb = valid_token.end_user_model_max_budget + if ( + end_user_mmb is not None + and isinstance(end_user_mmb, dict) + and len(end_user_mmb) > 0 + and current_model is not None + and valid_token.end_user_id is not None + ): + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_mmb, + model=current_model, + ) + # Check 6: Additional Common Checks across jwt + key auth if valid_token.team_id is not None: try: @@ -1304,9 +1345,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if _end_user_object is not None: valid_token_dict.update(end_user_params) - valid_token_dict["end_user_object_permission"] = ( - _end_user_object.object_permission - ) + valid_token_dict[ + "end_user_object_permission" + ] = _end_user_object.object_permission # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions # sso/login, ui/login, /key functions and /user functions @@ -1492,3 +1533,218 @@ def _update_key_budget_with_temp_budget_increase( temp_budget_increase = _get_temp_budget_increase(valid_token) or 0.0 valid_token.max_budget = valid_token.max_budget + temp_budget_increase return valid_token + + +async def _lookup_end_user_and_apply_budget( + valid_token: UserAPIKeyAuth, + route: str, + parent_otel_span: Optional[Span], + prisma_client, + user_api_key_cache, + proxy_logging_obj, +): + """Look up end_user from DB and apply budget limits to valid_token.""" + end_user_object = None + try: + end_user_object = await get_end_user_object( + end_user_id=valid_token.end_user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + route=route, + ) + if end_user_object is not None: + end_user_params = { + "end_user_id": valid_token.end_user_id, + "allowed_model_region": end_user_object.allowed_model_region, + } + if end_user_object.litellm_budget_table is not None: + _apply_budget_limits_to_end_user_params( + end_user_params=end_user_params, + budget_info=end_user_object.litellm_budget_table, + end_user_id=valid_token.end_user_id, + ) + valid_token = update_valid_token_with_end_user_params( + valid_token=valid_token, end_user_params=end_user_params + ) + elif litellm.max_end_user_budget_id is not None: + from litellm.proxy.auth.auth_checks import get_default_end_user_budget + + default_budget = await get_default_end_user_budget( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + ) + if default_budget is not None: + end_user_params = {"end_user_id": valid_token.end_user_id} + _apply_budget_limits_to_end_user_params( + end_user_params=end_user_params, + budget_info=default_budget, + end_user_id=valid_token.end_user_id, + ) + valid_token = update_valid_token_with_end_user_params( + valid_token=valid_token, end_user_params=end_user_params + ) + except Exception as e: + if isinstance(e, litellm.BudgetExceededError): + raise e + verbose_proxy_logger.debug(f"Unable to find user in db. Error - {str(e)}") + return valid_token, end_user_object + + +async def _run_post_custom_auth_checks( + valid_token: UserAPIKeyAuth, + request: Request, + request_data: dict, + route: str, + parent_otel_span: Optional[Span], +) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import ( + prisma_client, + user_api_key_cache, + proxy_logging_obj, + general_settings, + llm_router, + model_max_budget_limiter, + ) + + # 1. Look up end_user object from DB if end_user_id is set + end_user_object = None + if valid_token.end_user_id is not None: + valid_token, end_user_object = await _lookup_end_user_and_apply_budget( + valid_token=valid_token, + route=route, + parent_otel_span=parent_otel_span, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + # 2. Check token expiry + if valid_token.expires is not None: + current_time = datetime.now(timezone.utc) + if isinstance(valid_token.expires, datetime): + expiry_time = valid_token.expires + else: + expiry_time = datetime.fromisoformat(valid_token.expires) + if ( + expiry_time.tzinfo is None + or expiry_time.tzinfo.utcoffset(expiry_time) is None + ): + expiry_time = expiry_time.replace(tzinfo=timezone.utc) + if expiry_time < current_time: + raise ProxyException( + message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", + type=ProxyErrorTypes.expired_key, + code=400, + param=abbreviate_api_key(api_key=valid_token.token) + if valid_token.token + else "", + ) + + current_model = request_data.get("model", None) + + # 3. Check key-level model_max_budget + max_budget_per_model = valid_token.model_max_budget + if ( + max_budget_per_model is not None + and isinstance(max_budget_per_model, dict) + and len(max_budget_per_model) > 0 + and current_model is not None + and valid_token.token is not None + ): + await model_max_budget_limiter.is_key_within_model_budget( + user_api_key_dict=valid_token, + model=current_model, + ) + + # 4. Check end-user model_max_budget + end_user_mmb = valid_token.end_user_model_max_budget + if ( + end_user_mmb is not None + and isinstance(end_user_mmb, dict) + and len(end_user_mmb) > 0 + and current_model is not None + and valid_token.end_user_id is not None + ): + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_mmb, + model=current_model, + ) + + # 5. Look up user object if user_id is set + user_object = None + if valid_token.user_id is not None: + try: + user_object = await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception: + # If user_role is PROXY_ADMIN on the token, create a synthetic user object + # so that admin route checks pass for custom auth + if valid_token.user_role == LitellmUserRoles.PROXY_ADMIN: + user_object = LiteLLM_UserTable( + user_id=valid_token.user_id, + user_role=LitellmUserRoles.PROXY_ADMIN, + spend=0.0, + ) + + # 6. Run common checks + if valid_token.team_id is not None: + try: + _team_obj = await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException: + _team_obj = LiteLLM_TeamTableCachedObj( + team_id=valid_token.team_id, + max_budget=valid_token.team_max_budget, + soft_budget=valid_token.team_soft_budget, + spend=valid_token.team_spend, + tpm_limit=valid_token.team_tpm_limit, + rpm_limit=valid_token.team_rpm_limit, + blocked=valid_token.team_blocked, + models=valid_token.team_models, + metadata=valid_token.team_metadata, + object_permission_id=valid_token.team_object_permission_id, + ) + else: + _team_obj = None + + _project_obj = None + if valid_token.project_id is not None: + _project_obj = await get_project_object( + project_id=valid_token.project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + _ = await common_checks( + request=request, + request_body=request_data, + team_object=_team_obj, + user_object=user_object, + end_user_object=end_user_object, + general_settings=general_settings, + global_proxy_spend=None, + route=route, + llm_router=llm_router, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + skip_budget_checks=False, + project_object=_project_obj, + ) + + return valid_token diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index b8c073dd061..5e48ef2879e 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -15,6 +15,7 @@ from litellm.types.utils import ( ) VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX = "virtual_key_spend" +END_USER_SPEND_CACHE_KEY_PREFIX = "end_user_model_spend" class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): @@ -83,6 +84,81 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return True + async def is_end_user_within_model_budget( + self, + end_user_id: str, + end_user_model_max_budget: dict, + model: str, + ) -> bool: + """ + Check if the end_user is within the model budget + + Raises: + BudgetExceededError: If the end_user has exceeded the model budget + """ + internal_model_max_budget: GenericBudgetConfigType = {} + + for _model, _budget_info in end_user_model_max_budget.items(): + internal_model_max_budget[_model] = BudgetConfig(**_budget_info) + + verbose_proxy_logger.debug( + "end_user internal_model_max_budget %s", + json.dumps(internal_model_max_budget, indent=4, default=str), + ) + + # check if current model is in internal_model_max_budget + _current_model_budget_info = self._get_request_model_budget_config( + model=model, internal_model_max_budget=internal_model_max_budget + ) + if _current_model_budget_info is None: + verbose_proxy_logger.debug( + f"Model {model} not found in end_user_model_max_budget" + ) + return True + + # check if current model is within budget + if ( + _current_model_budget_info.max_budget + and _current_model_budget_info.max_budget > 0 + ): + _current_spend = await self._get_end_user_spend_for_model( + end_user_id=end_user_id, + model=model, + key_budget_config=_current_model_budget_info, + ) + if ( + _current_spend is not None + and _current_model_budget_info.max_budget is not None + and _current_spend > _current_model_budget_info.max_budget + ): + raise litellm.BudgetExceededError( + message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}", + current_cost=_current_spend, + max_budget=_current_model_budget_info.max_budget, + ) + + return True + + async def _get_end_user_spend_for_model( + self, + end_user_id: str, + model: str, + key_budget_config: BudgetConfig, + ) -> Optional[float]: + # 1. model: directly look up `model` + end_user_model_spend_cache_key = f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{key_budget_config.budget_duration}" + _current_spend = await self.dual_cache.async_get_cache( + key=end_user_model_spend_cache_key, + ) + + if _current_spend is None: + # 2. If 1, does not exist, check if passed as {custom_llm_provider}/model + end_user_model_spend_cache_key = f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{self._get_model_without_custom_llm_provider(model)}:{key_budget_config.budget_duration}" + _current_spend = await self.dual_cache.async_get_cache( + key=end_user_model_spend_cache_key, + ) + return _current_spend + async def _get_virtual_key_spend_for_model( self, user_api_key_hash: Optional[str], @@ -163,46 +239,77 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): user_api_key_model_max_budget: Optional[dict] = _metadata.get( "user_api_key_model_max_budget", None ) + user_api_key_end_user_model_max_budget: Optional[dict] = _metadata.get( + "user_api_key_end_user_model_max_budget", None + ) if ( user_api_key_model_max_budget is None or len(user_api_key_model_max_budget) == 0 + ) and ( + user_api_key_end_user_model_max_budget is None + or len(user_api_key_end_user_model_max_budget) == 0 ): verbose_proxy_logger.debug( - "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event because user_api_key_model_max_budget is None or empty. `user_api_key_model_max_budget`=%s", - user_api_key_model_max_budget, + "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event because user_api_key_model_max_budget and user_api_key_end_user_model_max_budget are None or empty." ) return + response_cost: float = standard_logging_payload.get("response_cost", 0) model = standard_logging_payload.get("model") virtual_key = standard_logging_payload.get("metadata", {}).get( "user_api_key_hash" ) + end_user_id = standard_logging_payload.get( + "end_user" + ) or standard_logging_payload.get("metadata", {}).get( + "user_api_key_end_user_id" + ) - if virtual_key is None or model is None: + if model is None: return - # Resolve per-model budget config (same logic as is_key_within_model_budget) - internal_model_max_budget: GenericBudgetConfigType = {} - for _model, _budget_info in user_api_key_model_max_budget.items(): - internal_model_max_budget[_model] = BudgetConfig(**_budget_info) - key_budget_config = self._get_request_model_budget_config( - model=model, internal_model_max_budget=internal_model_max_budget - ) - if key_budget_config is None or not key_budget_config.budget_duration: - verbose_proxy_logger.debug( - "Not incrementing model spend: no budget config or budget_duration for model=%s", - model, + if ( + virtual_key is not None + and user_api_key_model_max_budget is not None + and len(user_api_key_model_max_budget) > 0 + ): + internal_model_max_budget: GenericBudgetConfigType = {} + for _model, _budget_info in user_api_key_model_max_budget.items(): + internal_model_max_budget[_model] = BudgetConfig(**_budget_info) + key_budget_config = self._get_request_model_budget_config( + model=model, internal_model_max_budget=internal_model_max_budget ) - return + if key_budget_config is not None and key_budget_config.budget_duration: + virtual_spend_key = f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{key_budget_config.budget_duration}" + virtual_start_time_key = f"virtual_key_budget_start_time:{virtual_key}" + await self._increment_spend_for_key( + budget_config=key_budget_config, + spend_key=virtual_spend_key, + start_time_key=virtual_start_time_key, + response_cost=response_cost, + ) + + if ( + end_user_id is not None + and user_api_key_end_user_model_max_budget is not None + and len(user_api_key_end_user_model_max_budget) > 0 + ): + internal_model_max_budget: GenericBudgetConfigType = {} + for _model, _budget_info in user_api_key_end_user_model_max_budget.items(): + internal_model_max_budget[_model] = BudgetConfig(**_budget_info) + key_budget_config = self._get_request_model_budget_config( + model=model, internal_model_max_budget=internal_model_max_budget + ) + if key_budget_config is not None and key_budget_config.budget_duration: + end_user_spend_key = f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{key_budget_config.budget_duration}" + end_user_start_time_key = f"end_user_budget_start_time:{end_user_id}" + await self._increment_spend_for_key( + budget_config=key_budget_config, + spend_key=end_user_spend_key, + start_time_key=end_user_start_time_key, + response_cost=response_cost, + ) - virtual_spend_key = f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{key_budget_config.budget_duration}" - virtual_start_time_key = f"virtual_key_budget_start_time:{virtual_key}" - await self._increment_spend_for_key( - budget_config=key_budget_config, - spend_key=virtual_spend_key, - start_time_key=virtual_start_time_key, - response_cost=response_cost, - ) verbose_proxy_logger.debug( "current state of in memory cache %s", json.dumps( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 52f0b1d46e9..a8542df9420 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -10,10 +10,15 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.proxy._types import (AddTeamCallback, CommonProxyErrors, - LitellmDataForBackendLLMCall, - LitellmUserRoles, SpecialHeaders, - TeamCallbackMetadata, UserAPIKeyAuth) +from litellm.proxy._types import ( + AddTeamCallback, + CommonProxyErrors, + LitellmDataForBackendLLMCall, + LitellmUserRoles, + SpecialHeaders, + TeamCallbackMetadata, + UserAPIKeyAuth, +) from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers # Cache special headers as a frozenset for O(1) lookup performance @@ -23,9 +28,12 @@ _SPECIAL_HEADERS_CACHE = frozenset( from litellm.router import Router from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes -from litellm.types.utils import (LlmProviders, ProviderSpecificHeader, - StandardLoggingUserAPIKeyMetadata, - SupportedCacheControls) +from litellm.types.utils import ( + LlmProviders, + ProviderSpecificHeader, + StandardLoggingUserAPIKeyMetadata, + SupportedCacheControls, +) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL @@ -654,7 +662,8 @@ class LiteLLMProxyRequestSetup: return data from litellm.proxy._types import ( LiteLLM_ManagementEndpoint_MetadataFields, - LiteLLM_ManagementEndpoint_MetadataFields_Premium) + LiteLLM_ManagementEndpoint_MetadataFields_Premium, + ) # ignore any special fields added_metadata = {} @@ -1025,6 +1034,9 @@ async def add_litellm_data_to_request( # noqa: PLR0915 data[_metadata_variable_name][ "user_api_key_model_max_budget" ] = user_api_key_dict.model_max_budget + data[_metadata_variable_name][ + "user_api_key_end_user_model_max_budget" + ] = user_api_key_dict.end_user_model_max_budget # User spend, budget - used by prometheus.py # Follow same pattern as team and API key budgets @@ -1479,8 +1491,7 @@ async def move_guardrails_to_metadata( # Only check policy engine if no local config (avoid import + registry lookup) if not (has_key_config or has_team_config or has_request_config): - from litellm.proxy.policy_engine.policy_registry import \ - get_policy_registry + from litellm.proxy.policy_engine.policy_registry import get_policy_registry if not get_policy_registry().is_initialized(): # Nothing configured anywhere - clean up request body fields and return @@ -1544,16 +1555,14 @@ async def move_guardrails_to_metadata( def _is_policy_version_id(s: str) -> bool: """Return True if string is a policy version ID (starts with policy_ prefix).""" - from litellm.proxy.policy_engine.policy_registry import \ - POLICY_VERSION_ID_PREFIX + from litellm.proxy.policy_engine.policy_registry import POLICY_VERSION_ID_PREFIX return isinstance(s, str) and s.startswith(POLICY_VERSION_ID_PREFIX) def _extract_policy_id(s: str) -> Optional[str]: """Extract raw UUID from policy_ string, or None if not a valid version ID.""" - from litellm.proxy.policy_engine.policy_registry import \ - POLICY_VERSION_ID_PREFIX + from litellm.proxy.policy_engine.policy_registry import POLICY_VERSION_ID_PREFIX if not _is_policy_version_id(s): return None @@ -1574,9 +1583,10 @@ def _match_and_track_policies( """ from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.callback_utils import ( - add_policy_sources_to_metadata, add_policy_to_applied_policies_header) - from litellm.proxy.policy_engine.attachment_registry import \ - get_attachment_registry + add_policy_sources_to_metadata, + add_policy_to_applied_policies_header, + ) + from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher # Get matching policies via attachments (with match reasons for attribution) @@ -1721,8 +1731,7 @@ async def add_guardrails_from_policy_engine( user_api_key_dict: The user's API key authentication info """ from litellm._logging import verbose_proxy_logger - from litellm.proxy.common_utils.http_parsing_utils import \ - get_tags_from_request_body + from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.types.proxy.policy_engine import PolicyMatchContext diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index 352db384c88..030d452e55f 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -158,3 +158,109 @@ async def test_async_log_success_event_uses_per_model_budget_duration(budget_lim f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{budget_duration}" ) assert call_kwargs["response_cost"] == 0.05 + + +# Test is_end_user_within_model_budget +@pytest.mark.asyncio +async def test_is_end_user_within_model_budget(budget_limiter): + # Test when model is within budget + with patch.object( + budget_limiter, "_get_end_user_spend_for_model", return_value=50.0 + ): + assert ( + await budget_limiter.is_end_user_within_model_budget( + "test-user", + {"gpt-4": {"budget_limit": 100.0, "time_period": "1d"}}, + "gpt-4", + ) + is True + ) + + # Test when model exceeds budget + with patch.object( + budget_limiter, "_get_end_user_spend_for_model", return_value=150.0 + ): + with pytest.raises(litellm.BudgetExceededError): + await budget_limiter.is_end_user_within_model_budget( + "test-user", + {"gpt-4": {"budget_limit": 100.0, "time_period": "1d"}}, + "gpt-4", + ) + + # Test model not in budget config + assert ( + await budget_limiter.is_end_user_within_model_budget( + "test-user", + {"gpt-4": {"budget_limit": 100.0, "time_period": "1d"}}, + "non-existent", + ) + is True + ) + + +# Test _get_end_user_spend_for_model +@pytest.mark.asyncio +async def test_get_end_user_spend_for_model(budget_limiter): + budget_config = GenericBudgetInfo(budget_limit=100.0, time_period="1d") + + # Mock cache get + with patch.object(budget_limiter.dual_cache, "async_get_cache", return_value=50.0): + spend = await budget_limiter._get_end_user_spend_for_model( + end_user_id="test-user", model="gpt-4", key_budget_config=budget_config + ) + assert spend == 50.0 + + # Test with provider prefix + spend = await budget_limiter._get_end_user_spend_for_model( + end_user_id="test-user", + model="openai/gpt-4", + key_budget_config=budget_config, + ) + assert spend == 50.0 + + +@pytest.mark.asyncio +async def test_async_log_success_event_uses_end_user_model_budget_duration( + budget_limiter, +): + """ + async_log_success_event must use the per-model budget_duration for the end user cache key + """ + from litellm.proxy.hooks.model_max_budget_limiter import ( + END_USER_SPEND_CACHE_KEY_PREFIX, + ) + + end_user_id = "test-user" + model = "gpt-4" + budget_duration = "1d" + user_api_key_end_user_model_max_budget = { + model: {"budget_limit": 100.0, "time_period": budget_duration}, + } + kwargs = { + "standard_logging_object": { + "response_cost": 0.05, + "model": model, + "end_user": end_user_id, + "metadata": {"user_api_key_end_user_id": end_user_id}, + }, + "litellm_params": { + "metadata": { + "user_api_key_end_user_model_max_budget": user_api_key_end_user_model_max_budget + }, + }, + } + with patch.object( + budget_limiter, + "_increment_spend_for_key", + new_callable=AsyncMock, + ) as mock_increment: + await budget_limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_increment.assert_awaited_once() + call_kwargs = mock_increment.call_args.kwargs + spend_key = call_kwargs["spend_key"] + assert spend_key == ( + f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{budget_duration}" + ) + assert call_kwargs["response_cost"] == 0.05 diff --git a/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py new file mode 100644 index 00000000000..246048e4bce --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py @@ -0,0 +1,59 @@ +import pytest +from unittest.mock import AsyncMock, patch +import litellm +from litellm.proxy.auth.user_api_key_auth import _run_post_custom_auth_checks +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.mark.asyncio +async def test_custom_auth_run_post_custom_auth_checks_without_end_user_id(): + # Test backwards compatibility + valid_token = UserAPIKeyAuth(token="test_token") + + with patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ) as mock_common: + mock_common.return_value = True + result = await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data={}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + assert result.token == "test_token" + assert getattr(result, "end_user_id", None) is None + mock_common.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_custom_auth_run_post_custom_auth_checks_with_end_user_budget_exceeded(): + valid_token = UserAPIKeyAuth( + token="test_token", + end_user_id="test_user", + end_user_model_max_budget={ + "gpt-4": {"budget_limit": 10.0, "time_period": "1d"} + }, + ) + request_data = {"model": "gpt-4"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ): + with patch( + "litellm.proxy.proxy_server.model_max_budget_limiter.is_end_user_within_model_budget", + new_callable=AsyncMock, + ) as mock_budget_check: + mock_budget_check.side_effect = litellm.BudgetExceededError( + message="Exceeded budget", current_cost=20.0, max_budget=10.0 + ) + + with pytest.raises(litellm.BudgetExceededError): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_budget_check.assert_awaited_once() From 8f8ebbec8d96e15e73f2aee7f97c2f01358c0625 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 13:06:25 +0530 Subject: [PATCH 31/49] Fix test_vertex_passthrough_forwards_anthropic_beta_header --- .../test_vertex_passthrough_load_balancing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index d2fdb157c8d..eb4749549c2 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -253,6 +253,9 @@ async def test_vertex_passthrough_forwards_anthropic_beta_header(): "content-length": "1234", # Should be removed "host": "localhost:4000", # Should be removed }) + # Prevent MagicMock from auto-creating a truthy _cached_headers attribute, + # which would short-circuit _safe_get_request_headers before reading .headers + mock_request.state._cached_headers = None # Create mock vertex credentials mock_vertex_credentials = MagicMock() From 819581f6bfbe6a49c65e12be62b0fbd41e82b603 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 25 Feb 2026 23:43:13 -0800 Subject: [PATCH 32/49] fix(realtime): guardrails with pre_call/post_call mode now work on realtime WebSocket (#22161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(realtime): guardrails with pre_call/post_call mode now work on realtime WebSocket; return error directly to consumer * fix(realtime guardrails): address code review feedback - Restore session.update injection for audio/VAD path, but only when realtime_input_transcription guardrails are configured (not pre_call). Forward session.created to the client first so no error arrives before the client sees the session. - Change _swallow_next_response_create bool to int counter so consecutive blocked items are handled correctly. - Extract _build_litellm_metadata() helper to eliminate duplicated metadata-building logic across OpenAI/Azure/XAI provider branches. - Plumb litellm_metadata and user_api_key_dict to Azure and XAI handlers so guardrails work for those providers too. - Add tests for session.update injection, no-inject for pre_call-only, and consecutive-block counter. * simplify: remove response.create swallowing after guardrail block When an item is blocked, the error event is already sent to the client. The subsequent response.create from the client is fine to forward through — the LLM may respond to previous context which is acceptable behavior. Removing the swallow counter eliminates unnecessary state tracking. --- .../litellm_core_utils/realtime_streaming.py | 144 ++++++----- litellm/llms/azure/realtime/handler.py | 10 +- litellm/llms/openai/realtime/handler.py | 2 + litellm/realtime_api/main.py | 14 + .../test_realtime_streaming.py | 240 +++++++++++++++--- 5 files changed, 321 insertions(+), 89 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 5d7a5bfe318..231f3a975dc 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -43,6 +43,7 @@ class RealTimeStreaming: provider_config: Optional[BaseRealtimeConfig] = None, model: str = "", user_api_key_dict: Optional[Any] = None, + request_data: Optional[Dict] = None, ): self.websocket = websocket self.backend_ws = backend_ws @@ -68,6 +69,7 @@ class RealTimeStreaming: self.current_delta_type: Optional[ALL_DELTA_TYPES] = None self.session_configuration_request: Optional[str] = None self.user_api_key_dict = user_api_key_dict + self.request_data: Dict = request_data or {} def _should_store_message( self, @@ -231,14 +233,40 @@ class RealTimeStreaming: await self.backend_ws.send(message) def _has_realtime_guardrails(self) -> bool: - """Return True if any callback is registered for realtime_input_transcription.""" + """Return True if any callback is registered for realtime guardrail event types.""" + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + _realtime_event_types = [ + GuardrailEventHooks.realtime_input_transcription, + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + ] + return any( + isinstance(cb, CustomGuardrail) + and any( + cb.should_run_guardrail( + data=self.request_data, + event_type=et, + ) + for et in _realtime_event_types + ) + for cb in litellm.callbacks + ) + + def _has_audio_transcription_guardrails(self) -> bool: + """Return True if any callback needs to run on audio transcriptions (VAD path). + + When this returns True, we inject a session.update to disable the LLM's + auto-response so the guardrail can gate it first. + """ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks return any( isinstance(cb, CustomGuardrail) and cb.should_run_guardrail( - data={}, + data=self.request_data, event_type=GuardrailEventHooks.realtime_input_transcription, ) for cb in litellm.callbacks @@ -258,17 +286,25 @@ class RealTimeStreaming: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks + _realtime_event_types = [ + GuardrailEventHooks.realtime_input_transcription, + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + ] + _check_data = {**self.request_data, "transcript": transcript} + _already_run: set = set() + for callback in litellm.callbacks: if not isinstance(callback, CustomGuardrail): continue - if ( - callback.should_run_guardrail( - data={"transcript": transcript}, - event_type=GuardrailEventHooks.realtime_input_transcription, - ) - is not True + if id(callback) in _already_run: + continue + if not any( + callback.should_run_guardrail(data=_check_data, event_type=et) + for et in _realtime_event_types ): continue + _already_run.add(id(callback)) try: await callback.apply_guardrail( inputs={"texts": [transcript], "images": []}, @@ -293,20 +329,15 @@ class RealTimeStreaming: safe_msg = str(detail) else: safe_msg = str(e) or "I'm sorry, that request was blocked by the content filter." - # Cancel any in-flight response before speaking the warning. - # This handles the race where create_response fired before we could intercept. - await self._send_to_backend(json.dumps({"type": "response.cancel"})) - # Ask the model to speak the warning — TTS audio plays naturally in the client - await self._send_to_backend( + # Return the error directly to the WebSocket consumer. + await self.websocket.send_text( json.dumps( { - "type": "response.create", - "response": { - "modalities": ["text", "audio"], - "instructions": ( - f"Say exactly and only: \"{safe_msg}\". " - "Do not add anything else." - ), + "type": "error", + "error": { + "type": "guardrail_violation", + "message": safe_msg, + "code": "content_policy_violation", }, } ) @@ -348,25 +379,25 @@ class RealTimeStreaming: if isinstance(transformed_response, list) else [transformed_response] ) - for event in events: - ## GUARDRAIL: inject create_response=false on session.created - if isinstance(event, dict) and event.get("type") == "session.created": - if self._has_realtime_guardrails(): - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) for event in events: event_str = json.dumps(event) + ## For audio/VAD guardrail path: forward session.created first, then inject. + if ( + isinstance(event, dict) + and event.get("type") == "session.created" + and self._has_audio_transcription_guardrails() + ): + self.store_message(event_str) + await self.websocket.send_text(event_str) + await self._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + ) + ) + continue ## GUARDRAIL: run on transcription events in provider_config path too if ( isinstance(event, dict) @@ -397,27 +428,26 @@ class RealTimeStreaming: try: event_obj = json.loads(raw_response) - if event_obj.get("type") == "session.created": - # If any realtime guardrails are registered, proactively - # set create_response=false so the LLM never auto-responds - # before our guardrail has a chance to run. - if self._has_realtime_guardrails(): - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "turn_detection": { - "type": "server_vad", - "create_response": False, - } - }, - } - ) - ) - verbose_logger.debug( - "[realtime guardrail] injected create_response=false into session" + # For audio/VAD guardrail path: once the session is ready, tell the backend + # not to auto-respond after VAD detects end-of-speech. We send the + # session.created to the client FIRST so the client is always in sync, then + # inject the session.update so a potential error from the backend doesn't + # arrive before the client sees session.created. + if ( + event_obj.get("type") == "session.created" + and self._has_audio_transcription_guardrails() + ): + self.store_message(raw_response) + await self.websocket.send_text(raw_response) + await self._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } ) + ) + return True if ( event_obj.get("type") diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index e533978e07a..8f4291ec271 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -6,13 +6,13 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast +from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..azure import AzureChatCompletion -from litellm._logging import verbose_proxy_logger # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01" @@ -77,6 +77,8 @@ class AzureOpenAIRealtime(AzureChatCompletion): client: Optional[Any] = None, timeout: Optional[float] = None, realtime_protocol: Optional[str] = None, + user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[dict] = None, ): import websockets from websockets.asyncio.client import ClientConnection @@ -101,7 +103,11 @@ class AzureOpenAIRealtime(AzureChatCompletion): ssl=ssl_context, ) as backend_ws: realtime_streaming = RealTimeStreaming( - websocket, cast(ClientConnection, backend_ws), logging_obj + websocket, + cast(ClientConnection, backend_ws), + logging_obj, + user_api_key_dict=user_api_key_dict, + request_data={"litellm_metadata": litellm_metadata or {}}, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index c2fccfc7289..05915e36a69 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -99,6 +99,7 @@ class OpenAIRealtime(OpenAIChatCompletion): timeout: Optional[float] = None, query_params: Optional[RealtimeQueryParams] = None, user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[dict] = None, **kwargs: Any, ): import websockets @@ -142,6 +143,7 @@ class OpenAIRealtime(OpenAIChatCompletion): cast(ClientConnection, backend_ws), logging_obj, user_api_key_dict=user_api_key_dict, + request_data={"litellm_metadata": litellm_metadata or {}}, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e4c8f648190..df49d4c54b2 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -32,6 +32,15 @@ vertex_llm_base = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() +def _build_litellm_metadata(kwargs: dict) -> dict: + """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" + metadata: dict = {**(kwargs.get("litellm_metadata") or {})} + guardrails = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or [] + if guardrails: + metadata["guardrails"] = guardrails + return metadata + + @wrapper_client async def _arealtime( model: str, @@ -134,6 +143,8 @@ async def _arealtime( timeout=timeout, logging_obj=litellm_logging_obj, realtime_protocol=realtime_protocol, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "openai": api_base = ( @@ -160,6 +171,7 @@ async def _arealtime( timeout=timeout, query_params=query_params, user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "bedrock": # Extract AWS parameters from kwargs @@ -217,6 +229,8 @@ async def _arealtime( client=None, timeout=timeout, query_params=query_params, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "vertex_ai": vertex_credentials = ( diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index aaaab95ce6f..8db626a4d33 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -416,32 +416,32 @@ async def test_realtime_guardrail_blocks_prompt_injection(): streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # ASSERT 1: no bare response.create was sent to backend (injection blocked). - # The only response.create allowed is the warning one (has "instructions" field). + # ASSERT 1: no response.create was sent to backend (injection blocked). sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] - bare_response_creates = [ + response_creates = [ e for e in sent_to_backend if e.get("type") == "response.create" - and "instructions" not in e.get("response", {}) ] - assert len(bare_response_creates) == 0, ( - f"Guardrail should prevent bare response.create for injected content, " - f"but got: {bare_response_creates}" + assert len(response_creates) == 0, ( + f"Guardrail should prevent response.create for injected content, " + f"but got: {response_creates}" ) - # ASSERT 2: warning response.create was sent to backend (to speak the block message) - warning_creates = [ - e for e in sent_to_backend - if e.get("type") == "response.create" - and "instructions" in e.get("response", {}) + # ASSERT 2: error event was sent directly to the client WebSocket + sent_to_client = [ + json.loads(c.args[0]) for c in client_ws.send_text.call_args_list + if c.args ] - assert len(warning_creates) > 0, ( - f"Backend should receive a response.create with warning instructions, " - f"but got: {sent_to_backend}" + error_events = [e for e in sent_to_client if e.get("type") == "error"] + assert len(error_events) == 1, ( + f"Expected one error event sent to client, got: {sent_to_client}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation", ( + f"Expected guardrail_violation error type, got: {error_events[0]}" ) litellm.callbacks = [] # cleanup @@ -514,11 +514,91 @@ async def test_realtime_guardrail_allows_clean_transcript(): @pytest.mark.asyncio -async def test_realtime_session_created_injects_create_response_false(): +async def test_realtime_text_input_guardrail_blocks_and_returns_error(): """ - Test that when session.created arrives from the backend and realtime guardrails - are registered, the proxy injects a session.update with create_response=False - so the LLM never auto-responds before the guardrail runs. + Test that when conversation.item.create arrives with text that triggers a guardrail, + the proxy blocks it (doesn't forward to backend) and returns an error event directly + to the client WebSocket. + """ + from fastapi import HTTPException + + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class BlockingGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + texts = inputs.get("texts", []) + for text in texts: + if "@" in text: + raise HTTPException( + status_code=403, + detail={"error": "email address detected"}, + ) + return inputs + + guardrail = BlockingGuardrail( + guardrail_name="email-blocker", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + item_create_msg = json.dumps({ + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [{"type": "input_text", "text": "My email is test@example.com"}], + }, + }) + + # Simulate the client sending a conversation.item.create with an email + client_ws.receive_text = AsyncMock( + side_effect=[ + item_create_msg, + Exception("connection closed"), # stop the loop + ] + ) + + await streaming.client_ack_messages() + + # ASSERT: error event was sent to client + assert client_ws.send_text.called, "Expected error to be sent to client websocket" + sent_texts = [json.loads(c.args[0]) for c in client_ws.send_text.call_args_list] + error_events = [e for e in sent_texts if e.get("type") == "error"] + assert len(error_events) == 1, f"Expected one error event, got: {sent_texts}" + assert error_events[0]["error"]["type"] == "guardrail_violation" + + # ASSERT: blocked item was NOT forwarded to the backend + sent_to_backend = [c.args[0] for c in backend_ws.send.call_args_list if c.args] + forwarded_items = [ + json.loads(m) for m in sent_to_backend + if isinstance(m, str) and json.loads(m).get("type") == "conversation.item.create" + ] + assert len(forwarded_items) == 0, ( + f"Blocked item should not be forwarded to backend, got: {forwarded_items}" + ) + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_text_input_guardrail_uses_pre_call_mode(): + """ + Test that _has_realtime_guardrails returns True for a guardrail configured with + pre_call mode (not just realtime_input_transcription). """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -529,7 +609,46 @@ async def test_realtime_session_created_injects_create_response_false(): return inputs guardrail = DummyGuardrail( - guardrail_name="dummy", + guardrail_name="pre-call-guardrail", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + backend_ws = MagicMock() + logging_obj = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + assert streaming._has_realtime_guardrails() is True, ( + "pre_call guardrail should be recognized as a realtime guardrail" + ) + # pre_call guardrail should NOT trigger the audio/VAD session.update injection + assert streaming._has_audio_transcription_guardrails() is False, ( + "pre_call guardrail should not trigger audio transcription guardrail path" + ) + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_session_created_injects_session_update_for_audio_guardrail(): + """ + Test that when an audio transcription guardrail is configured, a session.created + event from the backend triggers a session.update injection (create_response: false) + AFTER forwarding session.created to the client. This prevents the LLM from + auto-responding before the guardrail can run on the transcript. + """ + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class AudioGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + guardrail = AudioGuardrail( + guardrail_name="audio-guardrail", event_hook=GuardrailEventHooks.realtime_input_transcription, default_on=True, ) @@ -538,34 +657,95 @@ async def test_realtime_session_created_injects_create_response_false(): client_ws = MagicMock() client_ws.send_text = AsyncMock() - session_created_event = json.dumps({"type": "session.created"}).encode() + session_created_event = json.dumps( + {"type": "session.created", "session": {"id": "sess_abc"}} + ).encode() backend_ws = MagicMock() backend_ws.recv = AsyncMock( - side_effect=[ - session_created_event, - ConnectionClosed(None, None), - ] + side_effect=[session_created_event, ConnectionClosed(None, None)] ) backend_ws.send = AsyncMock() logging_obj = MagicMock() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # ASSERT: proxy injected session.update with create_response=False to backend + # session.created must be forwarded to the client + sent_to_client = [ + json.loads(c.args[0]) for c in client_ws.send_text.call_args_list if c.args + ] + session_created_events = [e for e in sent_to_client if e.get("type") == "session.created"] + assert len(session_created_events) == 1, ( + f"session.created should be forwarded to client, got: {sent_to_client}" + ) + + # session.update must be sent to the backend AFTER session.created was forwarded sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] assert len(session_updates) == 1, ( - f"Expected proxy to inject session.update, got: {sent_to_backend}" + f"Expected one session.update injected to backend, got: {sent_to_backend}" ) - td = session_updates[0]["session"]["turn_detection"] - assert td["create_response"] is False, ( - f"Expected create_response=False, got: {td}" + assert session_updates[0]["session"]["turn_detection"]["create_response"] is False + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_realtime_session_created_no_injection_for_pre_call_only(): + """ + Test that when only a pre_call guardrail is configured (no audio transcription), + session.created does NOT trigger the session.update injection. + """ + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + class PreCallGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + guardrail = PreCallGuardrail( + guardrail_name="pre-call-only", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + session_created_event = json.dumps( + {"type": "session.created", "session": {"id": "sess_xyz"}} + ).encode() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[session_created_event, ConnectionClosed(None, None)] + ) + backend_ws.send = AsyncMock() + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + await streaming.backend_to_client_send_messages() + + # No session.update should be injected + sent_to_backend = [ + json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args + ] + session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] + assert len(session_updates) == 0, ( + f"pre_call guardrail should NOT inject session.update, got: {sent_to_backend}" ) litellm.callbacks = [] # cleanup + + From 24fd841e83cef8f1ae503dc90c578665f1ac2efb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 13:16:09 +0530 Subject: [PATCH 33/49] Fix code qa --- docs/my-website/docs/proxy/config_settings.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 8dbebad884e..b694549cf40 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -796,6 +796,7 @@ router_settings: | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. | LITELLM_MASTER_KEY | Master key for proxy authentication +| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour) | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 @@ -991,6 +992,7 @@ router_settings: | TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150 | TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350 | TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4 +| TOOL_POLICY_CACHE_TTL_SECONDS | TTL in seconds for caching tool policy guardrail results. Default is 60 | UI_LOGO_PATH | Path to the logo image used in the UI | UI_PASSWORD | Password for accessing the UI | UI_USERNAME | Username for accessing the UI From 965ca117bc8a9528d32fa98b6bdfc503ac951811 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 25 Feb 2026 23:49:03 -0800 Subject: [PATCH 34/49] feat(realtime guardrails): end_session_after_n_fails + Endpoint Settings wizard step (#22165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(realtime guardrails): end_session_after_n_fails + Endpoint Settings wizard step Adds per-session violation thresholds and an optional endpoint-settings step to the guardrail wizard for /v1/realtime. Backend: - Add end_session_after_n_fails, on_violation, realtime_violation_message fields to BaseLitellmParams (no DB migration — stored in existing JSON column) - Store same fields on CustomGuardrail instance attrs - Pass through in litellm_content_filter initializer - Track _violation_count per RealTimeStreaming session; close backend_ws when on_violation=end_session OR violation count >= end_session_after_n_fails - Use realtime_violation_message as the spoken text (falls back to guardrail error string if not configured) UI (add_guardrail_form.tsx): - Rename "Default Categories" step to "Topics" - Add step 5 "Endpoint Settings (Optional)" for content filter guardrails - Call type dropdown shows /v1/realtime - Settings are in a collapsed accordion (closed by default) - "End session after X violations" + on_violation radio + spoken message field Tests: 2 new tests in test_realtime_streaming.py - test_end_session_after_n_fails_closes_connection - test_on_violation_end_session_closes_on_first_fail * fix(test): move inline imports to module level in realtime streaming tests * Update ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 9 + .../litellm_core_utils/realtime_streaming.py | 27 ++- .../litellm_content_filter/__init__.py | 8 +- litellm/types/guardrails.py | 15 ++ .../test_realtime_streaming.py | 107 ++++++++++++ .../guardrails/add_guardrail_form.tsx | 156 +++++++++++++++++- 6 files changed, 316 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index bf330944ef8..5d11fd68475 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -92,6 +92,9 @@ class CustomGuardrail(CustomLogger): mask_request_content: bool = False, mask_response_content: bool = False, violation_message_template: Optional[str] = None, + end_session_after_n_fails: Optional[int] = None, + on_violation: Optional[str] = None, + realtime_violation_message: Optional[str] = None, **kwargs, ): """ @@ -104,6 +107,9 @@ class CustomGuardrail(CustomLogger): default_on: If True, the guardrail will be run by default on all requests mask_request_content: If True, the guardrail will mask the request content mask_response_content: If True, the guardrail will mask the response content + end_session_after_n_fails: For /v1/realtime sessions, end the session after this many violations + on_violation: For /v1/realtime sessions, 'warn' or 'end_session' + realtime_violation_message: Message the bot speaks aloud when a /v1/realtime guardrail fires """ self.guardrail_name = guardrail_name self.supported_event_hooks = supported_event_hooks @@ -114,6 +120,9 @@ class CustomGuardrail(CustomLogger): self.mask_request_content: bool = mask_request_content self.mask_response_content: bool = mask_response_content self.violation_message_template: Optional[str] = violation_message_template + self.end_session_after_n_fails: Optional[int] = end_session_after_n_fails + self.on_violation: Optional[str] = on_violation + self.realtime_violation_message: Optional[str] = realtime_violation_message if supported_event_hooks: ## validate event_hook is in supported_event_hooks diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 231f3a975dc..6ba1b48c647 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -70,6 +70,8 @@ class RealTimeStreaming: self.session_configuration_request: Optional[str] = None self.user_api_key_dict = user_api_key_dict self.request_data: Dict = request_data or {} + # Violation counter for end_session_after_n_fails support + self._violation_count: int = 0 def _should_store_message( self, @@ -329,6 +331,10 @@ class RealTimeStreaming: safe_msg = str(detail) else: safe_msg = str(e) or "I'm sorry, that request was blocked by the content filter." + + # Use realtime_violation_message if configured; fall back to guardrail error text. + error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg + # Return the error directly to the WebSocket consumer. await self.websocket.send_text( json.dumps( @@ -336,14 +342,31 @@ class RealTimeStreaming: "type": "error", "error": { "type": "guardrail_violation", - "message": safe_msg, + "message": error_msg, "code": "content_policy_violation", }, } ) ) + + self._violation_count += 1 + end_session_after: Optional[int] = getattr( + callback, "end_session_after_n_fails", None + ) + should_end = getattr(callback, "on_violation", None) == "end_session" or ( + end_session_after is not None + and self._violation_count >= end_session_after + ) + if should_end: + verbose_logger.warning( + "[realtime guardrail] ending session after violation %d", + self._violation_count, + ) + await self.backend_ws.close() + verbose_logger.warning( - "[realtime guardrail] BLOCKED transcript: %r", + "[realtime guardrail] BLOCKED transcript (violation %d): %r", + self._violation_count, transcript[:80], ) return True diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/__init__.py index d9a44094ad2..111f8dc783a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/__init__.py @@ -1,8 +1,9 @@ from typing import TYPE_CHECKING, Optional import litellm -from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import \ - ContentFilterGuardrail +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, +) from litellm.types.guardrails import SupportedGuardrailIntegrations if TYPE_CHECKING: @@ -46,6 +47,9 @@ def initialize_guardrail( competitor_intent_config=getattr( litellm_params, "competitor_intent_config", None ), + end_session_after_n_fails=getattr(litellm_params, "end_session_after_n_fails", None), + on_violation=getattr(litellm_params, "on_violation", None), + realtime_violation_message=getattr(litellm_params, "realtime_violation_message", None), ) litellm.logging_callback_manager.add_litellm_callback(content_filter_guardrail) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index df411f220a0..0e71f20700e 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -649,6 +649,21 @@ class BaseLitellmParams( description="Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", ) + ################## Realtime API params ################ + ######################################################## + end_session_after_n_fails: Optional[int] = Field( + default=None, + description="For /v1/realtime sessions: automatically close the session after this many guardrail violations.", + ) + on_violation: Optional[Literal["warn", "end_session"]] = Field( + default=None, + description="For /v1/realtime sessions: 'warn' speaks the violation message and continues; 'end_session' speaks the message and closes the connection.", + ) + realtime_violation_message: Optional[str] = Field( + default=None, + description="The message the bot speaks aloud when a /v1/realtime guardrail fires. Falls back to violation_message_template if not set.", + ) + # Model Armor params template_id: Optional[str] = Field( default=None, description="The ID of your Model Armor template" diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 8db626a4d33..bcda3c7bfac 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -6,17 +6,31 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from websockets.exceptions import ConnectionClosed +import litellm + sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ( OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, ) +def _make_transcript_event(text: str, item_id: str = "item_x") -> bytes: + return json.dumps( + { + "type": "conversation.item.input_audio_transcription.completed", + "transcript": text, + "item_id": item_id, + } + ).encode() + + def test_realtime_streaming_store_message(): # Setup websocket = MagicMock() @@ -749,3 +763,96 @@ async def test_realtime_session_created_no_injection_for_pre_call_only(): litellm.callbacks = [] # cleanup +@pytest.mark.asyncio +async def test_end_session_after_n_fails_closes_connection(): + """ + Test that end_session_after_n_fails=2 closes the backend websocket after + the second guardrail violation in a session. + """ + + class BadWordGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for text in inputs.get("texts", []): + if "blocked" in text.lower(): + raise ValueError("Content blocked by guardrail.") + return inputs + + guardrail = BadWordGuardrail( + guardrail_name="bad_word_guard", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + end_session_after_n_fails=2, + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + _make_transcript_event("this is blocked"), # violation 1 — warn + _make_transcript_event("also blocked again"), # violation 2 — end session + ConnectionClosed(None, None), + ] + ) + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + await streaming.backend_to_client_send_messages() + + assert backend_ws.close.called, "Expected backend_ws.close() to be called after 2 violations" + assert streaming._violation_count == 2 + + litellm.callbacks = [] # cleanup + + +@pytest.mark.asyncio +async def test_on_violation_end_session_closes_on_first_fail(): + """ + Test that on_violation='end_session' closes the session immediately on the + first violation, regardless of end_session_after_n_fails. + """ + + class TopicGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for text in inputs.get("texts", []): + if "stock" in text.lower(): + raise ValueError("Topic not allowed: financial advice.") + return inputs + + guardrail = TopicGuardrail( + guardrail_name="topic_guard", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + on_violation="end_session", + ) + litellm.callbacks = [guardrail] + + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + _make_transcript_event("What stock should I buy today?", item_id="item_y"), + ConnectionClosed(None, None), + ] + ) + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + await streaming.backend_to_client_send_messages() + + assert backend_ws.close.called, "Expected session to close immediately with on_violation=end_session" + assert streaming._violation_count == 1 + + litellm.callbacks = [] # cleanup diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index f8c8cc17aef..3bdd18f2650 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -118,6 +118,14 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const [pendingCategorySelection, setPendingCategorySelection] = useState(""); const [competitorIntentEnabled, setCompetitorIntentEnabled] = useState(false); const [competitorIntentConfig, setCompetitorIntentConfig] = useState(null); + + // Endpoint Settings state (step 5) + const [selectedEndpointType, setSelectedEndpointType] = useState(""); + const [endSessionAfterNFails, setEndSessionAfterNFails] = useState(undefined); + const [onViolation, setOnViolation] = useState<"warn" | "end_session">("warn"); + const [realtimeViolationMessage, setRealtimeViolationMessage] = useState(""); + const [endpointSettingsOpen, setEndpointSettingsOpen] = useState(false); + const [toolPermissionConfig, setToolPermissionConfig] = useState({ rules: [], default_action: "deny", @@ -361,6 +369,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a on_disallowed_action: "block", violation_message_template: "", }); + setSelectedEndpointType(""); + setEndSessionAfterNFails(undefined); + setOnViolation("warn"); + setRealtimeViolationMessage(""); + setEndpointSettingsOpen(false); setCurrentStep(0); }; @@ -504,6 +517,19 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a } } + // Endpoint Settings (realtime) — content filter only + if (shouldRenderContentFilterConfigSettings(values.provider)) { + if (endSessionAfterNFails !== undefined && endSessionAfterNFails > 0) { + guardrailData.litellm_params.end_session_after_n_fails = endSessionAfterNFails; + } + if (onViolation && selectedEndpointType === "realtime") { + guardrailData.litellm_params.on_violation = onViolation; + } + if (realtimeViolationMessage.trim()) { + guardrailData.litellm_params.realtime_violation_message = realtimeViolationMessage.trim(); + } + } + /****************************** * Add provider-specific params * ---------------------------------- @@ -841,13 +867,15 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a return renderContentFilterConfiguration("keywords"); } return null; + case 4: + return renderEndpointSettings(); default: return null; } }; const renderStepButtons = () => { - const totalSteps = shouldRenderContentFilterConfigSettings(selectedProvider) ? 4 : 2; + const totalSteps = shouldRenderContentFilterConfigSettings(selectedProvider) ? 5 : 2; const isLastStep = currentStep === totalSteps - 1; const isCategoriesStep = shouldRenderContentFilterConfigSettings(selectedProvider) && currentStep === 1; const hasPendingCategory = pendingCategorySelection !== ""; @@ -888,13 +916,137 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a ); }; + const renderEndpointSettings = () => { + return ( +
+
+

+ Configure settings for a specific call type. Most guardrails don't need this — skip it + unless you're using a specific endpoint like /v1/realtime. +

+
+ +
+ + + setEndSessionAfterNFails( + e.target.value ? parseInt(e.target.value, 10) : undefined + ) + } + className="border border-gray-300 rounded px-3 py-1.5 text-sm w-32" + /> +
+ +
+ +
+ {(["warn", "end_session"] as const).map((opt) => ( + + ))} +
+
+ +
+ +

+ What the bot says aloud when this guardrail fires. Falls back to the default + violation message if empty. +

+