From 99ef233a69b6fef335a8b8177de6dbd4b77aa100 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 15 Jan 2026 13:30:43 +0530 Subject: [PATCH 001/117] fix: Preserved nullable object fields by carrying schema properties --- litellm/llms/vertex_ai/common_utils.py | 18 ++++++- .../test_amazing_vertex_completion.py | 54 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 1864ef734c0..63decfea854 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,4 +1,5 @@ import re +from copy import deepcopy from enum import Enum from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, get_type_hints @@ -617,7 +618,7 @@ def convert_anyof_null_to_nullable(schema, depth=0): if anyof is not None: contains_null = False for atype in anyof: - if atype == {"type": "null"}: + if isinstance(atype, dict) and atype.get("type") == "null": # remove null type anyof.remove(atype) contains_null = True @@ -735,7 +736,20 @@ def _convert_schema_types(schema, depth=0): type_val = schema["type"] if isinstance(type_val, list) and len(type_val) > 1: # Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]} - schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)] + # Preserve other schema fields by copying them into each non-null anyOf item. + base_schema = {k: v for k, v in schema.items() if k not in {"type", "anyOf"}} + any_of: List[Dict[str, Any]] = [] + for t in type_val: + if not isinstance(t, str): + continue + if t == "null": + # Keep null entry minimal so we can strip it later. + any_of.append({"type": "null"}) + continue + item_schema = deepcopy(base_schema) + item_schema["type"] = t + any_of.append(item_schema) + schema["anyOf"] = any_of schema.pop("type") elif isinstance(type_val, list) and len(type_val) == 1: schema["type"] = type_val[0] diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 0373f5f4356..745c90201f3 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3598,6 +3598,60 @@ def test_vertex_schema_test(): print(response) +def test_gemini_nullable_object_tool_schema_httpx(): + """ + Ensure nullable object tool params preserve nested properties in Vertex schema conversion. + """ + load_vertex_ai_credentials() + litellm._turn_on_debug() + + + tools = [{ + "type": "function", + "strict": True, + "function": { + "name": "create_support_ticket", + "description": "Create a paid user support ticket", + "parameters": { + "type": "object", + "additionalProperties": False, + "required": ["ticket_id", "customer_context"], + "properties": { + "ticket_id": { + "type": "string", + "description": "Unique identifier for the support ticket" + }, + "customer_context": { + "type": ["object", "null"], + "description": "Context about the paid customer, if available", + "additionalProperties": False, + "required": ["user_id", "plan"], + "properties": { + "user_id": { + "type": "string", + "description": "Internal user identifier" + }, + "plan": { + "type": "string", + "description": "Subscription plan name (e.g. pro, enterprise)" + } + } + } + } + } + } + }] + + response = litellm.completion( + model="vertex_ai/gemini-2.5-flash", + messages=[{"role": "user", "content": "call the tool"}], + tools=tools, + tool_choice="required", + ) + + print(response) + + def test_vertex_ai_response_id(): """Test that litellm preserves the response ID from Vertex AI's API for non-streaming responses""" from litellm.llms.custom_httpx.http_handler import HTTPHandler From 52519c26814b260e20e8c55de54ee548ddc5ad70 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 16 Jan 2026 18:09:15 +0530 Subject: [PATCH 002/117] Fix: _convert_schema_types --- litellm/llms/vertex_ai/common_utils.py | 34 +++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 63decfea854..e7d73e421a8 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -735,9 +735,15 @@ def _convert_schema_types(schema, depth=0): if "type" in schema: type_val = schema["type"] if isinstance(type_val, list) and len(type_val) > 1: - # Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]} - # Preserve other schema fields by copying them into each non-null anyOf item. - base_schema = {k: v for k, v in schema.items() if k not in {"type", "anyOf"}} + # Convert type arrays to anyOf format + # For object types, we need to move object-specific fields into the anyOf item + # For primitive types, we only include the type field + + # Fields that should stay at parent level (metadata) + metadata_fields = {"description", "title", "default", "examples"} + # Fields that are specific to object/array types and should move into anyOf + type_specific_fields = {"properties", "required", "additionalProperties", "items", "minItems", "maxItems", "minProperties", "maxProperties"} + any_of: List[Dict[str, Any]] = [] for t in type_val: if not isinstance(t, str): @@ -746,9 +752,25 @@ def _convert_schema_types(schema, depth=0): # Keep null entry minimal so we can strip it later. any_of.append({"type": "null"}) continue - item_schema = deepcopy(base_schema) - item_schema["type"] = t - any_of.append(item_schema) + + # For object/array types, include type-specific fields + if t in ("object", "array"): + item_schema = {"type": t} + # Move type-specific fields into this anyOf item + for field in type_specific_fields: + if field in schema: + item_schema[field] = deepcopy(schema[field]) + any_of.append(item_schema) + else: + # For primitive types, only include the type + any_of.append({"type": t}) + + # Remove type-specific fields from parent if we moved them into anyOf + has_object_or_array = any(t in ("object", "array") for t in type_val if isinstance(t, str)) + if has_object_or_array: + for field in type_specific_fields: + schema.pop(field, None) + schema["anyOf"] = any_of schema.pop("type") elif isinstance(type_val, list) and len(type_val) == 1: From f28340a39d95dd736578eea8b1b390688932bc9e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 16 Jan 2026 18:15:18 +0530 Subject: [PATCH 003/117] Fix all mypy issues --- litellm/llms/vertex_ai/common_utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index e7d73e421a8..54a6ba3ad7a 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -735,12 +735,7 @@ def _convert_schema_types(schema, depth=0): if "type" in schema: type_val = schema["type"] if isinstance(type_val, list) and len(type_val) > 1: - # Convert type arrays to anyOf format - # For object types, we need to move object-specific fields into the anyOf item - # For primitive types, we only include the type field - - # Fields that should stay at parent level (metadata) - metadata_fields = {"description", "title", "default", "examples"} + # Convert type arrays to anyOf format # Fields that are specific to object/array types and should move into anyOf type_specific_fields = {"properties", "required", "additionalProperties", "items", "minItems", "maxItems", "minProperties", "maxProperties"} From 666feef2a97d30b14b2ac4e1d7bc13614f5aff38 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:27:41 +0530 Subject: [PATCH 004/117] Add chat completion support for websearch --- .../websearch_interception/handler.py | 265 +++++++++++++++++- 1 file changed, 258 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 5d36b760afb..1e109dc9e39 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -48,7 +48,8 @@ class WebSearchInterceptionLogger(CustomLogger): Args: enabled_providers: List of LLM providers to enable interception for. Use LlmProviders enum values (e.g., [LlmProviders.BEDROCK]) - Default: [LlmProviders.BEDROCK] + If None or empty list, enables for ALL providers. + Default: None (all providers enabled) search_tool_name: Name of search tool configured in router's search_tools. If None, will attempt to use first available search tool. """ @@ -183,10 +184,10 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( f"WebSearchInterception: Pre-request hook called" f" - custom_llm_provider={custom_llm_provider}" - f" - enabled_providers={self.enabled_providers}" + f" - enabled_providers={self.enabled_providers or 'ALL'}" ) - if custom_llm_provider not in self.enabled_providers: + if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}" ) @@ -245,7 +246,12 @@ class WebSearchInterceptionLogger(CustomLogger): custom_llm_provider: str, kwargs: Dict, ) -> Tuple[bool, Dict]: - """Check if WebSearch tool interception is needed""" + """ + Check if WebSearch tool interception is needed for Anthropic Messages API. + + This is the legacy method for Anthropic-style responses. + For chat completions, use async_should_run_chat_completion_agentic_loop instead. + """ verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}") verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") @@ -253,7 +259,7 @@ class WebSearchInterceptionLogger(CustomLogger): # Check if provider should be intercepted # Note: custom_llm_provider is already normalized by get_llm_provider() # (e.g., "bedrock/invoke/..." -> "bedrock") - if custom_llm_provider not in self.enabled_providers: + if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: verbose_logger.debug( f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" ) @@ -267,10 +273,11 @@ class WebSearchInterceptionLogger(CustomLogger): ) return False, {} - # Detect WebSearch tool_use in response + # Detect WebSearch tool_use in response (Anthropic format) should_intercept, tool_calls = WebSearchTransformation.transform_request( response=response, stream=stream, + response_format="anthropic", ) if not should_intercept: @@ -288,6 +295,67 @@ class WebSearchInterceptionLogger(CustomLogger): "tool_calls": tool_calls, "tool_type": "websearch", "provider": custom_llm_provider, + "response_format": "anthropic", + } + return True, tools_dict + + async def async_should_run_chat_completion_agentic_loop( + self, + response: Any, + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + stream: bool, + custom_llm_provider: str, + kwargs: Dict, + ) -> Tuple[bool, Dict]: + """ + Check if WebSearch tool interception is needed for Chat Completions API. + + Similar to async_should_run_agentic_loop but for OpenAI-style chat completions. + """ + + verbose_logger.debug(f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}") + verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") + + # Check if provider should be intercepted + if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers: + verbose_logger.debug( + f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})" + ) + return False, {} + + # Check if tools include any web search tool + has_websearch_tool = any(is_web_search_tool(t) for t in (tools or [])) + if not has_websearch_tool: + verbose_logger.debug( + "WebSearchInterception: No web search tool in request" + ) + return False, {} + + # Detect WebSearch tool_calls in response (OpenAI format) + should_intercept, tool_calls = WebSearchTransformation.transform_request( + response=response, + stream=stream, + response_format="openai", + ) + + if not should_intercept: + verbose_logger.debug( + "WebSearchInterception: No WebSearch tool_calls detected in response" + ) + return False, {} + + verbose_logger.debug( + f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop" + ) + + # Return tools dict with tool calls + tools_dict = { + "tool_calls": tool_calls, + "tool_type": "websearch", + "provider": custom_llm_provider, + "response_format": "openai", } return True, tools_dict @@ -303,7 +371,11 @@ class WebSearchInterceptionLogger(CustomLogger): stream: bool, kwargs: Dict, ) -> Any: - """Execute agentic loop with WebSearch execution""" + """ + Execute agentic loop with WebSearch execution for Anthropic Messages API. + + This is the legacy method for Anthropic-style responses. + """ tool_calls = tools["tool_calls"] @@ -321,6 +393,41 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs, ) + async def async_run_chat_completion_agentic_loop( + self, + tools: Dict, + model: str, + messages: List[Dict], + response: Any, + optional_params: Dict, + logging_obj: Any, + stream: bool, + kwargs: Dict, + ) -> Any: + """ + Execute agentic loop with WebSearch execution for Chat Completions API. + + Similar to async_run_agentic_loop but for OpenAI-style chat completions. + """ + + tool_calls = tools["tool_calls"] + response_format = tools.get("response_format", "openai") + + verbose_logger.debug( + f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)" + ) + + return await self._execute_chat_completion_agentic_loop( + model=model, + messages=messages, + tool_calls=tool_calls, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs, + response_format=response_format, + ) + async def _execute_agentic_loop( self, model: str, @@ -521,6 +628,150 @@ class WebSearchInterceptionLogger(CustomLogger): ) raise + async def _execute_chat_completion_agentic_loop( + self, + model: str, + messages: List[Dict], + tool_calls: List[Dict], + optional_params: Dict, + logging_obj: Any, + stream: bool, + kwargs: Dict, + response_format: str = "openai", + ) -> Any: + """Execute litellm.search() and make follow-up chat completion request""" + + # Extract search queries from tool_calls + search_tasks = [] + for tool_call in tool_calls: + # Handle both Anthropic-style input and OpenAI-style function.arguments + query = None + if "input" in tool_call and isinstance(tool_call["input"], dict): + query = tool_call["input"].get("query") + elif "function" in tool_call: + func = tool_call["function"] + if isinstance(func, dict): + args = func.get("arguments", {}) + if isinstance(args, dict): + query = args.get("query") + + if query: + verbose_logger.debug( + f"WebSearchInterception: Queuing search for query='{query}'" + ) + search_tasks.append(self._execute_search(query)) + else: + verbose_logger.warning( + f"WebSearchInterception: Tool call {tool_call.get('id')} has no query" + ) + # Add empty result for tools without query + search_tasks.append(self._create_empty_search_result()) + + # Execute searches in parallel + verbose_logger.debug( + f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel" + ) + search_results = await asyncio.gather(*search_tasks, return_exceptions=True) + + # Handle any exceptions in search results + final_search_results: List[str] = [] + for i, result in enumerate(search_results): + if isinstance(result, Exception): + verbose_logger.error( + f"WebSearchInterception: Search {i} failed with error: {str(result)}" + ) + final_search_results.append( + f"Search failed: {str(result)}" + ) + elif isinstance(result, str): + final_search_results.append(cast(str, result)) + else: + verbose_logger.warning( + f"WebSearchInterception: Unexpected result type {type(result)} at index {i}" + ) + final_search_results.append(str(result)) + + # Build assistant and tool messages using transformation + assistant_message, tool_messages_or_user = WebSearchTransformation.transform_response( + tool_calls=tool_calls, + search_results=final_search_results, + response_format=response_format, + ) + + # Make follow-up request with search results + # For OpenAI format, tool_messages_or_user is a list of tool messages + if response_format == "openai": + follow_up_messages = messages + [assistant_message] + tool_messages_or_user + else: + # For Anthropic format (shouldn't happen in this method, but handle it) + follow_up_messages = messages + [assistant_message, tool_messages_or_user] + + verbose_logger.debug( + "WebSearchInterception: Making follow-up chat completion request with search results" + ) + verbose_logger.debug( + f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}" + ) + + # Use litellm.acompletion for follow-up request + try: + # Remove internal parameters that shouldn't be passed to follow-up request + internal_params = { + '_websearch_interception', + 'acompletion', + 'litellm_logging_obj', + 'custom_llm_provider', + 'model_alias_map', + 'stream_response', + 'custom_prompt_dict', + } + kwargs_for_followup = { + k: v for k, v in kwargs.items() + if not k.startswith('_websearch_interception') and k not in internal_params + } + + # Get full model name from kwargs + full_model_name = model + if "custom_llm_provider" in kwargs: + custom_llm_provider = kwargs["custom_llm_provider"] + # Reconstruct full model name with provider prefix if needed + if not model.startswith(custom_llm_provider): + # Check if model already has a provider prefix + if "/" not in model: + full_model_name = f"{custom_llm_provider}/{model}" + + verbose_logger.debug( + f"WebSearchInterception: Using model name: {full_model_name}" + ) + + # Prepare tools for follow-up request (same as original) + tools_param = optional_params.get("tools") + + # Remove tools and extra_body from optional_params to avoid issues + # extra_body often contains internal LiteLLM params that shouldn't be forwarded + optional_params_clean = { + k: v for k, v in optional_params.items() + if k not in {"tools", "extra_body", "model_alias_map","stream_response", "custom_prompt_dict" } + } + + final_response = await litellm.acompletion( + model=full_model_name, + messages=follow_up_messages, + tools=tools_param, + **optional_params_clean, + **kwargs_for_followup, + ) + + verbose_logger.debug( + f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}" + ) + return final_response + except Exception as e: + verbose_logger.exception( + f"WebSearchInterception: Follow-up request failed: {str(e)}" + ) + raise + async def _create_empty_search_result(self) -> str: """Create an empty search result for tool calls without queries""" return "No search query provided" From ea4e48e13a6d7b6a21d087e836201a52c55f2d72 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:28:06 +0530 Subject: [PATCH 005/117] Add chat completion tool calls support and response transformation --- .../websearch_interception/transformation.py | 185 ++++++++++++++++-- 1 file changed, 171 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 313358822a5..0884d408c84 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -1,7 +1,7 @@ """ WebSearch Tool Transformation -Transforms between Anthropic tool_use format and LiteLLM search format. +Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. """ from typing import Any, Dict, List, Tuple @@ -17,28 +17,31 @@ class WebSearchTransformation: Handles transformation between: - Anthropic tool_use format → LiteLLM search requests - - LiteLLM SearchResponse → Anthropic tool_result format + - OpenAI tool_calls format → LiteLLM search requests + - LiteLLM SearchResponse → Anthropic/OpenAI tool_result format """ @staticmethod def transform_request( response: Any, stream: bool, + response_format: str = "anthropic", ) -> Tuple[bool, List[Dict]]: """ - Transform Anthropic response to extract WebSearch tool calls. + Transform model response to extract WebSearch tool calls. - Detects if response contains WebSearch tool_use blocks and extracts + Detects if response contains WebSearch tool_use/tool_calls blocks and extracts the search queries for execution. Args: - response: Model response (dict or AnthropicMessagesResponse) + response: Model response (dict, AnthropicMessagesResponse, or ModelResponse) stream: Whether response is streaming + response_format: Response format - "anthropic" or "openai" (default: "anthropic") Returns: (has_websearch, tool_calls): has_websearch: True if WebSearch tool_use found - tool_calls: List of tool_use dicts with id, name, input + tool_calls: List of tool_use/tool_calls dicts with id, name, input/function Note: Streaming requests are handled by converting stream=True to stream=False @@ -54,8 +57,11 @@ class WebSearchTransformation: ) return False, [] - # Parse non-streaming response - return WebSearchTransformation._detect_from_non_streaming_response(response) + # Parse non-streaming response based on format + if response_format == "openai": + return WebSearchTransformation._detect_from_openai_response(response) + else: + return WebSearchTransformation._detect_from_non_streaming_response(response) @staticmethod def _detect_from_non_streaming_response( @@ -114,26 +120,143 @@ class WebSearchTransformation: return len(tool_calls) > 0, tool_calls + @staticmethod + def _detect_from_openai_response( + response: Any, + ) -> Tuple[bool, List[Dict]]: + """Parse OpenAI-style response for WebSearch tool_calls""" + + # Handle both dict and ModelResponse objects + if isinstance(response, dict): + choices = response.get("choices", []) + else: + if not hasattr(response, "choices"): + verbose_logger.debug( + "WebSearchInterception: Response has no choices attribute" + ) + return False, [] + choices = response.choices or [] + + if not choices: + verbose_logger.debug( + "WebSearchInterception: Response has empty choices" + ) + return False, [] + + # Get first choice's message + first_choice = choices[0] + if isinstance(first_choice, dict): + message = first_choice.get("message", {}) + else: + message = getattr(first_choice, "message", None) + + if not message: + verbose_logger.debug( + "WebSearchInterception: First choice has no message" + ) + return False, [] + + # Get tool_calls from message + if isinstance(message, dict): + openai_tool_calls = message.get("tool_calls", []) + else: + openai_tool_calls = getattr(message, "tool_calls", None) or [] + + if not openai_tool_calls: + verbose_logger.debug( + "WebSearchInterception: Message has no tool_calls" + ) + return False, [] + + # Find all WebSearch tool calls + tool_calls = [] + for tool_call in openai_tool_calls: + # Handle both dict and object tool calls + if isinstance(tool_call, dict): + tool_id = tool_call.get("id") + tool_type = tool_call.get("type") + function = tool_call.get("function", {}) + function_name = function.get("name") if isinstance(function, dict) else getattr(function, "name", None) + function_arguments = function.get("arguments") if isinstance(function, dict) else getattr(function, "arguments", None) + else: + tool_id = getattr(tool_call, "id", None) + tool_type = getattr(tool_call, "type", None) + function = getattr(tool_call, "function", None) + function_name = getattr(function, "name", None) if function else None + function_arguments = getattr(function, "arguments", None) if function else None + + # Check for LiteLLM standard or legacy web search tools + if tool_type == "function" and function_name in ( + LITELLM_WEB_SEARCH_TOOL_NAME, "WebSearch", "web_search" + ): + # Parse arguments (might be JSON string) + import json + if isinstance(function_arguments, str): + try: + arguments = json.loads(function_arguments) + except json.JSONDecodeError: + verbose_logger.warning( + f"WebSearchInterception: Failed to parse function arguments: {function_arguments}" + ) + arguments = {} + else: + arguments = function_arguments or {} + + # Convert to internal format (similar to Anthropic) + tool_call_dict = { + "id": tool_id, + "type": "function", + "name": function_name, + "function": { + "name": function_name, + "arguments": arguments, + }, + "input": arguments, # For compatibility with Anthropic format + } + tool_calls.append(tool_call_dict) + verbose_logger.debug( + f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}" + ) + + return len(tool_calls) > 0, tool_calls + @staticmethod def transform_response( tool_calls: List[Dict], search_results: List[str], + response_format: str = "anthropic", ) -> Tuple[Dict, Dict]: """ - Transform LiteLLM search results to Anthropic tool_result format. + Transform LiteLLM search results to Anthropic/OpenAI tool_result format. - Builds the assistant and user messages needed for the agentic loop + Builds the assistant and user/tool messages needed for the agentic loop follow-up request. Args: - tool_calls: List of tool_use dicts from transform_request + tool_calls: List of tool_use/tool_calls dicts from transform_request search_results: List of search result strings (one per tool_call) + response_format: Response format - "anthropic" or "openai" (default: "anthropic") Returns: - (assistant_message, user_message): - assistant_message: Message with tool_use blocks - user_message: Message with tool_result blocks + (assistant_message, user_or_tool_messages): + For Anthropic: assistant_message with tool_use blocks, user_message with tool_result blocks + For OpenAI: assistant_message with tool_calls, tool_messages list with tool results """ + if response_format == "openai": + return WebSearchTransformation._transform_response_openai( + tool_calls, search_results + ) + else: + return WebSearchTransformation._transform_response_anthropic( + tool_calls, search_results + ) + + @staticmethod + def _transform_response_anthropic( + tool_calls: List[Dict], + search_results: List[str], + ) -> Tuple[Dict, Dict]: + """Transform to Anthropic format (single user message with tool_result blocks)""" # Build assistant message with tool_use blocks assistant_message = { "role": "assistant", @@ -163,6 +286,40 @@ class WebSearchTransformation: return assistant_message, user_message + @staticmethod + def _transform_response_openai( + tool_calls: List[Dict], + search_results: List[str], + ) -> Tuple[Dict, List[Dict]]: + """Transform to OpenAI format (assistant with tool_calls, separate tool messages)""" + # Build assistant message with tool_calls + assistant_message = { + "role": "assistant", + "tool_calls": [ + { + "id": tc["id"], + "type": "function", + "function": { + "name": tc["name"], + "arguments": str(tc["input"]), + }, + } + for tc in tool_calls + ], + } + + # Build separate tool messages (one per tool call) + tool_messages = [ + { + "role": "tool", + "tool_call_id": tool_calls[i]["id"], + "content": search_results[i], + } + for i in range(len(tool_calls)) + ] + + return assistant_message, tool_messages + @staticmethod def format_search_response(result: SearchResponse) -> str: """ From 245d705e6ca7ee9c8cda2dccb2d40256480726f2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:28:28 +0530 Subject: [PATCH 006/117] Add new methods in chat completion --- litellm/integrations/custom_logger.py | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 07d237c4758..4a341863d4b 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -664,6 +664,37 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac return final_response """ pass + + async def async_should_run_chat_completion_agentic_loop( + self, + response: Any, + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + stream: bool, + custom_llm_provider: str, + kwargs: Dict, + ) -> Tuple[bool, Dict]: + """ + Hook to determine if chat completion agentic loop should be executed. + """ + return False, {} + + async def async_run_chat_completion_agentic_loop( + self, + tools: Dict, + model: str, + messages: List[Dict], + response: Any, + optional_params: Dict, + logging_obj: "LiteLLMLoggingObj", + stream: bool, + kwargs: Dict, + ) -> Any: + """ + Hook to execute chat completion agentic loop based on context from should_run hook. + """ + pass # Useful helpers for custom logger classes From 6207bf8f6856c185844c481aebe9f5eebb261801 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:28:43 +0530 Subject: [PATCH 007/117] Add chat completion tool format --- .../integrations/websearch_interception/tools.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/tools.py b/litellm/integrations/websearch_interception/tools.py index 4f8b7372fe3..c92c66f41ee 100644 --- a/litellm/integrations/websearch_interception/tools.py +++ b/litellm/integrations/websearch_interception/tools.py @@ -55,6 +55,7 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: Detects: - LiteLLM standard: name == "litellm_web_search" + - OpenAI format: type == "function" with function.name == "litellm_web_search" - Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305") - Claude Code: name == "web_search" with a type field - Custom: name == "WebSearch" (legacy format) @@ -68,15 +69,25 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: Example: >>> is_web_search_tool({"name": "litellm_web_search"}) True + >>> is_web_search_tool({"type": "function", "function": {"name": "litellm_web_search"}}) + True >>> is_web_search_tool({"type": "web_search_20250305", "name": "web_search"}) True >>> is_web_search_tool({"name": "calculator"}) False """ + print(f"🔥tool: {tool}") tool_name = tool.get("name", "") tool_type = tool.get("type", "") + + # Check for OpenAI format: {"type": "function", "function": {"name": "..."}} + if tool_type == "function" and "function" in tool: + function_def = tool.get("function", {}) + function_name = function_def.get("name", "") + if function_name == LITELLM_WEB_SEARCH_TOOL_NAME: + return True - # Check for LiteLLM standard tool + # Check for LiteLLM standard tool (Anthropic format) if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME: return True From 88778a871dce4378f84e56fbae5b10e60476dd5e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:29:21 +0530 Subject: [PATCH 008/117] Add callback for websearch in completion method --- litellm/llms/custom_httpx/llm_http_handler.py | 128 +++++++++++++++++- litellm/llms/openai/openai.py | 92 ++++++++++++- 2 files changed, 214 insertions(+), 6 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d2ea7e872a2..3907ff7abf7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -302,7 +302,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, signed_json_body=signed_json_body, ) - return provider_config.transform_response( + initial_response = provider_config.transform_response( model=model, raw_response=response, model_response=model_response, @@ -316,6 +316,20 @@ class BaseLLMHTTPHandler: json_mode=json_mode, ) + # Call agentic chat completion hooks + final_response = await self._call_agentic_chat_completion_hooks( + response=initial_response, + model=model, + messages=messages, + optional_params=optional_params, + logging_obj=logging_obj, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=litellm_params, + ) + + return final_response if final_response is not None else initial_response + def completion( self, model: str, @@ -412,6 +426,11 @@ class BaseLLMHTTPHandler: }, ) + # Check if stream was converted for WebSearch interception + # This is set by the async_pre_request_hook in WebSearchInterceptionLogger + if litellm_params.get("_websearch_interception_converted_stream", False): + logging_obj.model_call_details["websearch_interception_converted_stream"] = True + if acompletion is True: if stream is True: data = self._add_stream_param_to_request_body( @@ -419,7 +438,7 @@ class BaseLLMHTTPHandler: provider_config=provider_config, fake_stream=fake_stream, ) - return self.acompletion_stream_function( + response = self.acompletion_stream_function( model=model, messages=messages, api_base=api_base, @@ -4361,10 +4380,10 @@ class BaseLLMHTTPHandler: kwargs: Dict, ) -> Optional[Any]: """ - Call agentic completion hooks for all custom loggers. + Call agentic completion hooks for all custom loggers (Anthropic Messages API). - 1. Call async_should_run_agentic_completion to check if agentic loop is needed - 2. If yes, call async_run_agentic_completion to execute the loop + 1. Call async_should_run_agentic_loop to check if agentic loop is needed + 2. If yes, call async_run_agentic_loop to execute the loop Returns the response from agentic loop, or None if no hook runs. """ @@ -4453,6 +4472,105 @@ class BaseLLMHTTPHandler: return None + async def _call_agentic_chat_completion_hooks( + self, + response: Any, + model: str, + messages: List[Dict], + optional_params: Dict, + logging_obj: "LiteLLMLoggingObj", + stream: bool, + custom_llm_provider: str, + kwargs: Dict, + ) -> Optional[Any]: + """ + Call agentic chat completion hooks for all custom loggers (Chat Completions API). + + 1. Call async_should_run_chat_completion_agentic_loop to check if agentic loop is needed + 2. If yes, call async_run_chat_completion_agentic_loop to execute the loop + + Returns the response from agentic loop, or None if no hook runs. + """ + from litellm._logging import verbose_logger + from litellm.integrations.custom_logger import CustomLogger + + callbacks = litellm.callbacks + ( + logging_obj.dynamic_success_callbacks or [] + ) + tools = optional_params.get("tools", []) + + for callback in callbacks: + try: + if isinstance(callback, CustomLogger): + # Check if callback has the chat completion agentic loop method + if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"): + continue + + # First: Check if agentic loop should run + should_run, tool_calls = ( + await callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + ) + + if should_run: + # Second: Execute agentic loop + # Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name + kwargs_with_provider = kwargs.copy() if kwargs else {} + kwargs_with_provider["custom_llm_provider"] = custom_llm_provider + agentic_response = await callback.async_run_chat_completion_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ) + # First hook that runs agentic loop wins + return agentic_response + + except Exception as e: + verbose_logger.exception( + f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {str(e)}" + ) + + # Check if we need to convert response to fake stream for chat completions + # This happens when: + # 1. Stream was originally True but converted to False for WebSearch interception + # 2. No agentic loop ran (LLM didn't use the tool) + # 3. We have a non-streaming response that needs to be converted to streaming + websearch_converted_stream = ( + logging_obj.model_call_details.get("websearch_interception_converted_stream", False) + if logging_obj is not None + else False + ) + + if websearch_converted_stream: + from litellm._logging import verbose_logger + from litellm.llms.base_llm.base_model_iterator import ( + convert_model_response_to_streaming, + ) + + verbose_logger.debug( + "WebSearchInterception: No tool call made, converting non-streaming chat completion to fake stream" + ) + + # Convert the non-streaming ModelResponse to a fake stream + if hasattr(response, "choices"): + # Use the existing converter for ModelResponse + fake_stream = convert_model_response_to_streaming(response) + return fake_stream + + return None + def _handle_error( self, e: Exception, diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 8a8070240da..2f0e5e480b5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -501,6 +501,82 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): else: raise e + async def _call_agentic_completion_hooks_openai( + self, + response: Any, + model: str, + messages: List[Dict], + optional_params: Dict, + logging_obj: LiteLLMLoggingObj, + stream: bool, + litellm_params: Dict, + ) -> Optional[Any]: + """ + Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API). + + 1. Call async_should_run_chat_completion_agentic_loop to check if agentic loop is needed + 2. If yes, call async_run_chat_completion_agentic_loop to execute the loop + + Returns the response from agentic loop, or None if no hook runs. + """ + from litellm._logging import verbose_logger + from litellm.integrations.custom_logger import CustomLogger + + callbacks = litellm.callbacks + ( + logging_obj.dynamic_success_callbacks or [] + ) + print(f"🔥callbacks: {callbacks}") + tools = optional_params.get("tools", []) + print(f"🔥tools: {tools}") + # Get custom_llm_provider from litellm_params + custom_llm_provider = litellm_params.get("custom_llm_provider", "openai") + + for callback in callbacks: + try: + if isinstance(callback, CustomLogger): + # Check if the callback has the chat completion agentic loop methods + if not hasattr(callback, 'async_should_run_chat_completion_agentic_loop'): + continue + + # First: Check if agentic loop should run (using chat completion method) + should_run, tool_calls = ( + await callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=litellm_params, + ) + ) + + if should_run: + # Second: Execute agentic loop + kwargs_with_provider = litellm_params.copy() if litellm_params else {} + kwargs_with_provider["custom_llm_provider"] = custom_llm_provider + + # For OpenAI Chat Completions, use the chat completion agentic loop method + agentic_response = await callback.async_run_chat_completion_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ) + # First hook that runs agentic loop wins + return agentic_response + + except Exception as e: + verbose_logger.exception( + f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {str(e)}" + ) + + return None + def mock_streaming( self, response: ModelResponse, @@ -844,7 +920,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj=logging_obj, ) stringified_response = response.model_dump() - + print(f"🔥stringified_response: {stringified_response}") logging_obj.post_call( input=data["messages"], api_key=api_key, @@ -859,6 +935,20 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): _response_headers=headers, ) + # Call agentic completion hooks (e.g., for websearch_interception) + agentic_response = await self._call_agentic_completion_hooks_openai( + response=final_response_obj, + model=model, + messages=messages, + optional_params=optional_params, + logging_obj=logging_obj, + stream=False, + litellm_params=litellm_params, + ) + + if agentic_response is not None: + final_response_obj = agentic_response + if fake_stream is True: return self.mock_streaming( response=cast(ModelResponse, final_response_obj), From 4b0eb50ddddfe3bc0b70c4ef649391be603e8923 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Feb 2026 14:29:31 +0530 Subject: [PATCH 009/117] Add test for web search --- test_websearch_chat_completion.py | 136 ++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 test_websearch_chat_completion.py diff --git a/test_websearch_chat_completion.py b/test_websearch_chat_completion.py new file mode 100644 index 00000000000..e572e4d860c --- /dev/null +++ b/test_websearch_chat_completion.py @@ -0,0 +1,136 @@ +""" +Test script for WebSearch interception with chat completions API. + +This script demonstrates how to use the websearch_interception callback +with litellm.acompletion() for transparent server-side web search execution. +""" +import asyncio +import litellm + +# Enable verbose logging to see what's happening +litellm.set_verbose = True + + +async def test_websearch_chat_completion(): + """Test websearch interception with chat completions API.""" + + # Configure WebSearch interception + litellm.callbacks = ["websearch_interception"] + + print("\n" + "="*80) + print("Testing WebSearch Interception with Chat Completions API") + print("="*80 + "\n") + + # User makes a simple completion call with tools + print("Making request to GPT-4o with litellm_web_search tool...") + print("Question: What's the weather in San Francisco today?") + print("\nExpected behavior:") + print("1. Model calls litellm_web_search tool") + print("2. Server executes web search automatically") + print("3. Server makes follow-up request with search results") + print("4. User gets final answer\n") + + response = await litellm.acompletion( + model="gpt-4o", + messages=[ + {"role": "user", "content": "What's the weather in San Francisco today?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"] + } + } + } + ] + ) + + print("\n" + "-"*80) + print("FINAL RESPONSE:") + print("-"*80) + print(f"\nContent: {response.choices[0].message.content}") + print(f"\nFinish reason: {response.choices[0].finish_reason}") + + # Check if we got tool_calls (should NOT if agentic loop worked) + if hasattr(response.choices[0].message, 'tool_calls') and response.choices[0].message.tool_calls: + print("\n⚠️ WARNING: Got tool_calls in response!") + print("This means the agentic loop did NOT execute automatically.") + print(f"Tool calls: {response.choices[0].message.tool_calls}") + else: + print("\n✅ SUCCESS: No tool_calls in response!") + print("The agentic loop executed automatically and returned the final answer.") + + print("\n" + "="*80 + "\n") + + +async def test_streaming_websearch(): + """Test websearch interception with streaming.""" + + # Configure WebSearch interception + litellm.callbacks = ["websearch_interception"] + + print("\n" + "="*80) + print("Testing WebSearch Interception with STREAMING") + print("="*80 + "\n") + + print("Making STREAMING request to GPT-4o with litellm_web_search tool...") + print("Question: What are the latest AI news?") + + response = await litellm.acompletion( + model="gpt-4o", + messages=[ + {"role": "user", "content": "What are the latest AI news from today?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"} + } + } + } + } + ], + stream=True + ) + + print("\n" + "-"*80) + print("STREAMING RESPONSE:") + print("-"*80 + "\n") + + full_content = "" + async for chunk in response: + if hasattr(chunk.choices[0].delta, 'content') and chunk.choices[0].delta.content: + content = chunk.choices[0].delta.content + print(content, end="", flush=True) + full_content += content + + print("\n\n✅ Streaming completed successfully!") + print(f"Total content length: {len(full_content)} chars") + print("\n" + "="*80 + "\n") + + +if __name__ == "__main__": + print("\nWebSearch Interception Test Suite") + print("==================================\n") + print("This test demonstrates transparent server-side web search execution.") + print("The agentic loop happens automatically - user just gets the final answer.\n") + + # Run tests + asyncio.run(test_websearch_chat_completion()) + + # Uncomment to test streaming + # asyncio.run(test_streaming_websearch()) From a2e70a561d103497fed719829c48eb597142f214 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 6 Feb 2026 08:20:50 +0530 Subject: [PATCH 010/117] Potential fix for code scanning alert no. 4046: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/llms/openai/openai.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 2f0e5e480b5..c6f502d3a25 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -525,9 +525,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): callbacks = litellm.callbacks + ( logging_obj.dynamic_success_callbacks or [] ) - print(f"🔥callbacks: {callbacks}") + # Avoid logging full callback objects to prevent leaking sensitive data + verbose_logger.debug( + "LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks) + ) tools = optional_params.get("tools", []) - print(f"🔥tools: {tools}") + # Avoid logging full tools payloads; they may contain sensitive parameters + verbose_logger.debug( + "LiteLLM.AgenticHooks: tools_count=%s", len(tools) if isinstance(tools, list) else 1 if tools else 0 + ) # Get custom_llm_provider from litellm_params custom_llm_provider = litellm_params.get("custom_llm_provider", "openai") From f12875bd428d74cb46cc24d19ec44d4f3bdafc32 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 6 Feb 2026 08:21:09 +0530 Subject: [PATCH 011/117] Update litellm/integrations/websearch_interception/tools.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/integrations/websearch_interception/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/tools.py b/litellm/integrations/websearch_interception/tools.py index c92c66f41ee..be8808622da 100644 --- a/litellm/integrations/websearch_interception/tools.py +++ b/litellm/integrations/websearch_interception/tools.py @@ -76,7 +76,6 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: >>> is_web_search_tool({"name": "calculator"}) False """ - print(f"🔥tool: {tool}") tool_name = tool.get("name", "") tool_type = tool.get("type", "") From 51d565f619604cb60d2b50f6f19721da7d35f41c Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Sat, 7 Feb 2026 03:10:53 +0530 Subject: [PATCH 012/117] fix conflicts with main- (this PR is from upstream/main) --- litellm/proxy/_types.py | 16 +- .../proxy/hooks/model_max_budget_limiter.py | 38 ++- .../budget_management_endpoints.py | 39 ++- .../key_management_endpoints.py | 233 ++++++++++-------- tests/proxy_unit_tests/test_proxy_utils.py | 145 +++++++---- ...test_unit_test_max_model_budget_limiter.py | 63 +++-- .../test_budget_endpoints.py | 54 ++-- 7 files changed, 393 insertions(+), 195 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f38f94f4c98..7324c3ea1de 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -359,7 +359,6 @@ class LiteLLMRoutes(enum.Enum): "/v1/vector_stores/{vector_store_id}/files/{file_id}/content", "/vector_store/list", "/v1/vector_store/list", - # search "/search", "/v1/search", @@ -2220,13 +2219,22 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): last_refreshed_at: Optional[float] = None # last time joint view was pulled from db def __init__(self, **kwargs): - # Handle litellm_budget_table_* keys + # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) for key, value in list(kwargs.items()): if key.startswith("litellm_budget_table_") and value is not None: # Extract the corresponding attribute name attr_name = key.replace("litellm_budget_table_", "") - # Check if the value is None and set the corresponding attribute - if getattr(self, attr_name, None) is None: + # Use key's value from kwargs (from DB view), not class default + current = kwargs.get(attr_name) + if current is None: + current = getattr(self, attr_name, None) + # Apply budget value when key has no value, or for model_max_budget when key has empty dict + should_apply = current is None or ( + attr_name == "model_max_budget" + and isinstance(current, dict) + and len(current) == 0 + ) + if should_apply: kwargs[attr_name] = value if key == "end_user_id" and value is not None and isinstance(value, int): kwargs[key] = str(value) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index ac02c915366..69c7e92d82e 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -171,19 +171,35 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return response_cost: float = standard_logging_payload.get("response_cost", 0) model = standard_logging_payload.get("model") + virtual_key = standard_logging_payload.get("metadata", {}).get( + "user_api_key_hash" + ) - virtual_key = standard_logging_payload.get("metadata").get("user_api_key_hash") - model = standard_logging_payload.get("model") - if virtual_key is not None: - budget_config = BudgetConfig(time_period="1d", budget_limit=0.1) - virtual_spend_key = f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{budget_config.budget_duration}" - virtual_start_time_key = f"virtual_key_budget_start_time:{virtual_key}" - await self._increment_spend_for_key( - budget_config=budget_config, - spend_key=virtual_spend_key, - start_time_key=virtual_start_time_key, - response_cost=response_cost, + if virtual_key is None or model is None: + return + + # Resolve per-model budget config (same logic as is_key_within_model_budget) + internal_model_max_budget: GenericBudgetConfigType = {} + for _model, _budget_info in user_api_key_model_max_budget.items(): + internal_model_max_budget[_model] = BudgetConfig(**_budget_info) + key_budget_config = self._get_request_model_budget_config( + model=model, internal_model_max_budget=internal_model_max_budget + ) + if key_budget_config is None or not key_budget_config.budget_duration: + verbose_proxy_logger.debug( + "Not incrementing model spend: no budget config or budget_duration for model=%s", + model, ) + return + + virtual_spend_key = f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{key_budget_config.budget_duration}" + virtual_start_time_key = f"virtual_key_budget_start_time:{virtual_key}" + await self._increment_spend_for_key( + budget_config=key_budget_config, + spend_key=virtual_spend_key, + start_time_key=virtual_start_time_key, + response_cost=response_cost, + ) verbose_proxy_logger.debug( "current state of in memory cache %s", json.dumps( diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index e43da32565a..20c7f9ec412 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -59,14 +59,29 @@ async def new_budget( if budget_obj.max_budget is not None and budget_obj.max_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"} + detail={ + "error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}" + }, ) if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"} + detail={ + "error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}" + }, ) + # Validate model_max_budget if present + if budget_obj.model_max_budget is not None and len(budget_obj.model_max_budget) > 0: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + validate_model_max_budget, + ) + + try: + validate_model_max_budget(budget_obj.model_max_budget) + except ValueError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) + # if no budget_reset_at date is set, but a budget_duration is given, then set budget_reset_at initially to the first completed duration interval in future if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None: budget_obj.budget_reset_at = datetime.utcnow() + timedelta( @@ -123,14 +138,29 @@ async def update_budget( if budget_obj.max_budget is not None and budget_obj.max_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"} + detail={ + "error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}" + }, ) if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"} + detail={ + "error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}" + }, ) + # Validate model_max_budget if present in update + if budget_obj.model_max_budget is not None and len(budget_obj.model_max_budget) > 0: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + validate_model_max_budget, + ) + + try: + validate_model_max_budget(budget_obj.model_max_budget) + except ValueError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) + response = await prisma_client.db.litellm_budgettable.update( where={"budget_id": budget_obj.budget_id}, data={ @@ -226,6 +256,7 @@ async def budget_settings( "budget_duration": {"type": "String"}, "max_budget": {"type": "Float"}, "soft_budget": {"type": "Float"}, + "model_max_budget": {"type": "Object"}, } return_val = [] diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 9dadffca351..f1c97693455 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -518,7 +518,7 @@ async def _common_key_generation_helper( # noqa: PLR0915 ) # Handle special case where duration is "-1" (never expires) if value == "-1": - user_duration = float('inf') # Infinite duration + user_duration = float("inf") # Infinite duration else: user_duration = duration_in_seconds(duration=value) if user_duration > upperbound_duration: @@ -660,9 +660,9 @@ async def _common_key_generation_helper( # noqa: PLR0915 request_type="key", **data_json, table_name="key" ) - response["soft_budget"] = ( - data.soft_budget - ) # include the user-input soft budget in the response + response[ + "soft_budget" + ] = data.soft_budget # include the user-input soft budget in the response response = GenerateKeyResponse(**response) @@ -1083,12 +1083,16 @@ async def generate_key_fn( if data.max_budget is not None and data.max_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"} + detail={ + "error": f"max_budget cannot be negative. Received: {data.max_budget}" + }, ) if data.soft_budget is not None and data.soft_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"} + detail={ + "error": f"soft_budget cannot be negative. Received: {data.soft_budget}" + }, ) if user_custom_key_generate is not None: @@ -1399,8 +1403,13 @@ async def prepare_key_update_data( validate_model_max_budget(non_default_values["model_max_budget"]) # Serialize router_settings to JSON if present - if "router_settings" in non_default_values and non_default_values["router_settings"] is not None: - non_default_values["router_settings"] = safe_dumps(non_default_values["router_settings"]) + if ( + "router_settings" in non_default_values + and non_default_values["router_settings"] is not None + ): + non_default_values["router_settings"] = safe_dumps( + non_default_values["router_settings"] + ) non_default_values = prepare_metadata_fields( data=data, non_default_values=non_default_values, existing_metadata=_metadata @@ -1448,19 +1457,17 @@ def is_different_team( def _validate_max_budget(max_budget: Optional[float]) -> None: """ Validate that max_budget is not negative. - + Args: max_budget: The max_budget value to validate - + Raises: HTTPException: If max_budget is negative """ if max_budget is not None and max_budget < 0: raise HTTPException( status_code=400, - detail={ - "error": f"max_budget cannot be negative. Received: {max_budget}" - }, + detail={"error": f"max_budget cannot be negative. Received: {max_budget}"}, ) @@ -1469,14 +1476,14 @@ async def _get_and_validate_existing_key( ) -> LiteLLM_VerificationToken: """ Get existing key from database and validate it exists. - + Args: token: The key token to look up prisma_client: Prisma client instance - + Returns: LiteLLM_VerificationToken: The existing key row - + Raises: HTTPException: If key is not found """ @@ -1485,19 +1492,19 @@ async def _get_and_validate_existing_key( status_code=500, detail={"error": "Database not connected"}, ) - + existing_key_row = await prisma_client.get_data( token=token, table_name="key", query_type="find_unique", ) - + if existing_key_row is None: raise HTTPException( status_code=404, detail={"error": f"Key not found: {token}"}, ) - + return existing_key_row @@ -1512,10 +1519,10 @@ async def _process_single_key_update( ) -> Dict[str, Any]: """ Process a single key update with all validations and checks. - + This function encapsulates all the logic for updating a single key, including validation, permission checks, team checks, and database updates. - + Args: key_update_item: The key update request item user_api_key_dict: The authenticated user's API key info @@ -1524,22 +1531,22 @@ async def _process_single_key_update( user_api_key_cache: User API key cache proxy_logging_obj: Proxy logging object llm_router: LLM router instance - + Returns: Dict containing the updated key information - + Raises: HTTPException: For various validation and permission errors """ # Validate max_budget _validate_max_budget(key_update_item.max_budget) - + # Get and validate existing key existing_key_row = await _get_and_validate_existing_key( token=key_update_item.key, prisma_client=prisma_client, ) - + # Check team member permissions if prisma_client is not None: await TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint( @@ -1549,7 +1556,7 @@ async def _process_single_key_update( existing_key_row=existing_key_row, user_api_key_cache=user_api_key_cache, ) - + # Create UpdateKeyRequest from BulkUpdateKeyRequestItem update_key_request = UpdateKeyRequest( key=key_update_item.key, @@ -1558,7 +1565,7 @@ async def _process_single_key_update( team_id=key_update_item.team_id, tags=key_update_item.tags, ) - + # Get team object and check team limits if team_id is provided team_obj: Optional[LiteLLM_TeamTableCachedObj] = None if update_key_request.team_id is not None: @@ -1568,18 +1575,16 @@ async def _process_single_key_update( user_api_key_cache=user_api_key_cache, check_db_only=True, ) - + if team_obj is not None and prisma_client is not None: await _check_team_key_limits( team_table=team_obj, data=update_key_request, prisma_client=prisma_client, ) - + # Validate team change if team is being changed - if is_different_team( - data=update_key_request, existing_key_row=existing_key_row - ): + if is_different_team(data=update_key_request, existing_key_row=existing_key_row): if llm_router is None: raise HTTPException( status_code=400, @@ -1590,9 +1595,7 @@ async def _process_single_key_update( if team_obj is None: raise HTTPException( status_code=500, - detail={ - "error": "Team object not found for team change validation" - }, + detail={"error": "Team object not found for team change validation"}, ) validate_key_team_change( key=existing_key_row, @@ -1600,31 +1603,29 @@ async def _process_single_key_update( change_initiated_by=user_api_key_dict, llm_router=llm_router, ) - + # Prepare update data non_default_values = await prepare_key_update_data( data=update_key_request, existing_key_row=existing_key_row ) - + # Update key in database if prisma_client is None: raise HTTPException( status_code=500, detail={"error": "Database not connected"}, ) - + _data = {**non_default_values, "token": key_update_item.key} - response = await prisma_client.update_data( - token=key_update_item.key, data=_data - ) - + response = await prisma_client.update_data(token=key_update_item.key, data=_data) + # Delete cache await _delete_cache_key_object( hashed_token=hash_token(key_update_item.key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) - + # Trigger async hook asyncio.create_task( KeyManagementEventHooks.async_key_updated_hook( @@ -1635,19 +1636,19 @@ async def _process_single_key_update( litellm_changed_by=litellm_changed_by, ) ) - + if response is None: raise ValueError("Failed to update key got response = None") - + # Extract and format updated key info updated_key_info = response.get("data", {}) if hasattr(updated_key_info, "model_dump"): updated_key_info = updated_key_info.model_dump() elif hasattr(updated_key_info, "dict"): updated_key_info = updated_key_info.dict() - + updated_key_info.pop("token", None) - + return updated_key_info @@ -1740,7 +1741,9 @@ async def update_key_fn( if data.max_budget is not None and data.max_budget < 0: raise HTTPException( status_code=400, - detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"} + detail={ + "error": f"max_budget cannot be negative. Received: {data.max_budget}" + }, ) data_json: dict = data.model_dump(exclude_unset=True, exclude_none=True) @@ -1959,13 +1962,11 @@ async def bulk_update_keys( proxy_logging_obj, user_api_key_cache, ) - + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: raise HTTPException( status_code=403, - detail={ - "error": "Only proxy admins can perform bulk key updates" - }, + detail={"error": "Only proxy admins can perform bulk key updates"}, ) if prisma_client is None: @@ -2381,10 +2382,10 @@ async def info_key_fn( # if using pydantic v1 key_info = key_info.dict() key_info.pop("token") - + # Attach object_permission if object_permission_id is set key_info = await attach_object_permission_to_dict(key_info, prisma_client) - + return {"key": key, "info": key_info} except Exception as e: raise handle_exception_on_proxy(e) @@ -2509,7 +2510,9 @@ async def generate_key_helper_fn( # noqa: PLR0915 aliases_json = json.dumps(aliases) config_json = json.dumps(config) permissions_json = json.dumps(permissions) - router_settings_json = safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) + router_settings_json = ( + safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) + ) # Add model_rpm_limit and model_tpm_limit to metadata if model_rpm_limit is not None: @@ -2676,10 +2679,12 @@ async def generate_key_helper_fn( # noqa: PLR0915 ) key_data["created_at"] = getattr(create_key_response, "created_at", None) key_data["updated_at"] = getattr(create_key_response, "updated_at", None) - + # Deserialize router_settings from JSON string to dict for response router_settings_value = key_data.get("router_settings") - if router_settings_value is not None and isinstance(router_settings_value, str): + if router_settings_value is not None and isinstance( + router_settings_value, str + ): try: key_data["router_settings"] = yaml.safe_load(router_settings_value) except yaml.YAMLError: @@ -2762,27 +2767,27 @@ async def can_modify_verification_token( ) -> bool: """ Check if user has permission to modify (delete/regenerate) a verification token. - + Rules: - Proxy admin can modify any key - For team keys: only team admin or key owner can modify - For personal keys: only key owner can modify - + Args: key_info: The verification token to check user_api_key_cache: Cache for user API keys user_api_key_dict: The user making the request prisma_client: Prisma client for database access - + Returns: True if user can modify the key, False otherwise """ is_team_key = _is_team_key(data=key_info) - + # 1. Proxy admin can modify any key if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: return True - + # 2. For team keys: only team admin or key owner can modify if is_team_key and key_info.team_id is not None: # Get team object to check if user is team admin @@ -2792,34 +2797,35 @@ async def can_modify_verification_token( user_api_key_cache=user_api_key_cache, check_db_only=True, ) - + if team_table is None: return False - + # Check if user is team admin if _is_user_team_admin( user_api_key_dict=user_api_key_dict, team_obj=team_table, ): return True - + # Check if the key belongs to the user (they own it) - if key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id: + if ( + key_info.user_id is not None + and key_info.user_id == user_api_key_dict.user_id + ): return True - + # Not team admin and doesn't own the key return False - + # 3. For personal keys: only key owner can modify if key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id: return True - + # Default: deny return False - - async def delete_verification_tokens( tokens: List, user_api_key_cache: DualCache, @@ -2849,10 +2855,10 @@ async def delete_verification_tokens( try: if prisma_client: tokens = [_hash_token_if_needed(token=key) for key in tokens] - _keys_being_deleted: List[LiteLLM_VerificationToken] = ( - await prisma_client.db.litellm_verificationtoken.find_many( - where={"token": {"in": tokens}} - ) + _keys_being_deleted: List[ + LiteLLM_VerificationToken + ] = await prisma_client.db.litellm_verificationtoken.find_many( + where={"token": {"in": tokens}} ) if len(_keys_being_deleted) == 0: @@ -2952,11 +2958,24 @@ def _transform_verification_tokens_to_deleted_records( if org_id_value is not None: record["organization_id"] = org_id_value - for json_field in ["aliases", "config", "permissions", "metadata", "model_spend", "model_max_budget", "router_settings"]: + for json_field in [ + "aliases", + "config", + "permissions", + "metadata", + "model_spend", + "model_max_budget", + "router_settings", + ]: if json_field in record and record[json_field] is not None: record[json_field] = json.dumps(record[json_field]) - for rel_key in ("litellm_budget_table", "litellm_organization_table", "object_permission", "id"): + for rel_key in ( + "litellm_budget_table", + "litellm_organization_table", + "object_permission", + "id", + ): record.pop(rel_key, None) records.append(record) @@ -2971,9 +2990,7 @@ async def _save_deleted_verification_token_records( """Save deleted verification token records to the database.""" if not records: return - await prisma_client.db.litellm_deletedverificationtoken.create_many( - data=records - ) + await prisma_client.db.litellm_deletedverificationtoken.create_many(data=records) async def _persist_deleted_verification_tokens( @@ -3036,9 +3053,9 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config try: - models: Optional[List] = ( - await prisma_client.db.litellm_proxymodeltable.find_many() - ) + models: Optional[ + List + ] = await prisma_client.db.litellm_proxymodeltable.find_many() except Exception: models = None # 2. process model table @@ -3115,7 +3132,9 @@ async def _rotate_master_key( updated_patch=decrypted_cred, new_encryption_key=new_master_key, ) - credential_object_jsonified = jsonify_object(encrypted_cred.model_dump()) + credential_object_jsonified = jsonify_object( + encrypted_cred.model_dump() + ) await prisma_client.db.litellm_credentialstable.update( where={"credential_name": cred.credential_name}, data={ @@ -3427,7 +3446,9 @@ def _validate_reset_spend_value( if reset_to > current_spend: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": f"reset_to ({reset_to}) must be <= current spend ({current_spend})"}, + detail={ + "error": f"reset_to ({reset_to}) must be <= current spend ({current_spend})" + }, ) max_budget = key_in_db.max_budget @@ -3553,11 +3574,11 @@ async def validate_key_list_check( param="user_id", code=status.HTTP_403_FORBIDDEN, ) - complete_user_info_db_obj: Optional[BaseModel] = ( - await prisma_client.db.litellm_usertable.find_unique( - where={"user_id": user_api_key_dict.user_id}, - include={"organization_memberships": True}, - ) + complete_user_info_db_obj: Optional[ + BaseModel + ] = await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_api_key_dict.user_id}, + include={"organization_memberships": True}, ) if complete_user_info_db_obj is None: @@ -3643,10 +3664,10 @@ async def get_admin_team_ids( if complete_user_info is None: return [] # Get all teams that user is an admin of - teams: Optional[List[BaseModel]] = ( - await prisma_client.db.litellm_teamtable.find_many( - where={"team_id": {"in": complete_user_info.teams}} - ) + teams: Optional[ + List[BaseModel] + ] = await prisma_client.db.litellm_teamtable.find_many( + where={"team_id": {"in": complete_user_info.teams}} ) if teams is None: return [] @@ -3691,8 +3712,12 @@ async def list_keys( description="Column to sort by (e.g. 'user_id', 'created_at', 'spend')", ), sort_order: str = Query(default="desc", description="Sort order ('asc' or 'desc')"), - expand: Optional[List[str]] = Query(None, description="Expand related objects (e.g. 'user')"), - status: Optional[str] = Query(None, description="Filter by status (e.g. 'deleted')"), + expand: Optional[List[str]] = Query( + None, description="Expand related objects (e.g. 'user')" + ), + status: Optional[str] = Query( + None, description="Filter by status (e.g. 'deleted')" + ), ) -> KeyListResponseObject: """ List all keys for a given user / team / organization. @@ -3784,7 +3809,9 @@ async def list_keys( message=getattr(e, "detail", f"error({str(e)})"), type=ProxyErrorTypes.internal_server_error, param=getattr(e, "param", "None"), - code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR), + code=getattr( + e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR + ), ) elif isinstance(e, ProxyException): raise e @@ -4617,10 +4644,16 @@ def validate_model_max_budget(model_max_budget: Optional[Dict]) -> None: for _model, _budget_info in model_max_budget.items(): assert isinstance(_model, str) + # Normalize to dict (Pydantic may already parse nested values as BudgetConfig) + _info = ( + _budget_info.model_dump() + if hasattr(_budget_info, "model_dump") + else dict(_budget_info) + ) # /CRUD endpoints can pass budget_limit as a string, so we need to convert it to a float - if "budget_limit" in _budget_info: - _budget_info["budget_limit"] = float(_budget_info["budget_limit"]) - BudgetConfig(**_budget_info) + if "budget_limit" in _info: + _info["budget_limit"] = float(_info["budget_limit"]) + BudgetConfig(**_info) except Exception as e: raise ValueError( f"Invalid model_max_budget: {str(e)}. Example of valid model_max_budget: https://docs.litellm.ai/docs/proxy/users" diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 64f1ec24234..54b9e31a6da 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -22,7 +22,6 @@ from litellm.proxy.litellm_pre_call_utils import ( _get_dynamic_logging_metadata, add_litellm_data_to_request, ) -from litellm.types.utils import SupportedCacheControls @pytest.fixture @@ -496,9 +495,7 @@ def test_add_litellm_data_for_backend_llm_call( from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - user_api_key_dict = UserAPIKeyAuth( - api_key="test_api_key", user_id="test_user_id", org_id="test_org_id" - ) + UserAPIKeyAuth(api_key="test_api_key", user_id="test_user_id", org_id="test_org_id") data = LiteLLMProxyRequestSetup.get_user_from_headers( headers=headers, @@ -1059,7 +1056,7 @@ def test_update_config_fields_default_internal_user_params(monkeypatch): }, }, } - updated_config = proxy_config._update_config_fields(**args) + proxy_config._update_config_fields(**args) assert litellm.default_internal_user_params == { "user_role": "proxy_admin", @@ -1320,6 +1317,61 @@ def test_litellm_verification_token_view_response_with_budget_table( ) +def test_litellm_verification_token_view_budget_does_not_override_key_model_max_budget(): + """ + When key has non-empty model_max_budget, budget's model_max_budget is NOT applied. + Regression test for per-model budget: only apply budget's model_max_budget when key's is empty. + """ + from litellm.proxy._types import LiteLLM_VerificationTokenView + + key_model_max_budget = {"gpt-4": {"max_budget": 50.0, "budget_duration": "1d"}} + args = { + "token": "sk-test-mock-token-303", + "key_name": "sk-...if_g", + "key_alias": None, + "soft_budget_cooldown": False, + "spend": 0.0, + "expires": None, + "models": [], + "aliases": {}, + "config": {}, + "user_id": None, + "team_id": "test", + "permissions": {}, + "max_parallel_requests": None, + "metadata": {}, + "blocked": None, + "tpm_limit": None, + "rpm_limit": None, + "max_budget": None, + "budget_duration": None, + "budget_reset_at": None, + "allowed_cache_controls": [], + "model_spend": {}, + "model_max_budget": key_model_max_budget, + "budget_id": "my-test-tier", + "created_at": "2024-12-26T02:28:52.615+00:00", + "updated_at": "2024-12-26T03:01:51.159+00:00", + "team_spend": None, + "team_max_budget": None, + "team_tpm_limit": None, + "team_rpm_limit": None, + "team_models": [], + "team_metadata": {}, + "team_blocked": False, + "team_alias": None, + "team_members_with_roles": [], + "team_member_spend": None, + "team_model_aliases": None, + "team_member": None, + "litellm_budget_table_model_max_budget": { + "gpt-4o": {"max_budget": 100.0, "budget_duration": "1d"} + }, + } + resp = LiteLLM_VerificationTokenView(**args) + assert resp.model_max_budget == key_model_max_budget + + def test_is_allowed_to_make_key_request(): from litellm.proxy._types import LitellmUserRoles from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -1381,13 +1433,6 @@ def test_get_model_group_info(): assert len(model_list) == 1 -import asyncio -import json -from unittest.mock import AsyncMock, patch - -import pytest - - @pytest.fixture def mock_team_data(): return [ @@ -1444,7 +1489,6 @@ async def test_get_user_info_for_proxy_admin(mock_team_data, mock_key_data): "litellm.proxy.proxy_server.prisma_client", MockPrismaClientDB(mock_team_data, mock_key_data), ): - from litellm.proxy.management_endpoints.internal_user_endpoints import ( _get_user_info_for_proxy_admin, ) @@ -1558,9 +1602,6 @@ def test_update_key_budget_with_temp_budget_increase(): assert _update_key_budget_with_temp_budget_increase(valid_token).max_budget == 200 -from unittest.mock import AsyncMock, MagicMock - - @pytest.mark.asyncio async def test_health_check_not_called_when_disabled(monkeypatch): from litellm.proxy.proxy_server import ProxyStartupEvent @@ -1603,18 +1644,12 @@ async def test_health_check_not_called_when_disabled(monkeypatch): }, ) def test_custom_openapi(mock_get_openapi_schema): - from litellm.proxy.proxy_server import app, custom_openapi + from litellm.proxy.proxy_server import custom_openapi openapi_schema = custom_openapi() assert openapi_schema is not None -import asyncio -from datetime import timedelta -from unittest.mock import AsyncMock, MagicMock - -import pytest - from litellm.proxy.utils import ProxyUpdateSpend @@ -1639,6 +1674,7 @@ async def test_end_user_transactions_reset(): async def test_spend_logs_cleanup_after_error(): # Setup test data import asyncio + mock_client = MagicMock() mock_client.spend_log_transactions = [ {"id": 1, "amount": 10.0}, @@ -1826,7 +1862,7 @@ def test_provider_specific_header_in_request(custom_llm_provider, expected_resul client = HTTPHandler() with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: - resp = litellm.completion( + litellm.completion( model="anthropic/claude-3-5-sonnet-v2@20241022", messages=[{"role": "user", "content": "Hello world"}], provider_specific_header=ProviderSpecificHeader( @@ -2063,7 +2099,7 @@ async def test_post_call_failure_hook_auth_error_key_info_route(): Test that post_call_failure_hook does NOT call _handle_logging_proxy_only_error when we get an auth error from /key/info route (since it's not an LLM API route). """ - from unittest.mock import AsyncMock, Mock, patch + from unittest.mock import AsyncMock, patch from fastapi import HTTPException @@ -2117,7 +2153,7 @@ async def test_post_call_failure_hook_auth_error_llm_api_route(): Test that post_call_failure_hook DOES call _handle_logging_proxy_only_error when we get an auth error from /v1/chat/completions route (since it is an LLM API route). """ - from unittest.mock import AsyncMock, Mock, patch + from unittest.mock import AsyncMock, patch from fastapi import HTTPException @@ -2182,27 +2218,27 @@ async def test_during_call_hook_parallel_execution(): cache = DualCache() proxy_logging = ProxyLogging(user_api_key_cache=cache) execution_order = [] - + class TestGuardrail(CustomGuardrail): def __init__(self, name): super().__init__( guardrail_name=name, event_hook=GuardrailEventHooks.during_call, - default_on=True + default_on=True, ) self.name = name - + async def async_moderation_hook(self, data, user_api_key_dict, call_type): execution_order.append(f"{self.name}_start") await asyncio.sleep(0.1) execution_order.append(f"{self.name}_end") return data - + original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] - + try: litellm.callbacks = [TestGuardrail(f"g{i}") for i in range(3)] - + start_time = asyncio.get_event_loop().time() result = await proxy_logging.during_call_hook( data={"model": "gpt-4", "messages": [{"role": "user", "content": "test"}]}, @@ -2210,14 +2246,22 @@ async def test_during_call_hook_parallel_execution(): call_type="completion", ) execution_time = asyncio.get_event_loop().time() - start_time - + # Verify parallel execution: all start before any end - first_end_idx = next(i for i, item in enumerate(execution_order) if "end" in item) - starts_before_end = sum(1 for item in execution_order[:first_end_idx] if "start" in item) - assert starts_before_end == 3, f"Expected 3 starts before first end, got {starts_before_end}" - + first_end_idx = next( + i for i, item in enumerate(execution_order) if "end" in item + ) + starts_before_end = sum( + 1 for item in execution_order[:first_end_idx] if "start" in item + ) + assert ( + starts_before_end == 3 + ), f"Expected 3 starts before first end, got {starts_before_end}" + # Verify timing: parallel ~0.1s vs sequential ~0.3s - assert execution_time < 0.2, f"Parallel execution took {execution_time}s, expected < 0.2s" + assert ( + execution_time < 0.2 + ), f"Parallel execution took {execution_time}s, expected < 0.2s" assert result["model"] == "gpt-4" finally: litellm.callbacks = original_callbacks @@ -2235,30 +2279,35 @@ async def test_during_call_hook_parallel_execution_with_error(): cache = DualCache() proxy_logging = ProxyLogging(user_api_key_cache=cache) - + class FailingGuardrail(CustomGuardrail): def __init__(self): super().__init__( guardrail_name="failing_guardrail", event_hook=GuardrailEventHooks.during_call, - default_on=True + default_on=True, ) - + async def async_moderation_hook(self, data, user_api_key_dict, call_type): raise ValueError("Guardrail violation detected!") - + original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] - + try: litellm.callbacks = [FailingGuardrail()] - + with pytest.raises(ValueError) as exc_info: await proxy_logging.during_call_hook( - data={"model": "gpt-4", "messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=UserAPIKeyAuth(api_key="test_key", user_id="test_user"), + data={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "test"}], + }, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", user_id="test_user" + ), call_type="completion", ) - + assert "Guardrail violation detected!" in str(exc_info.value) finally: - litellm.callbacks = original_callbacks \ No newline at end of file + litellm.callbacks = original_callbacks diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index fc8373a1746..352db384c88 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -1,30 +1,20 @@ -import json import os import sys -from datetime import datetime -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system-path -from datetime import datetime as dt_object -import time -import pytest -import litellm -import json -from litellm.types.utils import BudgetConfig as GenericBudgetInfo -import os -import sys -from datetime import datetime -from unittest.mock import AsyncMock, patch import pytest + +import litellm from litellm.caching.caching import DualCache from litellm.proxy.hooks.model_max_budget_limiter import ( _PROXY_VirtualKeyModelMaxBudgetLimiter, ) from litellm.proxy._types import UserAPIKeyAuth -import litellm +from litellm.types.utils import BudgetConfig as GenericBudgetInfo # Test class setup @@ -123,3 +113,48 @@ async def test_get_virtual_key_spend_for_model(budget_limiter): key_budget_config=budget_config, ) assert spend == 50.0 + + +@pytest.mark.asyncio +async def test_async_log_success_event_uses_per_model_budget_duration(budget_limiter): + """ + async_log_success_event must use the per-model budget_duration for the cache key + so spend is tracked per model correctly. Regression test for per-model budget implementation. + """ + from litellm.proxy.hooks.model_max_budget_limiter import ( + VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX, + ) + + virtual_key = "test-key-hash" + model = "gpt-4" + budget_duration = "1d" + user_api_key_model_max_budget = { + model: {"budget_limit": 100.0, "time_period": budget_duration}, + } + kwargs = { + "standard_logging_object": { + "response_cost": 0.05, + "model": model, + "metadata": {"user_api_key_hash": virtual_key}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": user_api_key_model_max_budget + }, + }, + } + with patch.object( + budget_limiter, + "_increment_spend_for_key", + new_callable=AsyncMock, + ) as mock_increment: + await budget_limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_increment.assert_awaited_once() + call_kwargs = mock_increment.call_args.kwargs + spend_key = call_kwargs["spend_key"] + assert spend_key == ( + f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{budget_duration}" + ) + assert call_kwargs["response_cost"] == 0.05 diff --git a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py index d5c3ecae7d6..d8c505223d9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py @@ -11,7 +11,6 @@ import litellm.proxy.proxy_server as ps from litellm.proxy.proxy_server import app from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles, CommonProxyErrors -import litellm.proxy.management_endpoints.budget_management_endpoints as bm sys.path.insert( 0, os.path.abspath("../../../") @@ -22,13 +21,12 @@ sys.path.insert( def client_and_mocks(monkeypatch): # Setup MagicMock Prisma mock_prisma = MagicMock() - mock_table = MagicMock() mock_table.create = AsyncMock(side_effect=lambda *, data: data) mock_table.update = AsyncMock(side_effect=lambda *, where, data: {**where, **data}) mock_prisma.db = types.SimpleNamespace( - litellm_budgettable = mock_table, - litellm_dailyspend = mock_table, + litellm_budgettable=mock_table, + litellm_dailyspend=mock_table, ) # Monkeypatch Mocked Prisma client into the server module @@ -79,6 +77,7 @@ async def test_new_budget_db_not_connected(client_and_mocks, monkeypatch): # override the prisma_client that the handler imports at runtime import litellm.proxy.proxy_server as ps + monkeypatch.setattr(ps, "prisma_client", None) # Call /budget/new endpoint @@ -123,6 +122,7 @@ async def test_update_budget_db_not_connected(client_and_mocks, monkeypatch): # override the prisma_client that the handler imports at runtime import litellm.proxy.proxy_server as ps + monkeypatch.setattr(ps, "prisma_client", None) payload = {"budget_id": "any", "max_budget": 1.0} @@ -136,7 +136,7 @@ async def test_update_budget_db_not_connected(client_and_mocks, monkeypatch): async def test_update_budget_allows_null_max_budget(client_and_mocks): """ Test that /budget/update allows setting max_budget to null. - + Previously, using exclude_none=True would drop null values, making it impossible to remove a budget limit. With exclude_unset=True, explicitly setting max_budget to null should include it in the update. @@ -144,11 +144,11 @@ async def test_update_budget_allows_null_max_budget(client_and_mocks): client, _, mock_table = client_and_mocks captured_data = {} - + async def capture_update(*, where, data): captured_data.update(data) return {**where, **data} - + mock_table.update = AsyncMock(side_effect=capture_update) payload = { @@ -159,9 +159,11 @@ async def test_update_budget_allows_null_max_budget(client_and_mocks): assert resp.status_code == 200, resp.text # Verify that max_budget=None was included in the update data - assert "max_budget" in captured_data, "max_budget should be included when explicitly set to null" + assert ( + "max_budget" in captured_data + ), "max_budget should be included when explicitly set to null" assert captured_data["max_budget"] is None, "max_budget should be None" - + mock_table.update.assert_awaited_once() @@ -169,7 +171,7 @@ async def test_update_budget_allows_null_max_budget(client_and_mocks): async def test_new_budget_negative_max_budget(client_and_mocks): """ Test that /budget/new rejects negative max_budget values. - + This prevents the issue where negative budgets would always trigger budget exceeded errors. """ @@ -181,7 +183,7 @@ async def test_new_budget_negative_max_budget(client_and_mocks): } resp = client.post("/budget/new", json=payload) assert resp.status_code == 400, resp.text - + detail = resp.json()["detail"] assert "max_budget cannot be negative" in str(detail) @@ -199,7 +201,7 @@ async def test_new_budget_negative_soft_budget(client_and_mocks): } resp = client.post("/budget/new", json=payload) assert resp.status_code == 400, resp.text - + detail = resp.json()["detail"] assert "soft_budget cannot be negative" in str(detail) @@ -217,7 +219,7 @@ async def test_update_budget_negative_max_budget(client_and_mocks): } resp = client.post("/budget/update", json=payload) assert resp.status_code == 400, resp.text - + detail = resp.json()["detail"] assert "max_budget cannot be negative" in str(detail) @@ -235,6 +237,30 @@ async def test_update_budget_negative_soft_budget(client_and_mocks): } resp = client.post("/budget/update", json=payload) assert resp.status_code == 400, resp.text - + detail = resp.json()["detail"] assert "soft_budget cannot be negative" in str(detail) + + +@pytest.mark.asyncio +async def test_new_budget_invalid_model_max_budget(client_and_mocks, monkeypatch): + """ + Test that /budget/new validates model_max_budget and returns 400 for invalid structure. + Per-model budget implementation: validate_model_max_budget is called in new_budget. + """ + import litellm.proxy.proxy_server as ps + + monkeypatch.setattr(ps, "premium_user", True) + + client, _, _ = client_and_mocks + + payload = { + "budget_id": "budget_invalid_mmb", + "max_budget": 10.0, + "model_max_budget": {"gpt-4": "not-a-dict"}, + } + resp = client.post("/budget/new", json=payload) + # Pydantic may reject invalid structure with 422 before our validator runs + assert resp.status_code in (400, 422), resp.text + detail = resp.json()["detail"] + assert "model_max_budget" in str(detail) or "dictionary" in str(detail).lower() From e24ea2897a9b8217eb89c004782afdb34db85022 Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Sat, 7 Feb 2026 22:22:12 -0800 Subject: [PATCH 013/117] fix: empty guardrails/policies arrays should not trigger enterprise license check (#20567) * fix: empty guardrails/policies arrays should not trigger enterprise license check (#20304) The UI sends empty arrays for enterprise-only fields (guardrails, policies, logging) even when the user has not configured these features. The backend `is not None` check treated `[]` as a truthy intent to use the feature, falsely requiring an enterprise license for basic team operations. Backend: Add `and updated_kv[field] != [] and updated_kv[field] != {}` guards in `_update_metadata_fields` so empty collections are skipped. UI: Conditionally omit guardrails, logging, and policies from the payload when empty instead of defaulting to `[]`. Fixes #20304 * fix: allow clearing fields with empty collections while skipping enterprise check Address PR review feedback: 1. Move the empty-collection guard into _update_metadata_field (singular) so that empty lists/dicts skip only the premium license check but still get written into metadata. This lets users intentionally clear a previously-set field (e.g. guardrails: []) without being blocked, while the UI's default empty arrays still don't trigger a false enterprise error. 2. Remove sys.path hack from test file; use standard imports that work with pytest discovery. 3. Add tests verifying that empty collections are moved into metadata (field clearing works) even though they bypass the premium check. Fixes #20304 --- .../management_endpoints/common_utils.py | 9 +- .../management_endpoints/test_common_utils.py | 162 ++++++++++++++++++ .../src/components/team/team_info.tsx | 6 +- 3 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 tests/test_litellm/proxy/management_endpoints/test_common_utils.py diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 8f7dd4f8dfa..24a41a2361b 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -216,7 +216,14 @@ def _update_metadata_field(updated_kv: dict, field_name: str) -> None: field_name: Name of the metadata field being updated """ if field_name in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - _premium_user_check() + value = updated_kv.get(field_name) + # Skip the premium check for empty collections ([] or {}). + # The UI sends these as defaults even when the user hasn't configured + # any enterprise features (see issue #20304). However, we still + # proceed with the update so that users can intentionally clear a + # previously-set field by sending an empty list/dict. + if value is not None and value != [] and value != {}: + _premium_user_check() if field_name in updated_kv and updated_kv[field_name] is not None: # remove field from updated_kv diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_utils.py b/tests/test_litellm/proxy/management_endpoints/test_common_utils.py new file mode 100644 index 00000000000..b372476c3d6 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_common_utils.py @@ -0,0 +1,162 @@ +""" +Tests for litellm/proxy/management_endpoints/common_utils.py + +Covers the fix for GitHub issue #20304: +Empty guardrails/policies arrays sent by the UI should NOT trigger the +enterprise (premium) license check, but should still be applied so that +users can intentionally clear previously-set fields. +""" + +from unittest.mock import patch + +from litellm.proxy.management_endpoints.common_utils import ( + _update_metadata_fields, +) + + +class TestUpdateMetadataFieldsEmptyCollections: + """ + Regression tests for issue #20304. + + The UI sends empty arrays (`[]`) for enterprise-only fields like + guardrails, policies, and logging even when the user hasn't configured + these features. The backend must not treat empty collections as an + intent to use the feature, and therefore must not trigger the premium + license check. + + However, empty collections must still be written into metadata so that + users can intentionally clear a previously-set field (e.g. removing all + guardrails by sending `guardrails: []`). + """ + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_empty_list_does_not_trigger_premium_check(self, mock_premium_check): + """Empty lists for premium fields must not trigger the premium check.""" + updated_kv = { + "team_id": "test-team", + "guardrails": [], + "policies": [], + "logging": [], + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_empty_list_still_updates_metadata(self, mock_premium_check): + """ + Empty lists must still be moved into metadata so users can clear + previously-set fields (e.g. remove all guardrails). + """ + updated_kv = { + "team_id": "test-team", + "guardrails": [], + "policies": [], + } + _update_metadata_fields(updated_kv=updated_kv) + # The fields should have been moved into metadata + assert "guardrails" not in updated_kv, ( + "guardrails should be popped from top-level" + ) + assert "policies" not in updated_kv, ( + "policies should be popped from top-level" + ) + assert updated_kv["metadata"]["guardrails"] == [] + assert updated_kv["metadata"]["policies"] == [] + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_empty_dict_does_not_trigger_premium_check(self, mock_premium_check): + """Empty dicts for premium fields must not trigger the premium check.""" + updated_kv = { + "team_id": "test-team", + "secret_manager_settings": {}, + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_empty_dict_still_updates_metadata(self, mock_premium_check): + """ + Empty dicts must still be moved into metadata so users can clear + previously-set fields. + """ + updated_kv = { + "team_id": "test-team", + "secret_manager_settings": {}, + } + _update_metadata_fields(updated_kv=updated_kv) + assert "secret_manager_settings" not in updated_kv, ( + "secret_manager_settings should be popped from top-level" + ) + assert updated_kv["metadata"]["secret_manager_settings"] == {} + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_none_value_does_not_trigger_premium_check(self, mock_premium_check): + """None values for premium fields should be silently ignored.""" + updated_kv = { + "team_id": "test-team", + "guardrails": None, + "policies": None, + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_absent_fields_do_not_trigger_premium_check(self, mock_premium_check): + """Fields not present in the dict should not trigger premium check.""" + updated_kv = { + "team_id": "test-team", + "team_alias": "example-team", + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_non_empty_list_triggers_premium_check(self, mock_premium_check): + """Non-empty lists for premium fields should trigger the premium check.""" + updated_kv = { + "team_id": "test-team", + "guardrails": ["my-guardrail"], + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_non_empty_value_triggers_premium_check(self, mock_premium_check): + """Non-empty string values for premium fields should trigger the premium check.""" + updated_kv = { + "team_id": "test-team", + "tags": ["production"], + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_non_empty_list_updates_metadata(self, mock_premium_check): + """Non-empty lists should be moved into metadata.""" + updated_kv = { + "team_id": "test-team", + "guardrails": ["my-guardrail"], + } + _update_metadata_fields(updated_kv=updated_kv) + assert "guardrails" not in updated_kv + assert updated_kv["metadata"]["guardrails"] == ["my-guardrail"] + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_ui_typical_payload_does_not_trigger_premium_check(self, mock_premium_check): + """ + Simulate the exact payload the UI sends when no enterprise features + are configured. This must NOT trigger the premium check. + """ + # This is the payload structure the UI sends (from issue #20304) + updated_kv = { + "team_id": "67848772-1a8b-4343-938c-17e60f1db860", + "team_alias": "example-team", + "models": ["gpt-4"], + "metadata": { + "guardrails": [], + "logging": [], + }, + "policies": [], + } + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 35f5b87e071..014f8fb9010 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -465,8 +465,8 @@ const TeamInfoView: React.FC = ({ budget_duration: values.budget_duration, metadata: { ...parsedMetadata, - guardrails: values.guardrails || [], - logging: values.logging_settings || [], + ...(values.guardrails?.length > 0 ? { guardrails: values.guardrails } : {}), + ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), disable_global_guardrails: values.disable_global_guardrails || false, soft_budget_alerting_emails: typeof values.soft_budget_alerting_emails === "string" @@ -477,7 +477,7 @@ const TeamInfoView: React.FC = ({ : values.soft_budget_alerting_emails || [], ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), }, - policies: values.policies || [], + ...(values.policies?.length > 0 ? { policies: values.policies } : {}), organization_id: values.organization_id, }; From 3b043ee8bfebe29b9f9071e658f209087042f5f0 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:53:01 +0530 Subject: [PATCH 014/117] fix critical CVE vulnerabliltes (#20683) --- .dockerignore | 2 +- Dockerfile | 31 ++++++++++++++++++- ci_cd/security_scans.sh | 5 +-- docker/Dockerfile.custom_ui | 13 +++++++- docker/Dockerfile.database | 29 ++++++++++++++--- docker/Dockerfile.dev | 27 +++++++++++++++- docker/Dockerfile.non_root | 29 ++++++++++++++--- docs/my-website/package.json | 2 ++ litellm-js/spend-logs/package.json | 4 ++- package.json | 4 ++- requirements.txt | 5 +++ tests/proxy_admin_ui_tests/package.json | 4 ++- .../ui_unit_tests/package.json | 4 ++- ui/litellm-dashboard/package.json | 2 ++ 14 files changed, 141 insertions(+), 20 deletions(-) diff --git a/.dockerignore b/.dockerignore index 76e31546c2f..a487d2a859a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -48,7 +48,7 @@ dist/ build/ *.egg-info/ .DS_Store -node_modules/ +**/node_modules *.log .env .env.local diff --git a/Dockerfile b/Dockerfile index 717ec2bcb77..5e93a0c627e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,22 @@ USER root # Install runtime dependencies (libsndfile needed for audio processing on ARM64) RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@latest + npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ + # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested + # levels inside its dependency tree. `npm install -g ` only creates a + # SEPARATE global package, it does NOT replace npm's internal copies. + # We must find and replace EVERY copy inside npm's directory. + GLOBAL="$(npm root -g)" && \ + find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + npm cache clean --force WORKDIR /app # Copy the current directory contents into the container at /app @@ -71,6 +86,20 @@ RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/nul RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ find /usr/lib -type d -path "*/tornado/test" -delete +# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete +# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. +# Patch every copy of tar, glob, and brace-expansion inside that tree. +RUN GLOBAL="$(npm root -g)" && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done + # Install semantic_router and aurelio-sdk using script # Convert Windows line endings to Unix and make executable RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 770610c2a3b..3ffa13c444f 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -155,10 +155,7 @@ run_grype_scans() { "CVE-2025-12781" # No fix available yet "CVE-2025-11468" # No fix available yet "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization - "GHSA-7h2j-956f-4vf2" # @isaacs/brace-expansion ReDoS - npm tooling dependency, not used in application runtime - "GHSA-hx9q-6w63-j58v" # orjson deep recursion - no fix available yet - "GHSA-8qq5-rm4j-mr97" # node-tar symlink poisoning - npm tooling dependency, tar CLI not exposed in application code - "GHSA-29xp-372q-xqph" # node-tar race condition - npm tooling dependency, tar CLI not exposed in application code + "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index 57926bcd170..177d7b7b12a 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -6,7 +6,18 @@ WORKDIR /app # Install Node.js and npm (adjust version as needed) RUN apt-get update && apt-get install -y nodejs npm && \ - npm install -g npm@latest tar@latest + npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ + GLOBAL="$(npm root -g)" && \ + find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + npm cache clean --force # Copy the UI source into the container COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index ecbe76446f6..a6fcd98ab6d 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -50,7 +50,18 @@ USER root # Install runtime dependencies RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ - npm install -g npm@latest tar@latest + npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \ + GLOBAL="$(npm root -g)" && \ + find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + npm cache clean --force WORKDIR /app # Copy the current directory contents into the container at /app @@ -64,9 +75,19 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels -# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130) -RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \ - if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi +# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete +# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. +# Patch every copy of tar, glob, and brace-expansion inside that tree. +RUN GLOBAL="$(npm root -g)" && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done # Install semantic_router and aurelio-sdk using script # Convert Windows line endings to Unix and make executable diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index ae557d4647f..bc1d22d5e05 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -62,7 +62,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ nodejs \ npm \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@latest tar@latest + && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \ + && GLOBAL="$(npm root -g)" \ + && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done \ + && npm cache clean --force WORKDIR /app @@ -80,6 +91,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ rm -f *.whl && \ rm -rf /wheels +# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete +# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. +# Patch every copy of tar, glob, and brace-expansion inside that tree. +RUN GLOBAL="$(npm root -g)" && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done + # Generate prisma client and set permissions # Convert Windows line endings to Unix for entrypoint scripts RUN prisma generate && \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4b09755ed7d..64126bb0292 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -104,7 +104,18 @@ RUN for i in 1 2 3; do \ && for i in 1 2 3; do \ apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ done \ - && npm install -g npm@latest tar@latest + && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \ + && GLOBAL="$(npm root -g)" \ + && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done \ + && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done \ + && npm cache clean --force # Copy artifacts from builder COPY --from=builder /app/requirements.txt /app/requirements.txt @@ -146,9 +157,19 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ fi; \ fi -# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130) -RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \ - if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi +# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete +# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. +# Patch every copy of tar, glob, and brace-expansion inside that tree. +RUN GLOBAL="$(npm root -g)" && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done # Permissions, cleanup, and Prisma prep # Convert Windows line endings to Unix for entrypoint scripts diff --git a/docs/my-website/package.json b/docs/my-website/package.json index 4c3db680565..4af7a168f83 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -61,6 +61,8 @@ "mermaid": ">=11.10.0", "gray-matter": "4.0.3", "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2", "mdast-util-to-hast": ">=13.2.1", "lodash-es": ">=4.17.23" diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index 9c1c2d4f6dc..67292567145 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -11,6 +11,8 @@ "tsx": "^4.7.1" }, "overrides": { - "glob": ">=11.1.0" + "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1" } } diff --git a/package.json b/package.json index 7f90fd0aeb9..ab9e15f46a7 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,8 @@ "jest": "^29.7.0" }, "overrides": { - "glob": ">=11.1.0" + "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1" } } diff --git a/requirements.txt b/requirements.txt index 1f21cc62bc6..f680de120c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,9 @@ # LITELLM PROXY DEPENDENCIES # +# Security: explicit pins for transitive deps (CVE fixes) +urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 +tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 +filelock>=3.20.1 # CVE-2025-68146 + anyio==4.8.0 # openai + http req. httpx==0.28.1 openai==2.9.0 # openai req. diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index cbd25be8816..48de2c1dba2 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -12,6 +12,8 @@ "@types/node": "^22.5.5" }, "overrides": { - "glob": ">=11.1.0" + "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1" } } diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json index 7d82ee2e1a6..4c7d7addf0e 100644 --- a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json @@ -24,6 +24,8 @@ "react-dom": "^18.2.0" }, "overrides": { - "glob": ">=11.1.0" + "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1" } } \ No newline at end of file diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index efb11fec386..76ac97f008c 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -84,6 +84,8 @@ "mermaid": ">=11.10.0", "js-yaml": ">=4.1.1", "glob": ">=11.1.0", + "tar": ">=7.5.7", + "@isaacs/brace-expansion": ">=5.0.1", "node-forge": ">=1.3.2", "lodash-es": ">=4.17.23", "lodash": ">=4.17.23" From 7f93ff9e83ef18510000ce8a72c09cf24172b8e0 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Sun, 8 Feb 2026 12:12:19 +0530 Subject: [PATCH 015/117] fix: add hook to handle db case (#20635) --- litellm/integrations/datadog/datadog.py | 113 ++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 127b0e53fa8..64e0b26a8e7 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -45,7 +45,14 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus -from litellm.types.integrations.datadog import * +from litellm.types.integrations.datadog import ( + DD_ERRORS, + DD_MAX_BATCH_SIZE, + DataDogStatus, + DatadogInitParams, + DatadogPayload, + DatadogProxyFailureHookJsonMessage, +) from litellm.types.services import ServiceLoggerPayload, ServiceTypes from litellm.types.utils import StandardLoggingPayload @@ -85,12 +92,14 @@ class DataDogLogger( """ try: verbose_logger.debug("Datadog: in init datadog logger") - + self.is_mock_mode = should_use_datadog_mock() - + if self.is_mock_mode: create_mock_datadog_client() - verbose_logger.debug("[DATADOG MOCK] Datadog logger initialized in mock mode") + verbose_logger.debug( + "[DATADOG MOCK] Datadog logger initialized in mock mode" + ) ######################################################### # Handle datadog_params set as litellm.datadog_params @@ -209,6 +218,96 @@ class DataDogLogger( ) pass + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: Any, + traceback_str: Optional[str] = None, + ) -> Optional[Any]: + """ + Log proxy-level failures (e.g. 401 auth, DB connection errors) to Datadog. + + Ensures failures that occur before or outside the LLM completion flow + (e.g. ConnectError during auth when DB is down) are visible in Datadog + alongside Prometheus. + """ + try: + from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + ) + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + error_information = StandardLoggingPayloadSetup.get_error_information( + original_exception=original_exception, + traceback_str=traceback_str, + ) + _code = error_information.get("error_code") or "" + status_code: Optional[int] = None + if _code and str(_code).strip().isdigit(): + status_code = int(_code) + + # Use project-standard sanitized user context when running in proxy + user_context: Dict[str, Any] = {} + try: + from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + ) + + _meta = ( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) + ) + user_context = dict(_meta) if isinstance(_meta, dict) else _meta + except Exception: + # Fallback if proxy not available (e.g. SDK-only): minimal safe fields + if hasattr(user_api_key_dict, "request_route"): + user_context["request_route"] = getattr( + user_api_key_dict, "request_route", None + ) + if hasattr(user_api_key_dict, "team_id"): + user_context["team_id"] = getattr( + user_api_key_dict, "team_id", None + ) + if hasattr(user_api_key_dict, "user_id"): + user_context["user_id"] = getattr( + user_api_key_dict, "user_id", None + ) + if hasattr(user_api_key_dict, "end_user_id"): + user_context["end_user_id"] = getattr( + user_api_key_dict, "end_user_id", None + ) + + message_payload: DatadogProxyFailureHookJsonMessage = { + "exception": error_information.get("error_message") + or str(original_exception), + "error_class": error_information.get("error_class") + or original_exception.__class__.__name__, + "status_code": status_code, + "traceback": error_information.get("traceback") or "", + "user_api_key_dict": user_context, + } + + dd_payload = DatadogPayload( + ddsource=get_datadog_source(), + ddtags=get_datadog_tags(), + hostname=get_datadog_hostname(), + message=safe_dumps(message_payload), + service=get_datadog_service(), + status=DataDogStatus.ERROR, + ) + self._add_trace_context_to_payload(dd_payload=dd_payload) + self.log_queue.append(dd_payload) + + if len(self.log_queue) >= self.batch_size: + await self.async_send_batch() + except Exception as e: + verbose_logger.exception( + f"Datadog: async_post_call_failure_hook - {str(e)}\n{traceback.format_exc()}" + ) + return None + async def async_send_batch(self): """ Sends the in memory logs queue to datadog api @@ -230,9 +329,11 @@ class DataDogLogger( len(self.log_queue), self.intake_url, ) - + if self.is_mock_mode: - verbose_logger.debug("[DATADOG MOCK] Mock mode enabled - API calls will be intercepted") + verbose_logger.debug( + "[DATADOG MOCK] Mock mode enabled - API calls will be intercepted" + ) response = await self.async_send_compressed_data(self.log_queue) if response.status_code == 413: From c9df996b7725b17f095ffced523ca2234b51de31 Mon Sep 17 00:00:00 2001 From: jwang-gif Date: Sat, 7 Feb 2026 22:44:17 -0800 Subject: [PATCH 016/117] Add team policy mapping for zguard (#20608) * support policy mapping on team key level * update document * update document * address comments * update document * add unit test for new feature * add more test case --- .../docs/proxy/guardrails/zscaler_ai_guard.md | 28 +++- .../zscaler_ai_guard/zscaler_ai_guard.py | 60 +++++--- .../guardrails_tests/test_zscaler_ai_guard.py | 129 +++++++++++++++++- 3 files changed, 196 insertions(+), 21 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md b/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md index 94f31c3bfdf..2e626004238 100644 --- a/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md +++ b/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md @@ -100,7 +100,7 @@ In cases where encounter other errors when apply Zscaler AI Guard, return exampl } } ``` -## 6. Sending User Information to Zscaler AI Guard for Analysis (Optional) +## 6. Sending User Information to Zscaler AI Guard (Optional) If you need to send end-user information to Zscaler AI Guard for analysis, you can set the configuration in the environment variables to True and include the relevant information in custom_headers on Zscaler AI Guard. - To send user_api_key_alias: @@ -133,4 +133,30 @@ curl -i http://localhost:8165/v1/chat/completions \ "zguard_policy_id": } }' +``` + +## 8. Set Custom Zscaler AI Guard Policy on Litellm Team OR Key Metadata (Optional) +In addition to setting `zguard_policy_id` in a request or the configuration file, you can also set it in the metadata for LiteLLM Team or Key. The `zguard_policy_id` is determined using the following order of precedence: request, Key, Team, config file. This logic is illustrated below: +``` +user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {} +team_metadata = metadata.get("team_metadata", {}) or {} +policy_id = ( + metadata.get("zguard_policy_id") + if "zguard_policy_id" in metadata + else ( + user_api_key_metadata.get("zguard_policy_id") + if "zguard_policy_id" in user_api_key_metadata + else ( + team_metadata.get("zguard_policy_id") + if "zguard_policy_id" in team_metadata + else self.policy_id + ) + ) + ) +``` +You can leverage this feature to apply multiple policies configured on the Zscaler AI Guard (ZGuard) to traffic from different applications. (Note: It is recommended to map policies using either Team or Key metadata, but not a mix of both.) + +Example set in Team/Key Metadata, you can set From UI: +``` +{"zguard_policy_id": 100} ``` \ No newline at end of file diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index d62bbb0b459..c60752d7952 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -92,14 +92,34 @@ class ZscalerAIGuard(CustomGuardrail): Raises: Exception: If content is blocked by Zscaler AI Guard """ + texts = inputs.get("texts", []) try: verbose_proxy_logger.debug(f"ZscalerAIGuard: Checking {len(texts)} text(s)") + metadata = request_data.get("metadata", {}) - custom_policy_id = request_data.get("metadata", {}).get( - "zguard_policy_id", self.policy_id + user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {} + team_metadata = metadata.get("team_metadata", {}) or {} + + # Precedence for policy_id: + # 1. metadata.zguard_policy_id # request level + # 2. user_api_key_metadata.zguard_policy_id # Key level + # 3. team_metadata.zguard_policy_id # Team level + # 4. self.policy_id (from environment) # Global + policy_id = ( + metadata.get("zguard_policy_id") + if "zguard_policy_id" in metadata + else ( + user_api_key_metadata.get("zguard_policy_id") + if "zguard_policy_id" in user_api_key_metadata + else ( + team_metadata.get("zguard_policy_id") + if "zguard_policy_id" in team_metadata + else self.policy_id + ) + ) ) - verbose_proxy_logger.debug(f"custom_policy_id: {custom_policy_id}") + verbose_proxy_logger.info(f"policy_id applied: {policy_id}") kwargs = {} if self.send_user_api_key_alias: @@ -116,27 +136,29 @@ class ZscalerAIGuard(CustomGuardrail): ) verbose_proxy_logger.debug(f"inside apply_guardrail kwargs: {kwargs}") - # Check each text (Zscaler processes one at a time) - for text in texts: + zscaler_ai_guard_result = None + direction = "OUT" if input_type == "response" else "IN" + verbose_proxy_logger.debug(f"direction: {direction}") + # Concatenate all texts and send to Zscaler AI Guard + if texts: + concatenated_text = " ".join(texts) zscaler_ai_guard_result = await self.make_zscaler_ai_guard_api_call( zscaler_ai_guard_url=self.zscaler_ai_guard_url, api_key=self.api_key, - policy_id=self.policy_id, - direction="IN", - content=text, + policy_id=policy_id, + direction=direction, + content=concatenated_text, **kwargs, ) - - if ( - zscaler_ai_guard_result - and zscaler_ai_guard_result.get("action") == "BLOCK" - ): - blocking_info = zscaler_ai_guard_result.get( - "zscaler_ai_guard_response" - ) - error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}" - raise Exception(error_message) - + if ( + zscaler_ai_guard_result + and zscaler_ai_guard_result.get("action") == "BLOCK" + ): + blocking_info = zscaler_ai_guard_result.get( + "zscaler_ai_guard_response" + ) + error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}" + raise Exception(error_message) except Exception as e: verbose_proxy_logger.error( "ZscalerAIGuard: Failed to apply guardrail: %s", str(e) diff --git a/tests/guardrails_tests/test_zscaler_ai_guard.py b/tests/guardrails_tests/test_zscaler_ai_guard.py index cf70af510c8..9d519c17f6a 100644 --- a/tests/guardrails_tests/test_zscaler_ai_guard.py +++ b/tests/guardrails_tests/test_zscaler_ai_guard.py @@ -116,4 +116,131 @@ def test_extract_blocking_info(): blocking_info = guardrail.extract_blocking_info(response) assert blocking_info["transactionId"] == "12345" - assert blocking_info["blockingDetectors"] == ["detector1"] \ No newline at end of file + assert blocking_info["blockingDetectors"] == ["detector1"] + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_apply_guardrail_text_concatenation(mock_api_call): + """ + Test that `apply_guardrail` correctly concatenates texts. + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["Hello", "world"]} + request_data = {} + + await guardrail.apply_guardrail(inputs, request_data, "request") + + mock_api_call.assert_called_once() + call_args = mock_api_call.call_args + assert call_args.kwargs["content"] == "Hello world" + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_policy_id_from_request_metadata(mock_api_call): + """ + Test policy_id is picked from request metadata (highest precedence). + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["test"]} + request_data = { + "metadata": { + "zguard_policy_id": 1, + "user_api_key_metadata": {"zguard_policy_id": 2}, + "team_metadata": {"zguard_policy_id": 3}, + } + } + + await guardrail.apply_guardrail(inputs, request_data, "request") + + mock_api_call.assert_called_once() + assert mock_api_call.call_args.kwargs["policy_id"] == 1 + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_policy_id_from_user_api_key_metadata(mock_api_call): + """ + Test policy_id is picked from user_api_key_metadata (2nd precedence). + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["test"]} + request_data = { + "metadata": { + "user_api_key_metadata": {"zguard_policy_id": 2}, + "team_metadata": {"zguard_policy_id": 3}, + } + } + + await guardrail.apply_guardrail(inputs, request_data, "request") + + mock_api_call.assert_called_once() + assert mock_api_call.call_args.kwargs["policy_id"] == 2 + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_policy_id_from_team_metadata(mock_api_call): + """ + Test policy_id is picked from team_metadata (3rd precedence). + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["test"]} + request_data = {"metadata": {"team_metadata": {"zguard_policy_id": 3}}} + + await guardrail.apply_guardrail(inputs, request_data, "request") + + mock_api_call.assert_called_once() + assert mock_api_call.call_args.kwargs["policy_id"] == 3 + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_policy_id_from_init(mock_api_call): + """ + Test policy_id is picked from guardrail initialization (lowest precedence). + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["test"]} + request_data = {"metadata": {}} + + await guardrail.apply_guardrail(inputs, request_data, "request") + + mock_api_call.assert_called_once() + assert mock_api_call.call_args.kwargs["policy_id"] == 100 + +@pytest.mark.asyncio +@patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call", + new_callable=AsyncMock, +) +async def test_policy_id_zero_from_request_metadata(mock_api_call): + """ + Test policy_id=0 is correctly picked. Make sure pick exact policy_id which users set + """ + guardrail = ZscalerAIGuard(policy_id=100) + inputs = {"texts": ["test"]} + request_data = { + "metadata": { + "zguard_policy_id": 0, + } + } + await guardrail.apply_guardrail(inputs, request_data, "request") + mock_api_call.assert_called_once() + assert mock_api_call.call_args.kwargs["policy_id"] == 0 From 55a89f279f219fc9b5a9527581a3b449c0fb8e4c Mon Sep 17 00:00:00 2001 From: nuernber Date: Sat, 7 Feb 2026 22:51:06 -0800 Subject: [PATCH 017/117] feat: add support for anthropic_messages call type in prompt caching (#19233) * feat: add support for anthropic_messages call type in prompt caching * test: move anthropic_messages prompt caching test to main router test file * add tutorial on using claude code with prompt cache routing --- .../claude_code_prompt_cache_routing.md | 43 +++++++ docs/my-website/sidebars.js | 1 + .../prompt_caching_deployment_check.py | 3 +- tests/test_litellm/test_router.py | 121 ++++++++++++++++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md diff --git a/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md new file mode 100644 index 00000000000..bbb29489856 --- /dev/null +++ b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md @@ -0,0 +1,43 @@ +# Claude Code - Prompt Cache Routing + +Claude's [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) feature helps to optimize API usage through attempting to cache prompts and re-use cached prompts during subsequent API calls. This feature is used by Claude Code. + +When LiteLLM [load balancing](../proxy/load_balancing.md) is enabled, to ensure this prompt caching feature still works with Claude Code, LiteLLM needs to be configured to use the `PromptCachingDeploymentCheck` pre-call check. This pre-call check will ensure that API calls that used prompt caching are remembered and that subsequent API calls that try to use that prompt caching are routed to the same model deployment where a cache write occurred. + +## Set Up + +1. Configure the router so that it uses the `PromptCachingDeploymentCheck` (via setting the `optional_pre_call_checks` property), and configure the models so that they can access multiple deployments of Claude; below, we show an example for multiple AWS accounts (referred to as `account-1` and `account-2`, using the `aws_profile_name` property): +```yaml +router_settings: + optional_pre_call_checks: ["prompt_caching"] + +model_list: +- litellm_params: + model: us.anthropic.claude-sonnet-4-5-20250929-v1:0 + aws_profile_name: account-1 + aws_region_name: us-west-2 + model_info: + litellm_provider: bedrock + model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0 +- litellm_params: + model: us.anthropic.claude-sonnet-4-5-20250929-v1:0 + aws_profile_name: account-2 + aws_region_name: us-west-2 + model_info: + litellm_provider: bedrock + model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0 +``` +2. Utilize Claude Code: + 1. Launch Claude Code, which will do a warm-up API call that tries to cache its warm-up prompt and its system prompt. + 2. Wait a few seconds, then quit Claude Code and re-open it. + 3. You'll notice that the warm-up API call successfully gets a cache hit (if using Claude Code in an IDE like VS Code, ensure that you don't do anything between step 2.1 and 2.2 here, otherwise there may not be a cache hit): + 1. Go to the [LiteLLM Request Logs page](../proxy/ui_logs.md) in the Admin UI + 2. Click on the individual requests to see (a) the cache creation and cache read tokens; and (b) the Model ID. In particular, the API call from step 2.1 should show a cache write, and the API call from step 2.2 should show a cache read; in addition, the Model ID should be equal (meaning the API call is getting forwarded to the same AWS account). + +## Related + +- [Claude Code - Quickstart](./claude_responses_api.md) +- [Claude Code - Customer Tracking](./claude_code_customer_tracking.md) +- [Claude Code - Plugin Marketplace](./claude_code_plugin_marketplace.md) +- [Claude Code - WebSearch](./claude_code_websearch.md) +- [Proxy - Load Balancing](../proxy/load_balancing.md) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6c354b7c041..9d9007a916c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -125,6 +125,7 @@ const sidebars = { "tutorials/claude_responses_api", "tutorials/claude_code_max_subscription", "tutorials/claude_code_customer_tracking", + "tutorials/claude_code_prompt_cache_routing", "tutorials/claude_code_websearch", "tutorials/claude_mcp", "tutorials/claude_non_anthropic_models", diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index d3d237d9f28..e9c4b69d8ef 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -61,9 +61,10 @@ class PromptCachingDeploymentCheck(CustomLogger): if ( call_type != CallTypes.completion.value and call_type != CallTypes.acompletion.value + and call_type != CallTypes.anthropic_messages.value ): # only use prompt caching for completion calls verbose_logger.debug( - "litellm.router_utils.pre_call_checks.prompt_caching_deployment_check: skipping adding model id to prompt caching cache, CALL TYPE IS NOT COMPLETION" + "litellm.router_utils.pre_call_checks.prompt_caching_deployment_check: skipping adding model id to prompt caching cache, CALL TYPE IS NOT COMPLETION or ANTHROPIC MESSAGE" ) return diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 08ae804ea80..75ec806ee17 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1869,3 +1869,124 @@ async def test_aguardrail(): assert result["result"] == "success" assert result["selected_guardrail"]["id"] == "guardrail-1" + +@pytest.mark.asyncio +async def test_anthropic_messages_call_type_is_cached(): + """ + Regression test: Verify that anthropic_messages call type is allowed + in PromptCachingDeploymentCheck.async_log_success_event. + """ + import asyncio + from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( + PromptCachingDeploymentCheck, + ) + from litellm.router_utils.prompt_caching_cache import PromptCachingCache + from litellm.caching.dual_cache import DualCache + from litellm.types.utils import CallTypes + from litellm.types.utils import ( + StandardLoggingPayload, + StandardLoggingModelInformation, + StandardLoggingMetadata, + StandardLoggingHiddenParams, + ) + + # Create mock standard logging payload inline + def create_standard_logging_payload() -> StandardLoggingPayload: + return StandardLoggingPayload( + id="test_id", + call_type="completion", + response_cost=0.1, + response_cost_failure_debug_info=None, + status="success", + total_tokens=30, + prompt_tokens=20, + completion_tokens=10, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=1234567890.5, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!"}], + response={"choices": [{"message": {"content": "Hi there!"}}]}, + error_str=None, + model_parameters={"stream": True}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.1", + additional_headers=None, + ), + ) + + cache = DualCache() + deployment_check = PromptCachingDeploymentCheck(cache=cache) + prompt_cache = PromptCachingCache(cache=cache) + + # Create messages with enough tokens to pass the caching threshold + test_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "test long message here" * 1024, + "cache_control": { + "type": "ephemeral", + "ttl": "5m" + } + } + ] + } + ] + test_model_id = "test-model-id-123" + + # Create a payload with anthropic_messages call type + payload = create_standard_logging_payload() + payload["call_type"] = CallTypes.anthropic_messages.value + payload["messages"] = test_messages + payload["model"] = "anthropic/claude-3-5-sonnet-20240620" + payload["model_id"] = test_model_id + + # Log the success event (should cache the model_id) + await deployment_check.async_log_success_event( + kwargs={"standard_logging_object": payload}, + response_obj={}, + start_time=1234567890.0, + end_time=1234567891.0, + ) + + # Small delay to ensure cache write completes + await asyncio.sleep(0.1) + + # Verify that the model_id was actually cached + cached_result = await prompt_cache.async_get_model_id( + messages=test_messages, + tools=None, + ) + + # This assertion will FAIL if anthropic_messages is filtered out + assert cached_result is not None, "Model ID should be cached for anthropic_messages call type" + assert cached_result["model_id"] == test_model_id, f"Expected {test_model_id}, got {cached_result['model_id']}" From 1fecae0399a284cd0c31322cbfdb9a1372f59958 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Sun, 8 Feb 2026 03:57:04 -0300 Subject: [PATCH 018/117] docs: add SDK proxy authentication (OAuth2/JWT auto-refresh) documentation (#20680) Adds documentation for the litellm.proxy_auth feature that automatically obtains and refreshes OAuth2/JWT tokens when connecting to a LiteLLM Proxy. --- .../docs/providers/litellm_proxy.md | 22 ++ docs/my-website/docs/proxy_auth.md | 333 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + 3 files changed, 356 insertions(+) create mode 100644 docs/my-website/docs/proxy_auth.md diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md index bfefc8a787c..918ac6755a5 100644 --- a/docs/my-website/docs/providers/litellm_proxy.md +++ b/docs/my-website/docs/providers/litellm_proxy.md @@ -227,6 +227,28 @@ response = litellm.completion( ) ``` +## OAuth2/JWT Authentication + +If your LiteLLM Proxy requires OAuth2/JWT authentication (e.g., Azure AD, Keycloak, Okta), the SDK can automatically obtain and refresh tokens for you. + +```python +import litellm +from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=AzureADCredential(), + scope="api://my-litellm-proxy/.default" +) +litellm.api_base = "https://my-proxy.example.com" + +response = litellm.completion( + model="gpt-4", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +[Learn more about SDK Proxy Authentication (OAuth2/JWT Auto-Refresh) →](../proxy_auth) + ## Sending `tags` to LiteLLM Proxy Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter. diff --git a/docs/my-website/docs/proxy_auth.md b/docs/my-website/docs/proxy_auth.md new file mode 100644 index 00000000000..91084b34a37 --- /dev/null +++ b/docs/my-website/docs/proxy_auth.md @@ -0,0 +1,333 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# SDK Proxy Authentication (OAuth2/JWT Auto-Refresh) + +Automatically obtain and refresh OAuth2/JWT tokens when using the LiteLLM Python SDK with a LiteLLM Proxy that requires JWT authentication. + +## Overview + +When your LiteLLM Proxy is protected by an OAuth2/OIDC provider (Azure AD, Keycloak, Okta, Auth0, etc.), your SDK clients need valid JWT tokens for every request. Instead of manually managing token lifecycle, `litellm.proxy_auth` handles this automatically: + +- Obtains tokens from your identity provider +- Caches tokens to avoid unnecessary requests +- Refreshes tokens before they expire (60-second buffer) +- Injects `Authorization: Bearer ` headers into every request + +## Quick Start + +### Azure AD + + + + +Uses the [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential) chain (environment variables, managed identity, Azure CLI, etc.): + +```python +import litellm +from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler + +# One-time setup +litellm.proxy_auth = ProxyAuthHandler( + credential=AzureADCredential(), # uses DefaultAzureCredential + scope="api://my-litellm-proxy/.default" +) +litellm.api_base = "https://my-proxy.example.com" + +# All requests now include Authorization headers automatically +response = litellm.completion( + model="gpt-4", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + + + + +Use a specific Azure AD app registration: + +```python +import litellm +from azure.identity import ClientSecretCredential +from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler + +azure_cred = ClientSecretCredential( + tenant_id="your-tenant-id", + client_id="your-client-id", + client_secret="your-client-secret" +) + +litellm.proxy_auth = ProxyAuthHandler( + credential=AzureADCredential(credential=azure_cred), + scope="api://my-litellm-proxy/.default" +) +litellm.api_base = "https://my-proxy.example.com" + +response = litellm.completion( + model="gpt-4", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + + + + +**Required package:** `pip install azure-identity` + +### Generic OAuth2 (Okta, Auth0, Keycloak, etc.) + +Works with any OAuth2 provider that supports the `client_credentials` grant type: + +```python +import litellm +from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=GenericOAuth2Credential( + client_id="your-client-id", + client_secret="your-client-secret", + token_url="https://your-idp.example.com/oauth2/token" + ), + scope="litellm_proxy_api" +) +litellm.api_base = "https://my-proxy.example.com" + +response = litellm.completion( + model="gpt-4", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +### Custom Credential Provider + +Implement the `TokenCredential` protocol to use any authentication mechanism: + +```python +import time +import litellm +from litellm.proxy_auth import AccessToken, ProxyAuthHandler + +class MyCustomCredential: + """Any class with a get_token(scope) -> AccessToken method works.""" + + def get_token(self, scope: str) -> AccessToken: + # Your custom logic to obtain a token + token = my_auth_system.get_jwt(scope=scope) + return AccessToken( + token=token, + expires_on=int(time.time()) + 3600 + ) + +litellm.proxy_auth = ProxyAuthHandler( + credential=MyCustomCredential(), + scope="my-scope" +) +``` + +## Supported Endpoints + +Auth headers are automatically injected for: + +| Endpoint | Function | +|----------|----------| +| Chat Completions | `litellm.completion()` / `litellm.acompletion()` | +| Embeddings | `litellm.embedding()` / `litellm.aembedding()` | + +## How It Works + +``` +┌──────────┐ ┌──────────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Your │ │ ProxyAuthHandler │ │ Identity │ │ LiteLLM │ +│ Code │────▶│ (token cache) │────▶│ Provider │ │ Proxy │ +│ │ │ │◀────│ (Azure AD, │ │ │ +│ │ │ │ │ Okta, etc) │ │ │ +│ │ └────────┬─────────┘ └──────────────┘ │ │ +│ │ │ Authorization: Bearer │ │ +│ │──────────────┼───────────────────────────────────▶│ │ +│ │◀─────────────┼────────────────────────────────────│ │ +└──────────┘ │ └──────────────┘ +``` + +1. You set `litellm.proxy_auth` once at startup +2. On each SDK call (`completion()`, `embedding()`), the handler checks its cached token +3. If the token is missing or expires within 60 seconds, it requests a new one from your identity provider +4. The `Authorization: Bearer ` header is injected into the request +5. If token retrieval fails, a warning is logged and the request proceeds without auth headers + +## API Reference + +### ProxyAuthHandler + +The main handler that manages the token lifecycle. + +```python +from litellm.proxy_auth import ProxyAuthHandler + +handler = ProxyAuthHandler( + credential=, # required - credential provider + scope="" # required - OAuth2 scope to request +) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `credential` | `TokenCredential` | Yes | A credential provider (AzureADCredential, GenericOAuth2Credential, or custom) | +| `scope` | `str` | Yes | The OAuth2 scope to request tokens for | + +**Methods:** + +| Method | Returns | Description | +|--------|---------|-------------| +| `get_token()` | `AccessToken` | Get a valid token, refreshing if needed | +| `get_auth_headers()` | `dict` | Get `{"Authorization": "Bearer "}` headers | + +### AzureADCredential + +Wraps any `azure-identity` credential with lazy initialization. + +```python +from litellm.proxy_auth import AzureADCredential + +# Uses DefaultAzureCredential (recommended) +cred = AzureADCredential() + +# Or wrap a specific azure-identity credential +from azure.identity import ManagedIdentityCredential +cred = AzureADCredential(credential=ManagedIdentityCredential()) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `credential` | Azure `TokenCredential` | No | An azure-identity credential. If `None`, uses `DefaultAzureCredential` | + +### GenericOAuth2Credential + +Standard OAuth2 client credentials flow for any provider. + +```python +from litellm.proxy_auth import GenericOAuth2Credential + +cred = GenericOAuth2Credential( + client_id="your-client-id", + client_secret="your-client-secret", + token_url="https://your-idp.com/oauth2/token" +) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `client_id` | `str` | Yes | OAuth2 client ID | +| `client_secret` | `str` | Yes | OAuth2 client secret | +| `token_url` | `str` | Yes | Token endpoint URL | + +### AccessToken + +Dataclass representing an OAuth2 access token. + +```python +from litellm.proxy_auth import AccessToken + +token = AccessToken( + token="eyJhbG...", # JWT string + expires_on=1234567890 # Unix timestamp +) +``` + +### TokenCredential Protocol + +Any class implementing this protocol can be used as a credential provider: + +```python +from litellm.proxy_auth import AccessToken + +class MyCredential: + def get_token(self, scope: str) -> AccessToken: + ... +``` + +## Provider-Specific Examples + +### Keycloak + +```python +from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=GenericOAuth2Credential( + client_id="litellm-client", + client_secret="your-keycloak-client-secret", + token_url="https://keycloak.example.com/realms/your-realm/protocol/openid-connect/token" + ), + scope="openid" +) +``` + +### Okta + +```python +from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=GenericOAuth2Credential( + client_id="your-okta-client-id", + client_secret="your-okta-client-secret", + token_url="https://your-org.okta.com/oauth2/default/v1/token" + ), + scope="litellm_api" +) +``` + +### Auth0 + +```python +from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=GenericOAuth2Credential( + client_id="your-auth0-client-id", + client_secret="your-auth0-client-secret", + token_url="https://your-tenant.auth0.com/oauth/token" + ), + scope="https://my-proxy.example.com/api" +) +``` + +### Azure AD with Managed Identity + +```python +from azure.identity import ManagedIdentityCredential +from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler + +litellm.proxy_auth = ProxyAuthHandler( + credential=AzureADCredential( + credential=ManagedIdentityCredential() + ), + scope="api://my-litellm-proxy/.default" +) +``` + +## Combining with `use_litellm_proxy` + +You can use `proxy_auth` together with [`use_litellm_proxy`](./providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy) to route all SDK requests through an authenticated proxy: + +```python +import os +import litellm +from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler + +# Route all requests through the proxy +os.environ["LITELLM_PROXY_API_BASE"] = "https://my-proxy.example.com" +litellm.use_litellm_proxy = True + +# Authenticate with OAuth2/JWT +litellm.proxy_auth = ProxyAuthHandler( + credential=AzureADCredential(), + scope="api://my-litellm-proxy/.default" +) + +# This request goes through the proxy with automatic JWT auth +response = litellm.completion( + model="vertex_ai/gemini-2.0-flash-001", + messages=[{"role": "user", "content": "Hello!"}] +) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9d9007a916c..343860cb158 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -224,6 +224,7 @@ const sidebars = { label: "Configuration", items: [ "set_keys", + "proxy_auth", "caching/all_caches", ], }, From 8dcd18301366b69d3cc808acfe011922e8d5e24b Mon Sep 17 00:00:00 2001 From: John Lathouwers Date: Sun, 8 Feb 2026 06:58:59 +0000 Subject: [PATCH 019/117] Fixes #20582 (#20663) --- litellm/llms/oci/chat/transformation.py | 37 +++++- litellm/types/llms/oci.py | 58 +++++----- .../oci/chat/test_oci_chat_transformation.py | 108 ++++++++++++++++++ .../oci/chat/test_oci_cohere_tool_calls.py | 104 +++++++++++++++++ 4 files changed, 280 insertions(+), 27 deletions(-) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 84f39ef2525..e66394ae5f5 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -218,6 +218,7 @@ class OCIChatConfig(BaseConfig): "parallel_tool_calls": False, "audio": False, "web_search_options": False, + "response_format": "responseFormat", } # Cohere and Gemini use the same parameter mapping as GENERIC @@ -269,6 +270,9 @@ class OCIChatConfig(BaseConfig): adapted_params[alias] = value + if alias == "responseFormat": + adapted_params["response_format"] = value + return adapted_params def _sign_with_oci_signer( @@ -673,6 +677,36 @@ class OCIChatConfig(BaseConfig): selected_params["tools"] = adapt_tool_definition_to_oci_standard( # type: ignore[assignment] selected_params["tools"], vendor # type: ignore[arg-type] ) + + # Transform response_format type to OCI uppercase format + if "responseFormat" in selected_params: + rf = selected_params["responseFormat"] + if isinstance(rf, dict) and "type" in rf: + rf_payload = dict(rf) + selected_params["responseFormat"] = rf_payload + + response_type = rf_payload["type"] + schema_payload: Optional[Any] = None + + if "json_schema" in rf_payload: + raw_schema_payload = rf_payload.pop("json_schema") + if isinstance(raw_schema_payload, dict): + schema_payload = dict(raw_schema_payload) + else: + schema_payload = raw_schema_payload + + if schema_payload is not None: + rf_payload["jsonSchema"] = schema_payload + + if vendor == OCIVendors.COHERE: + # Cohere expects lower-case type values + rf_payload["type"] = response_type + else: + format_type = response_type.upper() + if format_type == "JSON": + format_type = "JSON_OBJECT" + rf_payload["type"] = format_type + return selected_params def adapt_messages_to_cohere_standard(self, messages: List[AllMessageValues]) -> List[CohereMessage]: @@ -806,11 +840,12 @@ class OCIChatConfig(BaseConfig): # Create Cohere-specific chat request + optional_cohere_params = self._get_optional_params(OCIVendors.COHERE, optional_params) chat_request = CohereChatRequest( apiFormat="COHERE", message=self._extract_text_content(user_messages[-1]["content"]), chatHistory=self.adapt_messages_to_cohere_standard(messages), - **self._get_optional_params(OCIVendors.COHERE, optional_params) + **optional_cohere_params ) data = OCICompletionPayload( diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py index 9a654bc0f6c..cb1dd391434 100644 --- a/litellm/types/llms/oci.py +++ b/litellm/types/llms/oci.py @@ -102,6 +102,7 @@ class OCIChatRequestPayload(BaseModel): seed: Optional[int] = None frequencyPenalty: Optional[float] = None presencePenalty: Optional[float] = None + responseFormat: Optional[Dict[str, Any]] = None class OCIServingMode(BaseModel): @@ -125,14 +126,14 @@ class OCICompletionPayload(BaseModel): class OCICompletionTokenDetails(BaseModel): """Completion token details in the OCI response.""" - acceptedPredictionTokens: int - reasoningTokens: int + acceptedPredictionTokens: Optional[int] = None + reasoningTokens: Optional[int] = None class OCIPromptTokensDetails(BaseModel): """Prompt token details in the OCI response.""" - cachedTokens: int + cachedTokens: Optional[int] = None class OCIResponseUsage(BaseModel): @@ -205,40 +206,40 @@ class CohereStreamChunk(BaseModel): class CohereMessage(BaseModel): """Base model for Cohere messages.""" - + role: str - message: str + message: Optional[str] = None toolCalls: Optional[List[CohereToolCall]] = None class CohereUserMessage(CohereMessage): """User message in Cohere chat.""" - + role: Literal["USER"] = "USER" class CohereChatBotMessage(CohereMessage): """Chatbot message in Cohere chat.""" - + role: Literal["CHATBOT"] = "CHATBOT" class CohereSystemMessage(CohereMessage): """System message in Cohere chat.""" - + role: Literal["SYSTEM"] = "SYSTEM" class CohereToolMessage(CohereMessage): """Tool message in Cohere chat.""" - + role: Literal["TOOL"] = "TOOL" toolCallId: str class CohereParameterDefinition(BaseModel): """Parameter definition for Cohere tools.""" - + description: str type: str isRequired: bool = False @@ -246,7 +247,7 @@ class CohereParameterDefinition(BaseModel): class CohereTool(BaseModel): """Tool definition for Cohere.""" - + name: str description: str parameterDefinitions: Dict[str, CohereParameterDefinition] @@ -254,38 +255,44 @@ class CohereTool(BaseModel): class CohereToolCall(BaseModel): """Tool call made by Cohere model.""" - + name: str parameters: Dict[str, Any] class CohereToolResult(BaseModel): """Result of a tool call.""" - + callId: str result: str class CohereResponseFormat(BaseModel): """Response format for Cohere.""" - + type: str class CohereResponseTextFormat(CohereResponseFormat): """Text response format for Cohere.""" - + type: Literal["text"] = "text" +class CohereResponseJSONSchemaFormat(CohereResponseFormat): + """JSON schema response format for Cohere.""" + + type: Literal["json_schema"] = "json_schema" + jsonSchema: Dict[str, Any] + class CohereChatRequest(BaseModel): """Cohere chat request model.""" - + # Required fields message: str apiFormat: Literal["COHERE"] = "COHERE" - + # Optional fields chatHistory: Optional[List[CohereMessage]] = None maxTokens: Optional[int] = None @@ -298,7 +305,7 @@ class CohereChatRequest(BaseModel): seed: Optional[int] = None tools: Optional[List[CohereTool]] = None toolChoice: Optional[Union[str, Dict[str, Any]]] = None - responseFormat: Optional[CohereResponseFormat] = None + responseFormat: Optional[Union[CohereResponseTextFormat, CohereResponseJSONSchemaFormat, CohereResponseFormat]] = None preambleOverride: Optional[str] = None documents: Optional[List[Dict[str, Any]]] = None searchQueriesOnly: Optional[bool] = None @@ -318,7 +325,7 @@ class CohereChatRequest(BaseModel): class CohereUsage(BaseModel): """Usage information for Cohere response.""" - + promptTokens: int completionTokens: int totalTokens: int @@ -328,7 +335,7 @@ class CohereUsage(BaseModel): class CohereCitation(BaseModel): """Citation in Cohere response.""" - + start: int end: int text: str @@ -337,19 +344,19 @@ class CohereCitation(BaseModel): class CohereSearchQuery(BaseModel): """Search query generated by Cohere.""" - + text: str generation_id: str class CohereChatResponse(BaseModel): """Cohere chat response model.""" - + # Required fields text: str apiFormat: Literal["COHERE"] = "COHERE" finishReason: Literal["COMPLETE", "ERROR_TOXIC", "ERROR_LIMIT", "ERROR", "USER_CANCEL", "MAX_TOKENS"] - + # Optional fields chatHistory: Optional[List[CohereMessage]] = None citations: Optional[List[CohereCitation]] = None @@ -364,7 +371,7 @@ class CohereChatResponse(BaseModel): class CohereChatDetails(BaseModel): """Chat details for Cohere request.""" - + compartmentId: str servingMode: OCIServingMode chatRequest: CohereChatRequest @@ -372,8 +379,7 @@ class CohereChatDetails(BaseModel): class CohereChatResult(BaseModel): """Complete Cohere chat result.""" - + modelId: str modelVersion: str chatResponse: CohereChatResponse - diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 3bd46b84e6c..3b53f9de714 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -287,6 +287,114 @@ class TestOCIChatConfig: # Verify the message content assert transformed_request["chatRequest"]["message"] == "What is quantum computing?" + def test_transform_request_response_format_json_object(self): + """ + Tests that response_format type 'json_object' is uppercased to 'JSON_OBJECT' for generic OCI models. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json_object"}, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_OBJECT" + + def test_transform_request_response_format_text(self): + """ + Tests that response_format type 'text' is uppercased to 'TEXT' for generic OCI models. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "text"}, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "TEXT" + + def test_transform_request_response_format_json_shorthand(self): + """ + Tests that response_format type 'json' is mapped to 'JSON_OBJECT' for generic OCI models. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json"}, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_OBJECT" + + def test_transform_response_without_token_details(self): + """ + Tests that responses missing completionTokensDetails and promptTokensDetails + are handled correctly (fields are optional). + """ + config = OCIChatConfig() + created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z") + mock_oci_response = { + "modelId": TEST_MODEL_NAME, + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "Hello!"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 10, + "totalTokens": 15, + }, + }, + } + response = httpx.Response( + status_code=200, json=mock_oci_response, headers={"Content-Type": "application/json"} + ) + result = config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hello!" + assert result.usage.prompt_tokens == 5 # type: ignore + assert result.usage.completion_tokens == 10 # type: ignore + assert result.usage.total_tokens == 15 # type: ignore + def test_transform_response_simple_text(self): """ Tests if a simple text response is transformed correctly. diff --git a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py index abbb7e3e301..a9c4bead820 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py @@ -239,6 +239,110 @@ class TestOCICohereToolCalls: assert result.usage.completion_tokens == 22 assert result.usage.total_tokens == 48 + def test_cohere_request_preserves_json_schema_response_format(self): + """Ensure Cohere requests retain JSON schema payloads in responseFormat.""" + config = OCIChatConfig() + messages = [{"role": "user", "content": "Return structured info"}] + response_format = { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "strict": True, + "schema": { + "type": "object", + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + } + } + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": response_format, + } + + transformed_request = config.transform_request( + model="cohere.command-rplus", + messages=messages, # type: ignore[arg-type] + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + chat_request = transformed_request["chatRequest"] + assert chat_request["apiFormat"] == "COHERE" + assert "responseFormat" in chat_request + + cohere_response_format = chat_request["responseFormat"] + assert cohere_response_format["type"] == "json_schema" + assert "json_schema" not in cohere_response_format + assert "jsonSchema" in cohere_response_format + assert cohere_response_format["jsonSchema"] == response_format["json_schema"] + + def test_cohere_request_response_format_text_stays_lowercase(self): + """Ensure Cohere keeps response_format type lowercase (e.g. 'text' not 'TEXT').""" + config = OCIChatConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "text"}, + } + + transformed_request = config.transform_request( + model="cohere.command-latest", + messages=messages, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + chat_request = transformed_request["chatRequest"] + assert chat_request["apiFormat"] == "COHERE" + assert "responseFormat" in chat_request + assert chat_request["responseFormat"]["type"] == "text" + + def test_cohere_tool_call_only_message_no_text(self): + """Test chat history with an assistant message that has tool calls but no text content.""" + config = OCIChatConfig() + + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "content": "Sunny, 25C", + "tool_call_id": "call_1", + }, + ] + + chat_history = config.adapt_messages_to_cohere_standard(messages) + + # First message is the user message + assert chat_history[0].role == "USER" + assert chat_history[0].message == "What's the weather?" + + # Second message is the assistant with tool calls and no text + assistant_msg = chat_history[1] + assert assistant_msg.role == "CHATBOT" + assert assistant_msg.message is None or assistant_msg.message == "" + assert assistant_msg.toolCalls is not None + assert len(assistant_msg.toolCalls) == 1 + assert assistant_msg.toolCalls[0].name == "get_weather" + def test_cohere_chat_history_with_tool_calls(self): """Test chat history transformation with tool calls""" config = OCIChatConfig() From 7335965c12848f5b9cd07c865dc770c29c8281e0 Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Sat, 7 Feb 2026 22:59:49 -0800 Subject: [PATCH 020/117] fix: show error details instead of Data Not Available for failed requests (#20656) --- .../LogDetailsDrawer/LogDetailsDrawer.tsx | 9 ++- .../view_logs/RequestResponsePanel.test.tsx | 74 +++++++++++++++++++ .../view_logs/RequestResponsePanel.tsx | 4 +- .../src/components/view_logs/index.tsx | 2 +- 4 files changed, 83 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index 54946eb0964..a3f948296eb 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -85,7 +85,7 @@ export function LogDetailsDrawer({ // Check if request/response data is present const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); - const missingData = !hasMessages && !hasResponse; + const missingData = !hasMessages && !hasResponse && !hasError; // Guardrail data const guardrailInfo = metadata?.guardrail_information; @@ -206,6 +206,7 @@ export function LogDetailsDrawer({ {/* Request/Response JSON - Collapsible */} any; getFormattedResponse: () => any; logEntry: LogEntry; @@ -346,6 +348,7 @@ interface RequestResponseSectionProps { function RequestResponseSection({ hasResponse, + hasError, getRawRequest, getFormattedResponse, logEntry, @@ -423,7 +426,7 @@ function RequestResponseSection({ text: getCopyText(), tooltips: ["Copy JSON", "Copied!"] }} - disabled={activeTab === TAB_RESPONSE && !hasResponse} + disabled={activeTab === TAB_RESPONSE && !hasResponse && !hasError} /> } items={[ @@ -441,7 +444,7 @@ function RequestResponseSection({ label: "Response", children: (
- {hasResponse ? ( + {hasResponse || hasError ? ( ) : (
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.test.tsx index deeac3a8d0b..b7c0318d9fd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.test.tsx @@ -188,4 +188,78 @@ describe("RequestResponsePanel", () => { expect(responseData).toEqual({ responseData: "this should appear in response" }); expect(responseData).not.toEqual({ requestData: "this should not appear in response" }); }); + + it("should show error response data when hasError is true and hasResponse is false", () => { + const failedLogEntry: LogEntry = { + ...baseLogEntry, + messages: [], + response: {}, + metadata: { + status: "failure", + error_information: { + error_message: "Model not found", + error_class: "NotFoundError", + error_code: 404, + }, + additional_usage_values: { + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }; + const errorResponse = { error: { message: "Model not found", type: "NotFoundError", code: 404, param: null } }; + const mockGetRawRequest = vi.fn().mockReturnValue({ messages: [] }); + const mockFormattedResponse = vi.fn().mockReturnValue(errorResponse); + render( + , + ); + expect(screen.queryByText("Response data not available")).not.toBeInTheDocument(); + expect(mockFormattedResponse).toHaveBeenCalled(); + const copyButtons = screen.getAllByRole("button"); + const copyResponseButton = copyButtons.find((button) => button.getAttribute("title") === "Copy response"); + expect(copyResponseButton).not.toBeDisabled(); + }); + + it("should show Response data not available when hasResponse and hasError are both false", () => { + const mockGetRawRequest = vi.fn().mockReturnValue({ messages: [] }); + const mockFormattedResponse = vi.fn().mockReturnValue({}); + render( + , + ); + expect(screen.getByText("Response data not available")).toBeInTheDocument(); + }); + + it("should show error code in response header when hasError is true", () => { + const errorInfo = { error_message: "Rate limit exceeded", error_class: "RateLimitError", error_code: 429 }; + const mockGetRawRequest = vi.fn().mockReturnValue({ messages: [] }); + const mockFormattedResponse = vi.fn().mockReturnValue({ error: { message: "Rate limit exceeded", type: "RateLimitError", code: 429, param: null } }); + render( + , + ); + expect(screen.getByText(/HTTP code 429/)).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx index b2cae68184a..da9323f8172 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx @@ -113,7 +113,7 @@ export function RequestResponsePanel({ onClick={handleCopyResponse} className="p-1 hover:bg-gray-200 rounded" title="Copy response" - disabled={!hasResponse} + disabled={!hasResponse && !hasError} >
- {hasResponse ? ( + {hasResponse || hasError ? (
diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 3859a5e51fb..87e11e00c7c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -822,7 +822,7 @@ export function RequestViewer({ row, onOpenSettings }: { row: Row; onO ? row.original.messages.length > 0 : Object.keys(row.original.messages).length > 0); const hasResponse = row.original.response && Object.keys(formatData(row.original.response)).length > 0; - const missingData = !hasMessages && !hasResponse; + const missingData = !hasMessages && !hasResponse && !hasError; // Format the response with error details if present const formattedResponse = () => { From c8d95470957492ff1dd0f6b5a6cf19bb7694d1df Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Sat, 7 Feb 2026 23:00:33 -0800 Subject: [PATCH 021/117] fix(ui): add null guard for models in API keys table (#20655) The VirtualKeysTable crashed when rendering keys with null or undefined models field. The className expression tried to access .length on null, throwing a TypeError that broke the entire keys table. Added Array.isArray() guard before accessing .length on the models value. Fixes #20611 --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 83 +++++++++++++++++++ .../VirtualKeysPage/VirtualKeysTable.tsx | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index ef5bb2a0371..749396c82f3 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -542,3 +542,86 @@ it("should display 'Default Proxy Admin' for created_by when value is 'default_u expect(defaultProxyAdminElements.length).toBeGreaterThan(0); }); }); + + +it("should render table without crashing when models is null", async () => { + const keyWithNullModels = { + ...mockKey, + models: null as unknown as string[], + }; + + mockUseFilterLogic.mockReturnValue({ + filters: { + "Team ID": "", + "Organization ID": "", + "Key Alias": "", + "User ID": "", + "Sort By": "created_at", + "Sort Order": "desc", + }, + filteredKeys: [keyWithNullModels], + allKeyAliases: ["test-key-alias"], + allTeams: [mockTeam], + allOrganizations: [mockOrganization], + handleFilterChange: vi.fn(), + handleFilterReset: vi.fn(), + }); + + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + // This should not throw an error + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + }); +}); + +it("should render table without crashing when models is undefined", async () => { + const keyWithUndefinedModels = { + ...mockKey, + models: undefined as unknown as string[], + }; + + mockUseFilterLogic.mockReturnValue({ + filters: { + "Team ID": "", + "Organization ID": "", + "Key Alias": "", + "User ID": "", + "Sort By": "created_at", + "Sort Order": "desc", + }, + filteredKeys: [keyWithUndefinedModels], + allKeyAliases: ["test-key-alias"], + allTeams: [mockTeam], + allOrganizations: [mockOrganization], + handleFilterChange: vi.fn(), + handleFilterReset: vi.fn(), + }); + + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + // This should not throw an error + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index 465b9b8fbe0..f7c47943e7e 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -727,7 +727,7 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo whiteSpace: "pre-wrap", overflow: "hidden", }} - className={`py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap ${cell.column.id === "models" && (cell.getValue() as string[]).length > 3 ? "px-0" : ""}`} + className={`py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap ${cell.column.id === "models" && Array.isArray(cell.getValue()) && (cell.getValue() as string[]).length > 3 ? "px-0" : ""}`} > {flexRender(cell.column.columnDef.cell, cell.getContext())} From c9c6a5edc971c55cb5cc26cc5911faba6ee816aa Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Sat, 7 Feb 2026 23:02:29 -0800 Subject: [PATCH 022/117] Fix: Spend logs pickle error with Pydantic models and redaction (#20685) * docs: add callback registration optimization to v1.81.9 release notes (#20681) * docs: add callback registration optimization to v1.81.9 release notes * Update v1.81.9.md --------- Co-authored-by: Alexsander Hamir * Fix spend logs pickle error with Pydantic models Replace copy.deepcopy() with Pydantic-safe serialization to avoid "cannot pickle '_thread.RLock' object" errors when request/response redaction is enabled. Changes: - Add _convert_to_json_serializable_dict() helper that uses model_dump() for Pydantic models instead of pickle - Replace copy.deepcopy() calls in request and response redaction paths with the new helper function - Recursively handles nested dicts, lists, and Pydantic models Root cause: Pydantic v2 BaseModel instances contain internal _thread.RLock objects for thread-safety. When copy.deepcopy() attempts to pickle these objects, it fails because threading primitives cannot be pickled. Fixes #20647 * chore: remove unused copy import Remove unused copy import that was causing lint failure. The copy.deepcopy() calls were replaced with _convert_to_json_serializable_dict() helper function in the previous commit, making the copy module no longer needed. --------- Co-authored-by: ryan-crabbe <128659760+ryan-crabbe@users.noreply.github.com> Co-authored-by: Alexsander Hamir --- docs/my-website/release_notes/v1.81.9.md | 7 ++++ .../spend_tracking/spend_tracking_utils.py | 39 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9.md index 08b70e029e2..c34d3056cae 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9.md @@ -48,6 +48,13 @@ pip install litellm==1.81.9 - **UI Team Soft Budget Alerts** - [Set soft budgets on teams and receive email alerts when spending crosses the threshold — without blocking requests](../../docs/proxy/ui_team_soft_budget_alerts) - **Performance Optimizations** - Multiple performance improvements including ~40% Prometheus CPU reduction, LRU caching, and optimized logging paths - **LiteLLM Observatory** - [Automated 24-hour load tests](../../blog/litellm-observatory) +- **30% Faster Request Processing for Callback-Heavy Deployments** - [Performance improvement for callback heavy deployments][PR #20354](https://github.com/BerriAI/litellm/pull/20354) + +--- + +## 30% Faster Request Processing for Callback-Heavy Deployments + + If you use logging callbacks like Langfuse, Datadog, or Prometheus, every request was paying an unnecessary cost: three loops that re-sorted your callbacks on every single request, even though the callback list hadn't changed. The more callbacks you had configured, the more time was wasted. We moved this work to happen once at startup instead of on every request. For deployments with the default callback set, this is a ~30% speedup in request setup. For deployments with many callbacks configured, the improvement is even larger. --- diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index bd148ecb481..cb8b9ec0395 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,4 +1,3 @@ -import copy import hashlib import json import secrets @@ -642,6 +641,34 @@ def _sanitize_request_body_for_spend_logs_payload( return {k: _sanitize_value(v) for k, v in request_body.items()} +def _convert_to_json_serializable_dict(obj: Any) -> Any: + """ + Convert object to JSON-serializable dict, handling Pydantic models safely. + + This avoids pickle-based deepcopy which fails on Pydantic v2 models + containing _thread.RLock objects. + + Args: + obj: Object to convert (dict, list, Pydantic model, or primitive) + + Returns: + JSON-serializable version of the object + """ + if isinstance(obj, BaseModel): + # Use Pydantic's model_dump() instead of pickle + return obj.model_dump() + elif isinstance(obj, dict): + return {k: _convert_to_json_serializable_dict(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_convert_to_json_serializable_dict(item) for item in obj] + elif hasattr(obj, "__dict__"): + # Handle objects with __dict__ attribute + return _convert_to_json_serializable_dict(obj.__dict__) + else: + # Primitives (str, int, float, bool, None) pass through + return obj + + def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, litellm_params: dict, @@ -649,7 +676,7 @@ def _get_proxy_server_request_for_spend_logs_payload( ) -> str: """ Only store if _should_store_prompts_and_responses_in_spend_logs() is True - + If turn_off_message_logging is enabled, redact messages in the request body. """ if _should_store_prompts_and_responses_in_spend_logs(): @@ -674,9 +701,9 @@ def _get_proxy_server_request_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy request body before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - _request_body = copy.deepcopy(_request_body) + _request_body = _convert_to_json_serializable_dict(_request_body) perform_redaction(model_call_details=_request_body, result=None) _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) @@ -736,9 +763,9 @@ def _get_response_for_spend_logs_payload( ), } - # If redaction is enabled, deep copy response before redacting + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - response_obj = copy.deepcopy(response_obj) + response_obj = _convert_to_json_serializable_dict(response_obj) response_obj = perform_redaction(model_call_details={}, result=response_obj) sanitized_wrapper = _sanitize_request_body_for_spend_logs_payload( From 0458e734b2642aaf25fb381da423bdf0e71363e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elias=20H=C3=B6gbom=20Aronsson?= Date: Sun, 8 Feb 2026 08:05:17 +0100 Subject: [PATCH 023/117] fix(vertex_ai): propagate extra_headers anthropic-beta to request body (#20666) Vertex AI requires Anthropic beta flags in the request body (anthropic_beta array), not as HTTP headers. The Bedrock handler already extracts user-specified beta headers from the headers dict, but the Vertex handler was missing this, causing extra_headers like interleaved-thinking-2025-05-14 to be silently dropped. This extracts anthropic-beta values from optional_params extra_headers and merges them into the anthropic_beta request body field, and also removes extra_headers from the request body since the parent's transform_request spreads optional_params into data. --- .../anthropic/transformation.py | 46 ++- ...partner_models_anthropic_transformation.py | 341 ++++++++++++------ 2 files changed, 256 insertions(+), 131 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 0b728d88e76..6a5b934661a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -56,34 +56,36 @@ class VertexAIAnthropicConfig(AnthropicConfig): ) -> None: """ Add context_management beta headers to the beta_set. - + - If any edit has type "compact_20260112", add compact-2026-01-12 header - For all other edits, add context-management-2025-06-27 header - + Args: beta_set: Set of beta headers to modify in-place context_management: The context_management dict from optional_params """ from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - + edits = context_management.get("edits", []) has_compact = False has_other = False - + for edit in edits: edit_type = edit.get("type", "") if edit_type == "compact_20260112": has_compact = True else: has_other = True - + # Add compact header if any compact edits exist if has_compact: beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - + # Add context management header if any other edits exist if has_other: - beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_set.add( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + ) def transform_request( self, @@ -102,10 +104,10 @@ class VertexAIAnthropicConfig(AnthropicConfig): ) data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter - + # VertexAI doesn't support output_format parameter, remove it if present data.pop("output_format", None) - + tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) auto_betas = self.get_anthropic_beta_list( @@ -119,16 +121,30 @@ class VertexAIAnthropicConfig(AnthropicConfig): beta_set = set(auto_betas) if tool_search_used: - beta_set.add("tool-search-tool-2025-10-19") # Vertex requires this header for tool search - + beta_set.add( + "tool-search-tool-2025-10-19" + ) # Vertex requires this header for tool search + # Add context_management beta headers (compact and/or context-management) context_management = optional_params.get("context_management") if context_management: self._add_context_management_beta_headers(beta_set, context_management) + extra_headers = optional_params.get("extra_headers") or {} + anthropic_beta_value = extra_headers.get("anthropic-beta", "") + if isinstance(anthropic_beta_value, str) and anthropic_beta_value: + for beta in anthropic_beta_value.split(","): + beta = beta.strip() + if beta: + beta_set.add(beta) + elif isinstance(anthropic_beta_value, list): + beta_set.update(anthropic_beta_value) + + data.pop("extra_headers", None) + if beta_set: data["anthropic_beta"] = list(beta_set) - + return data def map_openai_params( @@ -148,7 +164,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): original_model = model if "response_format" in non_default_params: model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach - + # Call parent method with potentially modified model name optional_params = super().map_openai_params( non_default_params=non_default_params, @@ -156,10 +172,10 @@ class VertexAIAnthropicConfig(AnthropicConfig): model=model, drop_params=drop_params, ) - + # Restore original model name for any other processing model = original_model - + return optional_params def transform_response( diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 90ab41aadf6..4bcafd4c57e 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -45,68 +45,65 @@ def test_vertex_ai_anthropic_web_search_header_in_completion(): # Create the config instance model_info = AnthropicModelInfo() - + # Test the header generation directly tools = [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] - + # Check if web search tool is detected web_search_detected = model_info.is_web_search_tool_used(tools=tools) assert web_search_detected is True, "Web search tool should be detected" - + # Generate headers with is_vertex_request=True headers = model_info.get_anthropic_headers( api_key="test-key", web_search_tool_used=web_search_detected, is_vertex_request=True, ) - + # Assert that the anthropic-beta header with web-search is present assert "anthropic-beta" in headers, "anthropic-beta header should be present" - assert headers["anthropic-beta"] == "web-search-2025-03-05", \ - f"anthropic-beta should be 'web-search-2025-03-05', got: {headers['anthropic-beta']}" - + assert ( + headers["anthropic-beta"] == "web-search-2025-03-05" + ), f"anthropic-beta should be 'web-search-2025-03-05', got: {headers['anthropic-beta']}" + # Test that header is NOT added for non-Vertex requests headers_non_vertex = model_info.get_anthropic_headers( api_key="test-key", web_search_tool_used=web_search_detected, is_vertex_request=False, ) - + # For non-Vertex (Anthropic-hosted), the web search header should NOT be in anthropic-beta # because Anthropic doesn't require it - assert "anthropic-beta" not in headers_non_vertex or "web-search" not in headers_non_vertex.get("anthropic-beta", ""), \ - "anthropic-beta with web-search should not be present for non-Vertex requests" + assert ( + "anthropic-beta" not in headers_non_vertex + or "web-search" not in headers_non_vertex.get("anthropic-beta", "") + ), "anthropic-beta with web-search should not be present for non-Vertex requests" def test_vertex_ai_anthropic_context_management_compact_beta_header(): """Test that context_management with compact adds the correct beta header for Vertex AI""" config = VertexAIAnthropicConfig() - + messages = [{"role": "user", "content": "Hello"}] optional_params = { - "context_management": { - "edits": [ - { - "type": "compact_20260112" - } - ] - }, + "context_management": {"edits": [{"type": "compact_20260112"}]}, "max_tokens": 100, - "is_vertex_request": True + "is_vertex_request": True, } - + result = config.transform_request( model="claude-opus-4-6", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify context_management is included assert "context_management" in result assert result["context_management"]["edits"][0]["type"] == "compact_20260112" - + # Verify compact beta header is in anthropic_beta field assert "anthropic_beta" in result assert "compact-2026-01-12" in result["anthropic_beta"] @@ -115,33 +112,27 @@ def test_vertex_ai_anthropic_context_management_compact_beta_header(): def test_vertex_ai_anthropic_context_management_mixed_edits(): """Test that context_management with both compact and other edits adds both beta headers""" config = VertexAIAnthropicConfig() - + messages = [{"role": "user", "content": "Hello"}] optional_params = { "context_management": { "edits": [ - { - "type": "compact_20260112" - }, - { - "type": "replace", - "message_id": "msg_123", - "content": "new content" - } + {"type": "compact_20260112"}, + {"type": "replace", "message_id": "msg_123", "content": "new content"}, ] }, "max_tokens": 100, - "is_vertex_request": True + "is_vertex_request": True, } - + result = config.transform_request( model="claude-opus-4-6", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify both beta headers are present assert "anthropic_beta" in result assert "compact-2026-01-12" in result["anthropic_beta"] @@ -151,58 +142,65 @@ def test_vertex_ai_anthropic_context_management_mixed_edits(): def test_vertex_ai_anthropic_structured_output_header_not_added(): """Test that structured output beta headers are NOT added for Vertex AI requests""" from litellm.llms.anthropic.chat.transformation import AnthropicConfig - + config = AnthropicConfig() - + # Test case 1: Vertex request with output_format should NOT add beta header headers_vertex = {} optional_params_vertex = { - 'output_format': { - 'type': 'json_schema', - 'json_schema': { - 'name': 'MathResult', - 'schema': {'properties': {'result': {'type': 'integer'}}} - } + "output_format": { + "type": "json_schema", + "json_schema": { + "name": "MathResult", + "schema": {"properties": {"result": {"type": "integer"}}}, + }, }, - 'is_vertex_request': True + "is_vertex_request": True, } - result_vertex = config.update_headers_with_optional_anthropic_beta(headers_vertex, optional_params_vertex) - - assert "anthropic-beta" not in result_vertex, \ - f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" - + result_vertex = config.update_headers_with_optional_anthropic_beta( + headers_vertex, optional_params_vertex + ) + + assert ( + "anthropic-beta" not in result_vertex + ), f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + # Test case 2: Non-Vertex request with output_format SHOULD add beta header headers_non_vertex = {} optional_params_non_vertex = { - 'output_format': { - 'type': 'json_schema', - 'json_schema': { - 'name': 'MathResult', - 'schema': {'properties': {'result': {'type': 'integer'}}} - } + "output_format": { + "type": "json_schema", + "json_schema": { + "name": "MathResult", + "schema": {"properties": {"result": {"type": "integer"}}}, + }, }, - 'is_vertex_request': False + "is_vertex_request": False, } - result_non_vertex = config.update_headers_with_optional_anthropic_beta(headers_non_vertex, optional_params_non_vertex) - - assert "anthropic-beta" in result_non_vertex, \ - "Non-Vertex request SHOULD have anthropic-beta header for structured output" - assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", \ - f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" + result_non_vertex = config.update_headers_with_optional_anthropic_beta( + headers_non_vertex, optional_params_non_vertex + ) + + assert ( + "anthropic-beta" in result_non_vertex + ), "Non-Vertex request SHOULD have anthropic-beta header for structured output" + assert ( + result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13" + ), f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): """ - Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based + Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based structured outputs instead of output_format parameter. - + This test verifies that: 1. Claude Sonnet 4.5 uses tool-based structured outputs on VertexAI 2. output_format parameter is removed from the final request 3. The fix prevents "Extra inputs are not permitted" error """ config = VertexAIAnthropicConfig() - + # Test data matching the issue report response_format = { "type": "json_schema", @@ -212,29 +210,23 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): "schema": { "type": "object", "properties": { - "question": { - "type": "string" - }, - "response": { - "type": "string" - } + "question": {"type": "string"}, + "response": {"type": "string"}, }, "required": ["question", "response"], - "additionalProperties": False - } - } + "additionalProperties": False, + }, + }, } - - messages = [ - {"role": "user", "content": "Generate a question and answer about AI."} - ] - + + messages = [{"role": "user", "content": "Generate a question and answer about AI."}] + # Test parameters that would trigger the issue non_default_params = { "response_format": response_format, "max_tokens": 1000, } - + # Test 1: Verify map_openai_params forces tool-based approach for Claude Sonnet 4.5 optional_params = {} result_params = config.map_openai_params( @@ -243,17 +235,19 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): model="claude-3-5-sonnet-20241022", # Claude Sonnet 4.5 model drop_params=False, ) - + # Should have tools and tool_choice (tool-based approach) assert "tools" in result_params, "Tools should be present for structured output" - assert "tool_choice" in result_params, "Tool choice should be present for structured output" + assert ( + "tool_choice" in result_params + ), "Tool choice should be present for structured output" assert "json_mode" in result_params, "JSON mode should be enabled" - + # Verify the tool is the response format tool tools = result_params["tools"] assert len(tools) == 1, "Should have exactly one tool for response format" assert tools[0]["name"] == "json_tool_call", "Tool should be named json_tool_call" - + # Test 2: Verify transform_request removes output_format parameter # Simulate what would happen if parent class added output_format test_data = { @@ -264,20 +258,22 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): "tool_choice": result_params["tool_choice"], "output_format": { # This would be added by parent class for Sonnet 4.5 "type": "json_schema", - "schema": response_format["json_schema"]["schema"] - } + "schema": response_format["json_schema"]["schema"], + }, } - + # Mock the parent transform_request to return data with output_format original_transform = config.__class__.__bases__[0].transform_request - - def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): + + def mock_transform_request( + self, model, messages, optional_params, litellm_params, headers + ): # Return test data that includes output_format return test_data.copy() - + # Temporarily replace parent method config.__class__.__bases__[0].transform_request = mock_transform_request - + try: final_data = config.transform_request( model="claude-3-5-sonnet-20241022", @@ -286,13 +282,15 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): litellm_params={}, headers={}, ) - + # Verify that output_format was removed (fixes the "Extra inputs are not permitted" error) - assert "output_format" not in final_data, "output_format should be removed for VertexAI" + assert ( + "output_format" not in final_data + ), "output_format should be removed for VertexAI" assert "model" not in final_data, "model should be removed for VertexAI" assert "tools" in final_data, "tools should still be present" assert "tool_choice" in final_data, "tool_choice should still be present" - + finally: # Restore original method config.__class__.__bases__[0].transform_request = original_transform @@ -300,43 +298,149 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): def test_vertex_ai_anthropic_other_models_still_use_tools(): """ - Test that other Anthropic models (non-Sonnet 4.5) on VertexAI also use tool-based + Test that other Anthropic models (non-Sonnet 4.5) on VertexAI also use tool-based structured outputs, ensuring consistency across all models. """ config = VertexAIAnthropicConfig() - + response_format = { "type": "json_schema", "json_schema": { "name": "test_schema", - "schema": { - "type": "object", - "properties": { - "result": {"type": "string"} - } - } - } + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, } - + # Test with Claude 3 Sonnet (not 4.5) non_default_params = {"response_format": response_format} optional_params = {} - + result_params = config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model="claude-3-sonnet-20240229", drop_params=False, ) - + # Should still use tool-based approach - assert "tools" in result_params, "Claude 3 Sonnet should also use tool-based structured output" + assert ( + "tools" in result_params + ), "Claude 3 Sonnet should also use tool-based structured output" assert "tool_choice" in result_params, "Tool choice should be present" assert "json_mode" in result_params, "JSON mode should be enabled" + +def test_vertex_ai_anthropic_extra_headers_beta_propagation(): + """Test that anthropic-beta values from extra_headers are propagated to the + anthropic_beta request body field for Vertex AI requests. + + Vertex AI requires beta flags in the request body (anthropic_beta array), + not as HTTP headers. This mirrors the Bedrock handler's behavior of + extracting user-specified beta headers. + """ + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "is_vertex_request": True, + "extra_headers": { + "anthropic-beta": "interleaved-thinking-2025-05-14", + }, + } + + result = config.transform_request( + model="claude-sonnet-4-20250514", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "anthropic_beta" in result + assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"] + assert "extra_headers" not in result + + +def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas(): + """Test that extra_headers betas are merged with auto-detected betas + rather than replacing them.""" + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "is_vertex_request": True, + "extra_headers": { + "anthropic-beta": "interleaved-thinking-2025-05-14", + }, + "context_management": {"edits": [{"type": "compact_20260112"}]}, + } + + result = config.transform_request( + model="claude-opus-4-6", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "anthropic_beta" in result + assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"] + assert "compact-2026-01-12" in result["anthropic_beta"] + + +def test_vertex_ai_anthropic_extra_headers_comma_separated_betas(): + """Test that comma-separated beta values in extra_headers are all extracted.""" + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "is_vertex_request": True, + "extra_headers": { + "anthropic-beta": "interleaved-thinking-2025-05-14,dev-full-thinking-2025-05-14", + }, + } + + result = config.transform_request( + model="claude-sonnet-4-20250514", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "anthropic_beta" in result + assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"] + assert "dev-full-thinking-2025-05-14" in result["anthropic_beta"] + + +def test_vertex_ai_anthropic_no_extra_headers_unchanged(): + """Test that requests without extra_headers still work normally.""" + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "is_vertex_request": True, + } + + result = config.transform_request( + model="claude-sonnet-4-20250514", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "anthropic_beta" not in result + assert "extra_headers" not in result + + def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_header(): """ - Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05 + Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05 from the anthropic-beta headers. """ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( @@ -352,13 +456,18 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea headers = update_headers_with_filtered_beta(headers, "vertex_ai") beta_header = headers.get("anthropic-beta") - assert PROMPT_CACHING_BETA_HEADER not in (beta_header or ""), \ - f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" - assert "other-feature" in (beta_header or ""), \ - "Other non-excluded beta headers should remain" - assert "web-search-2025-03-05" in (beta_header or ""), \ - "Other non-excluded beta headers should remain" + assert PROMPT_CACHING_BETA_HEADER not in ( + beta_header or "" + ), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" + assert "other-feature" in ( + beta_header or "" + ), "Other non-excluded beta headers should remain" + assert "web-search-2025-03-05" in ( + beta_header or "" + ), "Other non-excluded beta headers should remain" # If prompt-caching was the only value, header should be removed completely headers2 = {"anthropic-beta": PROMPT_CACHING_BETA_HEADER} headers2 = update_headers_with_filtered_beta(headers2, "vertex_ai") - assert "anthropic-beta" not in headers2, "Header should be removed if no supported values remain" \ No newline at end of file + assert ( + "anthropic-beta" not in headers2 + ), "Header should be removed if no supported values remain" From f7d03f8a43fec1cc53358483e9bb30d2320403e5 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 8 Feb 2026 05:04:47 -0600 Subject: [PATCH 024/117] fix(streaming): preserve interleaved thinking/redacted blocks --- .../streaming_chunk_builder_utils.py | 58 ++++++++------- .../test_streaming_chunk_builder_utils.py | 74 ++++++++++++++++++- 2 files changed, 104 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 53252df0a28..76c7246b87e 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,6 +1,6 @@ import base64 import time -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast from litellm.types.llms.openai import ( ChatCompletionAssistantContentValue, @@ -326,10 +326,22 @@ class ChunkProcessor: thinking_blocks: List[ Union["ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"] ] = [] - combined_thinking_text: Optional[str] = None - data: Optional[str] = None - signature: Optional[str] = None - type: Literal["thinking", "redacted_thinking"] = "thinking" + current_thinking_text_parts: List[str] = [] + current_signature: Optional[str] = None + + def _flush_thinking_block() -> None: + nonlocal current_thinking_text_parts, current_signature + if len(current_thinking_text_parts) > 0 and current_signature: + thinking_blocks.append( + ChatCompletionThinkingBlock( + type="thinking", + thinking="".join(current_thinking_text_parts), + signature=current_signature, + ) + ) + current_thinking_text_parts = [] + current_signature = None + for chunk in chunks: choices = chunk["choices"] for choice in choices: @@ -339,33 +351,25 @@ class ChunkProcessor: for thinking_block in thinking: thinking_type = thinking_block.get("type", None) if thinking_type and thinking_type == "redacted_thinking": - type = "redacted_thinking" - data = thinking_block.get("data", None) + _flush_thinking_block() + redacted_data = thinking_block.get("data", None) + if redacted_data: + thinking_blocks.append( + ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=redacted_data, + ) + ) else: - type = "thinking" thinking_text = thinking_block.get("thinking", None) if thinking_text: - if combined_thinking_text is None: - combined_thinking_text = "" - - combined_thinking_text += thinking_text + current_thinking_text_parts.append(thinking_text) signature = thinking_block.get("signature", None) + if signature: + current_signature = signature + _flush_thinking_block() - if combined_thinking_text and type == "thinking" and signature: - thinking_blocks.append( - ChatCompletionThinkingBlock( - type=type, - thinking=combined_thinking_text, - signature=signature, - ) - ) - elif data and type == "redacted_thinking": - thinking_blocks.append( - ChatCompletionRedactedThinkingBlock( - type=type, - data=data, - ) - ) + _flush_thinking_block() if len(thinking_blocks) > 0: return thinking_blocks diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index eef206ca667..34c9efd8382 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -158,6 +158,78 @@ def test_get_combined_tool_content(): ] +def test_get_combined_thinking_content_preserves_interleaved_blocks(): + base_chunk = { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "claude-sonnet-4-20250514", + } + + def make_chunk(**delta_kwargs): + return ModelResponseStream( + **{ + **base_chunk, + "choices": [ + { + "index": 0, + "delta": delta_kwargs, + "finish_reason": None, + } + ], + } + ) + + chunks = [ + make_chunk(role="assistant", content=None), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": "Step 1 analysis...", "signature": None} + ] + ), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block1"} + ] + ), + make_chunk( + thinking_blocks=[ + { + "type": "redacted_thinking", + "data": "EuoBCoYBGAIi...encrypted...", + } + ] + ), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": "Step 2 analysis...", "signature": None} + ] + ), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block2"} + ] + ), + ] + + thinking_chunks = [ + chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks") + ] + processor = ChunkProcessor(chunks=chunks) + result = processor.get_combined_thinking_content(thinking_chunks) + + assert result is not None + assert len(result) == 3 + assert result[0]["type"] == "thinking" + assert result[0]["thinking"] == "Step 1 analysis..." + assert result[0]["signature"] == "sig_block1" + assert result[1]["type"] == "redacted_thinking" + assert result[1]["data"] == "EuoBCoYBGAIi...encrypted..." + assert result[2]["type"] == "thinking" + assert result[2]["thinking"] == "Step 2 analysis..." + assert result[2]["signature"] == "sig_block2" + + def test_cache_read_input_tokens_retained(): chunk1 = ModelResponseStream( id="chatcmpl-95aabb85-c39f-443d-ae96-0370c404d70c", @@ -441,4 +513,4 @@ def test_stream_chunk_builder_anthropic_web_search(): assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 assert usage.total_tokens == 77 - assert usage.server_tool_use['web_search_requests'] == 2 \ No newline at end of file + assert usage.server_tool_use['web_search_requests'] == 2 From c63d5fa0b5bc165dd6ae11a50b22358262078dbd Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 8 Feb 2026 05:08:50 -0600 Subject: [PATCH 025/117] test(streaming): build thinking chunks with typed Delta/StreamingChoices --- .../test_streaming_chunk_builder_utils.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 34c9efd8382..da6d8027921 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -168,16 +168,14 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): def make_chunk(**delta_kwargs): return ModelResponseStream( - **{ - **base_chunk, - "choices": [ - { - "index": 0, - "delta": delta_kwargs, - "finish_reason": None, - } - ], - } + **base_chunk, + choices=[ + StreamingChoices( + index=0, + delta=Delta(**delta_kwargs), + finish_reason=None, + ) + ], ) chunks = [ From 381c3756f45eafa8fbdbbf774b694cc99da02773 Mon Sep 17 00:00:00 2001 From: tshushan Date: Sun, 8 Feb 2026 16:24:14 +0200 Subject: [PATCH 026/117] Fix video list pagination cursors not encoded with provider metadata first_id and last_id in the video list response were returned as raw provider IDs while data[].id was properly wrapped with encode_video_id_with_provider(). This caused pagination to break when clients passed unencoded cursors back as the `after` parameter. - Encode first_id/last_id in transform_video_list_response - Decode the `after` param in transform_video_list_request via extract_original_video_id() - Add 6 unit tests covering encoding, decoding, passthrough, and full round-trip pagination Fixes #20708 Co-Authored-By: Claude Opus 4.6 --- litellm/llms/openai/videos/transformation.py | 45 +++-- tests/test_litellm/test_video_generation.py | 175 +++++++++++++++++++ 2 files changed, 209 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 3073b22e1ca..0dd7940a92e 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -269,26 +269,27 @@ class OpenAIVideoConfig(BaseVideoConfig): ) -> Tuple[str, Dict]: """ Transform the video list request for OpenAI API. - + OpenAI API expects the following request: - GET /v1/videos """ # Use the api_base directly for video list url = api_base - + # Prepare query parameters params = {} if after is not None: - params["after"] = after + # Decode the wrapped video ID back to the original provider ID + params["after"] = extract_original_video_id(after) if limit is not None: params["limit"] = str(limit) if order is not None: params["order"] = order - + # Add any extra query parameters if extra_query: params.update(extra_query) - + return url, params def transform_video_list_response( @@ -296,18 +297,40 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, - ) -> Dict[str,str]: + ) -> Dict[str, str]: response_data = raw_response.json() - + if custom_llm_provider and "data" in response_data: for video_obj in response_data.get("data", []): if isinstance(video_obj, dict) and "id" in video_obj: video_obj["id"] = encode_video_id_with_provider( - video_obj["id"], - custom_llm_provider, - video_obj.get("model") + video_obj["id"], + custom_llm_provider, + video_obj.get("model"), ) - + + # Encode pagination cursor IDs so they remain consistent + # with the wrapped data[].id format + data_list = response_data.get("data", []) + if response_data.get("first_id"): + first_model = None + if data_list and isinstance(data_list[0], dict): + first_model = data_list[0].get("model") + response_data["first_id"] = encode_video_id_with_provider( + response_data["first_id"], + custom_llm_provider, + first_model, + ) + if response_data.get("last_id"): + last_model = None + if data_list and isinstance(data_list[-1], dict): + last_model = data_list[-1].get("model") + response_data["last_id"] = encode_video_id_with_provider( + response_data["last_id"], + custom_llm_provider, + last_model, + ) + return response_data def transform_video_delete_request( diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index cfc1535052c..5446a0a7b3f 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -916,6 +916,181 @@ def test_encode_video_id_with_provider_handles_azure_video_prefix(): ) assert encoded_twice == encoded_id # Should return the same encoded ID +class TestVideoListTransformation: + """Tests for video list request/response transformation with provider ID encoding.""" + + def test_transform_video_list_response_encodes_first_id_and_last_id(self): + """Verify that first_id and last_id are encoded with provider metadata.""" + config = OpenAIVideoConfig() + + mock_http_response = MagicMock() + mock_http_response.json.return_value = { + "object": "list", + "data": [ + { + "id": "video_aaa", + "object": "video", + "model": "sora-2", + "status": "completed", + }, + { + "id": "video_bbb", + "object": "video", + "model": "sora-2", + "status": "completed", + }, + ], + "first_id": "video_aaa", + "last_id": "video_bbb", + "has_more": False, + } + + result = config.transform_video_list_response( + raw_response=mock_http_response, + logging_obj=MagicMock(), + custom_llm_provider="azure", + ) + + from litellm.types.videos.utils import decode_video_id_with_provider + + # data[].id should be encoded + for item in result["data"]: + decoded = decode_video_id_with_provider(item["id"]) + assert decoded["custom_llm_provider"] == "azure" + + # first_id and last_id should also be encoded + first_decoded = decode_video_id_with_provider(result["first_id"]) + assert first_decoded["custom_llm_provider"] == "azure" + assert first_decoded["video_id"] == "video_aaa" + assert first_decoded["model_id"] == "sora-2" + + last_decoded = decode_video_id_with_provider(result["last_id"]) + assert last_decoded["custom_llm_provider"] == "azure" + assert last_decoded["video_id"] == "video_bbb" + assert last_decoded["model_id"] == "sora-2" + + def test_transform_video_list_response_no_provider_leaves_ids_unchanged(self): + """When custom_llm_provider is None, all IDs should remain unchanged.""" + config = OpenAIVideoConfig() + + mock_http_response = MagicMock() + mock_http_response.json.return_value = { + "object": "list", + "data": [ + {"id": "video_aaa", "object": "video", "model": "sora-2", "status": "completed"}, + ], + "first_id": "video_aaa", + "last_id": "video_aaa", + "has_more": False, + } + + result = config.transform_video_list_response( + raw_response=mock_http_response, + logging_obj=MagicMock(), + custom_llm_provider=None, + ) + + assert result["data"][0]["id"] == "video_aaa" + assert result["first_id"] == "video_aaa" + assert result["last_id"] == "video_aaa" + + def test_transform_video_list_response_missing_pagination_fields(self): + """first_id / last_id may be absent or null; should not raise.""" + config = OpenAIVideoConfig() + + mock_http_response = MagicMock() + mock_http_response.json.return_value = { + "object": "list", + "data": [ + {"id": "video_aaa", "object": "video", "model": "sora-2", "status": "completed"}, + ], + "has_more": False, + } + + result = config.transform_video_list_response( + raw_response=mock_http_response, + logging_obj=MagicMock(), + custom_llm_provider="azure", + ) + + # data[].id should still be encoded + from litellm.types.videos.utils import decode_video_id_with_provider + + decoded = decode_video_id_with_provider(result["data"][0]["id"]) + assert decoded["custom_llm_provider"] == "azure" + + # first_id / last_id should not be present + assert "first_id" not in result + assert "last_id" not in result + + def test_transform_video_list_request_decodes_after_parameter(self): + """Encoded 'after' cursor should be decoded back to the raw provider ID.""" + from litellm.types.videos.utils import encode_video_id_with_provider + + config = OpenAIVideoConfig() + + raw_id = "video_69888baee890819086dd3366bfc372fe" + encoded_id = encode_video_id_with_provider(raw_id, "azure", "sora-2") + + url, params = config.transform_video_list_request( + api_base="https://my-resource.openai.azure.com/openai/v1/videos", + litellm_params=MagicMock(), + headers={}, + after=encoded_id, + limit=10, + ) + + assert params["after"] == raw_id + assert params["limit"] == "10" + + def test_transform_video_list_request_passes_through_plain_after(self): + """A plain (non-encoded) 'after' value should pass through unchanged.""" + config = OpenAIVideoConfig() + + url, params = config.transform_video_list_request( + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + after="video_plain_id", + ) + + assert params["after"] == "video_plain_id" + + def test_transform_video_list_roundtrip(self): + """first_id from list response should decode correctly when used as after parameter.""" + config = OpenAIVideoConfig() + + # Simulate a list response + mock_http_response = MagicMock() + mock_http_response.json.return_value = { + "object": "list", + "data": [ + {"id": "video_aaa", "object": "video", "model": "sora-2", "status": "completed"}, + {"id": "video_bbb", "object": "video", "model": "sora-2", "status": "completed"}, + ], + "first_id": "video_aaa", + "last_id": "video_bbb", + "has_more": True, + } + + list_result = config.transform_video_list_response( + raw_response=mock_http_response, + logging_obj=MagicMock(), + custom_llm_provider="azure", + ) + + # Use the encoded last_id as the 'after' cursor for the next page + _, params = config.transform_video_list_request( + api_base="https://my-resource.openai.azure.com/openai/v1/videos", + litellm_params=MagicMock(), + headers={}, + after=list_result["last_id"], + ) + + # The after param sent to the upstream API should be the raw video ID + assert params["after"] == "video_bbb" + + class TestVideoEndpointsProxyLitellmParams: """Test that video proxy endpoints (status, content, remix) respect litellm_params from proxy config.""" From 68d788c84d761b6a94d73480e43e6990f5eb4a19 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 8 Feb 2026 08:48:39 -0600 Subject: [PATCH 027/117] fix(responses): preserve streamed tool deltas when id is omitted --- .../streaming_iterator.py | 29 ++++- ...test_tool_call_streaming_transformation.py | 102 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 867c18b6dd4..ea9b8889d39 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -88,6 +88,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._pending_tool_events: List[BaseLiteLLMOpenAIResponseObject] = [] self._tool_output_index_by_call_id: dict[str, int] = {} self._tool_args_by_call_id: dict[str, str] = {} + self._tool_call_id_by_index: dict[int, str] = {} self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item self._final_tool_events_queued: bool = False self._sequence_number: int = 0 @@ -111,6 +112,19 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_output_index_by_call_id[call_id] = idx return idx + def _normalize_tool_call_index(self, tool_call: object) -> Optional[int]: + idx_raw = ( + tool_call.get("index") + if isinstance(tool_call, dict) + else getattr(tool_call, "index", None) + ) + if idx_raw is None: + return None + try: + return int(idx_raw) + except (TypeError, ValueError): + return None + def _is_reasoning_end(self, chunk): delta = chunk.choices[0].delta @@ -143,10 +157,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return for tc in tool_calls: + tc_index = self._normalize_tool_call_index(tc) call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) - if not call_id_raw: + call_id = "" + + if call_id_raw: + call_id = str(call_id_raw) + if tc_index is not None: + self._tool_call_id_by_index[tc_index] = call_id + elif tc_index is not None: + mapped_call_id = self._tool_call_id_by_index.get(tc_index) + if mapped_call_id: + call_id = mapped_call_id + + if not call_id: continue - call_id = str(call_id_raw) fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) fn_name = "" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py index 8d324bea611..4efdc217dce 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py @@ -229,3 +229,105 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): assert sequence_numbers == sorted(sequence_numbers) assert len(set(sequence_numbers)) == len(sequence_numbers) # All unique + +def test_tool_call_delta_without_id_uses_index_mapping(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + chunks = [ + [ + { + "index": 0, + "id": "call_abc123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"lo'}, + } + ], + [{"index": 0, "type": "function", "function": {"arguments": 'cation":'}}], + [{"index": 0, "type": "function", "function": {"arguments": ' "New'}}], + [{"index": 0, "type": "function", "function": {"arguments": ' York"}'}}], + ] + + for tool_calls in chunks: + iterator._queue_tool_call_delta_events(tool_calls) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + streamed_arguments = "".join(evt.delta for evt in delta_events) + + assert streamed_arguments == '{"location": "New York"}' + + output_item_added_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert len(output_item_added_events) == 1 + assert output_item_added_events[0].item.id == "call_abc123" + + +def test_parallel_tool_calls_without_ids_use_index_mapping(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_a", + "type": "function", + "function": {"name": "tool_a", "arguments": '{"x":'}, + }, + { + "index": 1, + "id": "call_b", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"y":'}, + }, + ] + ) + iterator._queue_tool_call_delta_events( + [ + {"index": 0, "type": "function", "function": {"arguments": "1}"}}, + {"index": 1, "type": "function", "function": {"arguments": "2}"}}, + ] + ) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + output_item_added_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert len(output_item_added_events) == 2 + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + arguments_by_call_id = {} + for evt in delta_events: + arguments_by_call_id.setdefault(evt.item_id, "") + arguments_by_call_id[evt.item_id] += evt.delta + + assert arguments_by_call_id["call_a"] == '{"x":1}' + assert arguments_by_call_id["call_b"] == '{"y":2}' From cf17a440cdb4a8ca3055c91971ed2c1569b8c1c5 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 8 Feb 2026 08:53:17 -0600 Subject: [PATCH 028/117] fix(responses): guard ambiguous tool-call index reuse --- .../streaming_iterator.py | 8 +++ ...test_tool_call_streaming_transformation.py | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index ea9b8889d39..5c05526442d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -89,6 +89,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_output_index_by_call_id: dict[str, int] = {} self._tool_args_by_call_id: dict[str, str] = {} self._tool_call_id_by_index: dict[int, str] = {} + self._ambiguous_tool_call_indexes: set[int] = set() self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item self._final_tool_events_queued: bool = False self._sequence_number: int = 0 @@ -164,8 +165,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if call_id_raw: call_id = str(call_id_raw) if tc_index is not None: + existing_call_id = self._tool_call_id_by_index.get(tc_index) + if existing_call_id is not None and existing_call_id != call_id: + # Reusing the same index for multiple call_ids is ambiguous for id-less deltas. + # Guard against silent misrouting by disabling index fallback for this index. + self._ambiguous_tool_call_indexes.add(tc_index) self._tool_call_id_by_index[tc_index] = call_id elif tc_index is not None: + if tc_index in self._ambiguous_tool_call_indexes: + continue mapped_call_id = self._tool_call_id_by_index.get(tc_index) if mapped_call_id: call_id = mapped_call_id diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py index 4efdc217dce..071eefaef47 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_call_streaming_transformation.py @@ -331,3 +331,62 @@ def test_parallel_tool_calls_without_ids_use_index_mapping(): assert arguments_by_call_id["call_a"] == '{"x":1}' assert arguments_by_call_id["call_b"] == '{"y":2}' + + +def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): + iterator = LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + ) + + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_a", + "type": "function", + "function": {"name": "tool_a", "arguments": '{"a":'}, + } + ] + ) + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "id": "call_b", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"b":'}, + } + ] + ) + # Ambiguous chunk: index reused and id missing. We should skip fallback rather than misroute. + iterator._queue_tool_call_delta_events( + [ + { + "index": 0, + "type": "function", + "function": {"arguments": "1}"}, + } + ] + ) + + all_events = [] + while iterator._pending_tool_events: + all_events.append(iterator._pending_tool_events.pop(0)) + + delta_events = [ + evt + for evt in all_events + if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA + ] + arguments_by_call_id = {} + for evt in delta_events: + arguments_by_call_id.setdefault(evt.item_id, "") + arguments_by_call_id[evt.item_id] += evt.delta + + assert arguments_by_call_id["call_a"] == '{"a":' + assert arguments_by_call_id["call_b"] == '{"b":' + assert arguments_by_call_id["call_a"] != '{"a":1}' + assert arguments_by_call_id["call_b"] != '{"b":1}' From 8cd8a01d5ac3946ac43032b14470319ec207bccc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 10:18:19 +0530 Subject: [PATCH 029/117] Add compaction for vertex ai --- .../anthropic_claude3_transformation.py | 29 ++--- .../transformation.py | 23 ++++ ...odel_prices_and_context_window_backup.json | 120 ------------------ model_prices_and_context_window.json | 120 ------------------ ...artner_models_anthropic_messages_config.py | 117 +++++++++++++++++ 5 files changed, 149 insertions(+), 260 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index c936b2cd23c..31119c73d72 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, List, Optional import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -133,27 +134,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): beta_set.add("tool-search-tool-2025-10-19") # Filter out beta headers that Bedrock Invoke doesn't support - # AWS Bedrock only supports a specific whitelist of beta flags - # Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html - BEDROCK_SUPPORTED_BETAS = { - "computer-use-2024-10-22", # Legacy computer use - "computer-use-2025-01-24", # Current computer use (Claude 3.7 Sonnet) - "token-efficient-tools-2025-02-19", # Tool use (Claude 3.7+ and Claude 4+) - "interleaved-thinking-2025-05-14", # Interleaved thinking (Claude 4+) - "output-128k-2025-02-19", # 128K output tokens (Claude 3.7 Sonnet) - "dev-full-thinking-2025-05-14", # Developer mode for raw thinking (Claude 4+) - "context-1m-2025-08-07", # 1 million tokens (Claude Sonnet 4) - "context-management-2025-06-27", # Context management (Claude Sonnet/Haiku 4.5) - "effort-2025-11-24", # Effort parameter (Claude Opus 4.5) - "tool-search-tool-2025-10-19", # Tool search (Claude Opus 4.5) - "tool-examples-2025-10-29", # Tool use examples (Claude Opus 4.5) - } - - # Only keep beta headers that Bedrock supports - beta_set = {beta for beta in beta_set if beta in BEDROCK_SUPPORTED_BETAS} + # Uses centralized configuration from anthropic_beta_headers_config.json + beta_list = list(beta_set) + filtered_beta_list = filter_and_transform_beta_headers( + beta_headers=beta_list, + provider="bedrock", + ) - if beta_set: - _anthropic_request["anthropic_beta"] = list(beta_set) + if filtered_beta_list: + _anthropic_request["anthropic_beta"] = filtered_beta_list return _anthropic_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 918b8ecc225..5a09168282d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -68,6 +68,29 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) + # Check for context management + context_management_param = optional_params.get("context_management") + if context_management_param is not None: + # Check edits array for compact_20260112 type + edits = context_management_param.get("edits", []) + has_compact = False + has_other = False + + for edit in edits: + edit_type = edit.get("type", "") + if edit_type == "compact_20260112": + has_compact = True + else: + has_other = True + + # Add compact header if any compact edits exist + if has_compact: + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) + + # Add context management header if any other edits exist + if has_other: + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + # Check for web search tool for tool in tools: if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5cad0db241f..45dbf14ffc9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -993,66 +993,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "global.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -1143,66 +1083,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "eu.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "apac.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "apac.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5cad0db241f..45dbf14ffc9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -993,66 +993,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "global.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_200k_tokens": 1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -1143,66 +1083,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, - "eu.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, - "apac.anthropic.claude-opus-4-6-v1": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": false, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "apac.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 623f8c579ff..7bb84b0a2c1 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -98,3 +98,120 @@ def test_web_search_header_not_added_without_tool(): # Assert that the anthropic-beta header is NOT present when no web search tool assert "anthropic-beta" not in updated_headers, \ "anthropic-beta header should not be present without web search tool" + + +def test_compact_context_management_header_added(): + """Test that compact-2026-01-12 beta header is added when context_management with compact_20260112 is used""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + headers = {} + litellm_params = { + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + } + # Include context_management with compact_20260112 + optional_params = { + "context_management": { + "edits": [ + {"type": "compact_20260112"} + ] + } + } + + with patch.object( + config, "_ensure_access_token", return_value=("token", "test-project") + ), patch.object( + config, "get_complete_vertex_url", return_value="https://mock-url" + ): + updated_headers, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params=optional_params, + litellm_params=litellm_params, + api_base=None, + ) + + # Assert that the anthropic-beta header with compact-2026-01-12 is present + assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present" + assert "compact-2026-01-12" in updated_headers["anthropic-beta"], \ + f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}" + + +def test_context_management_header_added_for_other_edits(): + """Test that context-management-2025-06-27 beta header is added for non-compact edits""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + headers = {} + litellm_params = { + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + } + # Include context_management with other edit types + optional_params = { + "context_management": { + "edits": [ + {"type": "some_other_type"} + ] + } + } + + with patch.object( + config, "_ensure_access_token", return_value=("token", "test-project") + ), patch.object( + config, "get_complete_vertex_url", return_value="https://mock-url" + ): + updated_headers, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params=optional_params, + litellm_params=litellm_params, + api_base=None, + ) + + # Assert that the anthropic-beta header with context-management-2025-06-27 is present + assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present" + assert "context-management-2025-06-27" in updated_headers["anthropic-beta"], \ + f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}" + + +def test_both_compact_and_context_management_headers_added(): + """Test that both compact and context-management beta headers are added when both edit types are present""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + headers = {} + litellm_params = { + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + } + # Include context_management with both compact and other edit types + optional_params = { + "context_management": { + "edits": [ + {"type": "compact_20260112"}, + {"type": "some_other_type"} + ] + } + } + + with patch.object( + config, "_ensure_access_token", return_value=("token", "test-project") + ), patch.object( + config, "get_complete_vertex_url", return_value="https://mock-url" + ): + updated_headers, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params=optional_params, + litellm_params=litellm_params, + api_base=None, + ) + + # Assert that both beta headers are present + assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present" + assert "compact-2026-01-12" in updated_headers["anthropic-beta"], \ + f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}" + assert "context-management-2025-06-27" in updated_headers["anthropic-beta"], \ + f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}" From d41df6053a0ba8346db4cc0f0f87ad180442e148 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 10:35:11 +0530 Subject: [PATCH 030/117] Add all new feat for v1/messages --- docs/my-website/blog/claude_opus_4_6/index.md | 127 ++++++++++++++++++ .../anthropic/messages_transformation.py | 9 ++ 2 files changed, 136 insertions(+) diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index 0397f1288f7..47dcd629226 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -401,3 +401,130 @@ Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceedin Available at 1.1× token pricing. LiteLLM supports this pricing model. +## Using `/v1/messages` Endpoint + +LiteLLM supports the Anthropic `/v1/messages` API format across all providers. This allows you to use Anthropic-specific features like adaptive thinking, compaction, and 1M token context with consistent syntax across Anthropic, Azure AI, Vertex AI, and Bedrock. + + +```yaml +model_list: + # Anthropic + - model_name: claude-opus-4-6 + litellm_params: + model: anthropic/claude-opus-4-6 + + # Azure AI + - model_name: claude-azure-opus-4-6 + litellm_params: + model: azure_ai/claude-opus-4-6 + api_base: https://your-resource.services.ai.azure.com/anthropic + + # Vertex AI + - model_name: claude-vertex-opus-4-6 + litellm_params: + model: vertex_ai/claude-opus-4-6 + vertex_project: your-project-id + vertex_location: us-east5 + + # Bedrock + - model_name: claude-bedrock-opus-4-6 + litellm_params: + model: bedrock/anthropic.claude-opus-4-6-v1:0 + aws_region_name: us-east-1 +``` + +### Feature Support Matrix + +| Feature | Anthropic | Bedrock Invoke | Bedrock Converse | Vertex AI | Azure AI | +|---------|-----------|----------------|------------------|-----------|----------| +| `/v1/messages` Compaction | ✅ | ❌ Not supported | ❌ Not supported | ✅ | ✅ | +| 1M Token Context | ✅ | ✅ | ✅ | ✅ | ✅ | +| US-Only Inference - cost tracking | ✅ | Not applicable | Not applicable | Not applicable | Not applicable | +| Adaptive Thinking | ✅ | ✅ | ✅ | ✅ | ✅ | + +### Adaptive Thinking + +Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode: + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 16000, + "thinking": { + "type": "adaptive" + }, + "messages": [ + { + "role": "user", + "content": "Explain why the sum of two even numbers is always even." + } + ] +}' +``` + +### Context Management - Compaction + +Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled. + +:::info +**Provider Support:** Compaction is supported on Anthropic, Bedrock Invoke Azure AI, and Vertex AI. It is **not supported** on Bedrock Converse API. +::: + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "Hi" + } + ], + "context_management": { + "edits": [ + { + "type": "compact_20260112" + } + ] + } +}' +``` + +LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled. + +### 1M Token Context Window + +To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider. + +**Step 1: Enable header forwarding in your config** + +```yaml +general_settings: + forward_client_headers_to_llm_api: true +``` + +**Step 2: Send requests with the beta header** + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'anthropic-beta: context-1m-2025-08-07' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 16000, + "messages": [ + { + "role": "user", + "content": "Explain why the sum of two even numbers is always even." + } + ] +}' +``` + diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 0d00c907031..f86ec7082f2 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -3,6 +3,9 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig """ from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from litellm.anthropic_beta_headers_manager import ( + update_headers_with_filtered_beta, +) from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -68,6 +71,12 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): optional_params=optional_params, ) + # Filter out unsupported beta headers for Azure AI + headers = update_headers_with_filtered_beta( + headers=headers, + provider="azure_ai", + ) + return headers, api_base def get_complete_url( From 3307f3d1c6c069ff0b9505bfdd4879f84771ea56 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 10:47:13 +0530 Subject: [PATCH 031/117] Add inference_geo as supported messages param --- .../experimental_pass_through/messages/transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index bb40f9df266..145a7157139 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -46,6 +46,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "thinking", "context_management", "output_format", + "inference_geo", # TODO: Add Anthropic `metadata` support # "metadata", ] From 20440bcadca46aec3aaa5704e4afc5459902251f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 10:51:50 +0530 Subject: [PATCH 032/117] Add inference based costing --- docs/my-website/blog/claude_opus_4_6/index.md | 283 ++++++++++++------ 1 file changed, 187 insertions(+), 96 deletions(-) diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index 47dcd629226..78411b90ba7 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -223,11 +223,16 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ -## Compaction +## Advanced Features + +### Compaction + + + Litellm supports enabling compaction for the new claude-opus-4-6. -### Enabling Compaction +**Enabling Compaction** To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type: @@ -255,8 +260,43 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ``` All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request. + + -### Response with Compaction Block +Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled. + +:::info +**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs). +::: + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "Hi" + } + ], + "context_management": { + "edits": [ + { + "type": "compact_20260112" + } + ] + } +}' +``` + + + + + +**Response with Compaction Block** The response will include the compaction summary in `provider_specific_fields.compaction_blocks`: @@ -292,7 +332,7 @@ The response will include the compaction summary in `provider_specific_fields.co } ``` -### Using Compaction Blocks in Follow-up Requests +**Using Compaction Blocks in Follow-up Requests** To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`: @@ -340,15 +380,17 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` -### Streaming Support +**Streaming Support** Compaction blocks are also supported in streaming mode. You'll receive: - `compaction_start` event when a compaction block begins - `compaction_delta` events with the compaction content - The accumulated `compaction_blocks` in `provider_specific_fields` +### Adaptive Thinking -## Adaptive Thinking + + LiteLLM supports adaptive thinking through the `reasoning_effort` parameter: @@ -368,81 +410,8 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` -## Effort Levels - -Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter: - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer $LITELLM_KEY' \ ---data '{ - "model": "claude-opus-4-6", - "messages": [ - { - "role": "user", - "content": "Explain quantum computing" - } - ], - "output_config": { - "effort": "medium" - } - -}' -``` - -You can use reasoning effort plus output_config to have more control on the model. - -## 1M Token Context (Beta) - -Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts. - -## US-Only Inference - -Available at 1.1× token pricing. LiteLLM supports this pricing model. - -## Using `/v1/messages` Endpoint - -LiteLLM supports the Anthropic `/v1/messages` API format across all providers. This allows you to use Anthropic-specific features like adaptive thinking, compaction, and 1M token context with consistent syntax across Anthropic, Azure AI, Vertex AI, and Bedrock. - - -```yaml -model_list: - # Anthropic - - model_name: claude-opus-4-6 - litellm_params: - model: anthropic/claude-opus-4-6 - - # Azure AI - - model_name: claude-azure-opus-4-6 - litellm_params: - model: azure_ai/claude-opus-4-6 - api_base: https://your-resource.services.ai.azure.com/anthropic - - # Vertex AI - - model_name: claude-vertex-opus-4-6 - litellm_params: - model: vertex_ai/claude-opus-4-6 - vertex_project: your-project-id - vertex_location: us-east5 - - # Bedrock - - model_name: claude-bedrock-opus-4-6 - litellm_params: - model: bedrock/anthropic.claude-opus-4-6-v1:0 - aws_region_name: us-east-1 -``` - -### Feature Support Matrix - -| Feature | Anthropic | Bedrock Invoke | Bedrock Converse | Vertex AI | Azure AI | -|---------|-----------|----------------|------------------|-----------|----------| -| `/v1/messages` Compaction | ✅ | ❌ Not supported | ❌ Not supported | ✅ | ✅ | -| 1M Token Context | ✅ | ✅ | ✅ | ✅ | ✅ | -| US-Only Inference - cost tracking | ✅ | Not applicable | Not applicable | Not applicable | Not applicable | -| Adaptive Thinking | ✅ | ✅ | ✅ | ✅ | ✅ | - -### Adaptive Thinking + + Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode: @@ -465,13 +434,40 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ }' ``` -### Context Management - Compaction + + -Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled. +### Effort Levels -:::info -**Provider Support:** Compaction is supported on Anthropic, Bedrock Invoke Azure AI, and Vertex AI. It is **not supported** on Bedrock Converse API. -::: + + + +Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-6", + "messages": [ + { + "role": "user", + "content": "Explain quantum computing" + } + ], + "output_config": { + "effort": "medium" + } +}' +``` + +You can use reasoning effort plus output_config to have more control on the model. + + + + +Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter: ```bash curl --location 'http://0.0.0.0:4000/v1/messages' \ @@ -483,22 +479,54 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ "messages": [ { "role": "user", - "content": "Hi" + "content": "Explain quantum computing" } ], - "context_management": { - "edits": [ - { - "type": "compact_20260112" - } - ] + "output_config": { + "effort": "medium" } }' ``` -LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled. + + -### 1M Token Context Window +### 1M Token Context (Beta) + +Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts. + + + + +To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider. + +**Step 1: Enable header forwarding in your config** + +```yaml +general_settings: + forward_client_headers_to_llm_api: true +``` + +**Step 2: Send requests with the beta header** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--header 'anthropic-beta: context-1m-2025-08-07' \ +--data '{ + "model": "claude-opus-4-6", + "messages": [ + { + "role": "user", + "content": "Analyze this large document..." + } + ] +}' +``` + + + To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider. @@ -522,9 +550,72 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ "messages": [ { "role": "user", - "content": "Explain why the sum of two even numbers is always even." + "content": "Analyze this large document..." } ] }' ``` +:::tip +You can combine multiple beta headers by separating them with commas: +```bash +--header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12' +``` +::: + + + + +### US-Only Inference + +Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference. + + + + +Use the `inference_geo` parameter to specify US-only inference: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-6", + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ], + "inference_geo": "us" +}' +``` + +LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking. + + + + +Use the `inference_geo` parameter to specify US-only inference: + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ], + "inference_geo": "us" +}' +``` + +LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking. + + + From 29e6efade9efe45ef88d2cc2ac334dda8ed2d190 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 10:57:04 +0530 Subject: [PATCH 033/117] Add inference_geo as supported messages param --- litellm/types/llms/anthropic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index fedf419efd6..d5172c7a266 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -355,6 +355,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): tool_choice: Optional[Union[AnthropicMessagesToolChoice, Dict]] tools: Optional[List[Union[AllAnthropicToolsValues, Dict]]] top_k: Optional[int] + inference_geo: Optional[str] top_p: Optional[float] mcp_servers: Optional[List[AnthropicMcpServerTool]] context_management: Optional[Dict[str, Any]] From b822e2e0ffe58ee2bbaae91038c7fa0f6ed33910 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 11:28:00 +0530 Subject: [PATCH 034/117] Add support for fast param --- litellm/llms/anthropic/chat/handler.py | 12 +- litellm/llms/anthropic/chat/transformation.py | 14 ++ litellm/llms/anthropic/cost_calculation.py | 15 +- .../messages/transformation.py | 8 +- ...odel_prices_and_context_window_backup.json | 93 ++++++++++ litellm/types/llms/anthropic.py | 2 + model_prices_and_context_window.json | 93 ++++++++++ .../test_anthropic_chat_transformation.py | 161 ++++++++++++++++++ 8 files changed, 389 insertions(+), 9 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 485e95d6489..e85c0d0d017 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -75,6 +75,7 @@ async def make_call( logging_obj, timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, + speed: Optional[str] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_aclient @@ -103,6 +104,7 @@ async def make_call( streaming_response=response.aiter_lines(), sync_stream=False, json_mode=json_mode, + speed=speed, ) # LOGGING @@ -126,6 +128,7 @@ def make_sync_call( logging_obj, timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, + speed: Optional[str] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_client # re-use a module level client @@ -159,7 +162,7 @@ def make_sync_call( ) completion_stream = ModelResponseIterator( - streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode + streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode, speed=speed ) # LOGGING @@ -213,6 +216,7 @@ class AnthropicChatCompletion(BaseLLM): logging_obj=logging_obj, timeout=timeout, json_mode=json_mode, + speed=optional_params.get("speed") if optional_params else None, ) streamwrapper = CustomStreamWrapper( completion_stream=completion_stream, @@ -427,6 +431,7 @@ class AnthropicChatCompletion(BaseLLM): logging_obj=logging_obj, timeout=timeout, json_mode=json_mode, + speed=optional_params.get("speed") if optional_params else None, ) return CustomStreamWrapper( completion_stream=completion_stream, @@ -485,13 +490,14 @@ class AnthropicChatCompletion(BaseLLM): class ModelResponseIterator: def __init__( - self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False + self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False, speed: Optional[str] = None ): self.streaming_response = streaming_response self.response_iterator = self.streaming_response self.content_blocks: List[ContentBlockDelta] = [] self.tool_index = -1 self.json_mode = json_mode + self.speed = speed # Generate response ID once per stream to match OpenAI-compatible behavior self.response_id = _generate_id() @@ -541,7 +547,7 @@ class ModelResponseIterator: def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage: return AnthropicConfig().calculate_usage( - usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None + usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None, speed=self.speed ) def _content_block_delta_helper(self, chunk: dict) -> Tuple[ diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 02b8d952445..82aa7390188 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -190,6 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "response_format", "user", "web_search_options", + "speed", ] if "claude-3-7-sonnet" in model or supports_reasoning( @@ -882,6 +883,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): elif param == "context_management" and isinstance(value, dict): # Pass through Anthropic-specific context_management parameter optional_params["context_management"] = value + elif param == "speed" and isinstance(value, str): + # Pass through Anthropic-specific speed parameter for fast mode + optional_params["speed"] = value ## handle thinking tokens self.update_optional_params_with_thinking_tokens( @@ -1096,6 +1100,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._ensure_beta_header( headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value ) + if optional_params.get("speed") == "fast": + self._ensure_beta_header( + headers, ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value + ) return headers def transform_request( @@ -1349,6 +1357,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None, + speed: Optional[str] = None, ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -1447,6 +1456,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else None ), inference_geo=inference_geo, + speed=speed, ) return usage @@ -1457,6 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response: ModelResponse, json_mode: Optional[bool] = None, prefix_prompt: Optional[str] = None, + speed: Optional[str] = None, ): _hidden_params: Dict = {} _hidden_params["additional_headers"] = process_anthropic_headers( @@ -1553,6 +1564,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage_object=completion_response["usage"], reasoning_content=reasoning_content, completion_response=completion_response, + speed=speed, ) setattr(model_response, "usage", usage) # type: ignore @@ -1621,6 +1633,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) prefix_prompt = self.get_prefix_prompt(messages=messages) + speed = optional_params.get("speed") model_response = self.transform_parsed_response( completion_response=completion_response, @@ -1628,6 +1641,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response=model_response, json_mode=json_mode, prefix_prompt=prefix_prompt, + speed=speed, ) return model_response diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 11b61cc92f0..271406f2f7d 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -22,13 +22,18 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - # If usage has inference_geo, prepend it as prefix to model name + model_with_prefix = model + + # First, prepend inference_geo if present if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]: - model_with_geo_prefix = f"{usage.inference_geo}/{model}" - else: - model_with_geo_prefix = model + model_with_prefix = f"{usage.inference_geo}/{model_with_prefix}" + + # Then, prepend speed if it's "fast" + if hasattr(usage, "speed") and usage.speed == "fast": + model_with_prefix = f"fast/{model_with_prefix}" + prompt_cost, completion_cost = generic_cost_per_token( - model=model_with_geo_prefix, usage=usage, custom_llm_provider="anthropic" + model=model_with_prefix, usage=usage, custom_llm_provider="anthropic" ) return prompt_cost, completion_cost diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 145a7157139..9e28c139686 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -47,6 +47,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "context_management", "output_format", "inference_geo", + "speed", # TODO: Add Anthropic `metadata` support # "metadata", ] @@ -184,10 +185,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): - context_management: adds 'context-management-2025-06-27' - tool_search: adds provider-specific tool search header - output_format: adds 'structured-outputs-2025-11-13' + - speed: adds 'fast-mode-2026-02-01' Args: headers: Request headers dict - optional_params: Optional parameters including tools, context_management, output_format + optional_params: Optional parameters including tools, context_management, output_format, speed custom_llm_provider: Provider name for looking up correct tool search header """ beta_values: set = set() @@ -224,6 +226,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if optional_params.get("output_format") is not None: beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value) + # Check for fast mode + if optional_params.get("speed") == "fast": + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value) + # Check for tool search tools tools = optional_params.get("tools") if tools: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 45dbf14ffc9..543f2f14d7a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7663,6 +7663,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, @@ -7694,6 +7725,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/us/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -7725,6 +7787,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6-20260205": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index d5172c7a266..84ac01a4ece 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -361,6 +361,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): context_management: Optional[Dict[str, Any]] container: Optional[Dict[str, Any]] # Container config with skills for code execution output_format: Optional[AnthropicOutputSchema] # Structured outputs support + speed: Optional[str] # Fast mode support for Opus models class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): @@ -637,6 +638,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): COMPACT_2026_01_12 = "compact-2026-01-12" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" + FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 45dbf14ffc9..543f2f14d7a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7663,6 +7663,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, @@ -7694,6 +7725,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/us/claude-opus-4-6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, @@ -7725,6 +7787,37 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 346 }, + "fast/claude-opus-4-6-20260205": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 3e-05, + "input_cost_per_token_above_200k_tokens": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00015, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "us/claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 49db7367c67..e3bd7d2bb31 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2506,3 +2506,164 @@ def test_compaction_block_empty_list_not_added(): provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None + + +def test_fast_mode_beta_header(): + """ + Test that fast mode correctly adds the fast-mode-2026-02-01 beta header. + """ + config = AnthropicConfig() + + headers = {} + optional_params = {"speed": "fast"} + + result_headers = config.update_headers_with_optional_anthropic_beta( + headers=headers, + optional_params=optional_params + ) + + assert "anthropic-beta" in result_headers + assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"] + + +def test_fast_mode_with_other_beta_headers(): + """ + Test that fast mode beta header is combined with other beta headers. + """ + config = AnthropicConfig() + + headers = {} + optional_params = { + "speed": "fast", + "output_format": {"type": "json_object"} + } + + result_headers = config.update_headers_with_optional_anthropic_beta( + headers=headers, + optional_params=optional_params + ) + + assert "anthropic-beta" in result_headers + assert "fast-mode-2026-02-01" in result_headers["anthropic-beta"] + assert "structured-outputs-2025-11-13" in result_headers["anthropic-beta"] + + +def test_fast_mode_usage_calculation(): + """ + Test that fast mode speed parameter is passed through to usage object. + """ + config = AnthropicConfig() + + usage_object = { + "input_tokens": 1000, + "output_tokens": 500, + } + + usage = config.calculate_usage( + usage_object=usage_object, + reasoning_content=None, + speed="fast" + ) + + assert usage.prompt_tokens == 1000 + assert usage.completion_tokens == 500 + assert hasattr(usage, "speed") + assert usage.speed == "fast" + + +def test_fast_mode_cost_calculation(): + """ + Test that fast mode correctly prepends 'fast/' to model name for pricing lookup. + """ + from unittest.mock import patch + + from litellm.llms.anthropic.cost_calculation import cost_per_token + from litellm.types.utils import Usage + + # Mock the generic_cost_per_token to verify correct model name is passed + with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: + mock_cost.return_value = (0.03, 0.15) # $30 and $150 per MTok + + # Test fast mode + usage_fast = Usage( + prompt_tokens=1000, + completion_tokens=1000, + speed="fast" + ) + + prompt_cost, completion_cost = cost_per_token( + model="claude-opus-4-6", + usage=usage_fast + ) + + # Verify that generic_cost_per_token was called with "fast/claude-opus-4-6" + mock_cost.assert_called_once() + call_args = mock_cost.call_args + assert call_args[1]['model'] == "fast/claude-opus-4-6" + assert call_args[1]['custom_llm_provider'] == "anthropic" + + +def test_fast_mode_with_inference_geo(): + """ + Test that fast mode works correctly with inference_geo prefix. + Expected format: fast/us/claude-opus-4-6 + """ + from unittest.mock import patch + + from litellm.llms.anthropic.cost_calculation import cost_per_token + from litellm.types.utils import Usage + + # Mock the generic_cost_per_token to verify correct model name is passed + with patch('litellm.llms.anthropic.cost_calculation.generic_cost_per_token') as mock_cost: + mock_cost.return_value = (0.03, 0.15) + + # Test with both speed and inference_geo + usage = Usage( + prompt_tokens=1000, + completion_tokens=1000, + speed="fast", + inference_geo="us" + ) + + # This should look up "fast/us/claude-opus-4-6" in pricing + prompt_cost, completion_cost = cost_per_token( + model="claude-opus-4-6", + usage=usage + ) + + # Verify that generic_cost_per_token was called with "fast/us/claude-opus-4-6" + mock_cost.assert_called_once() + call_args = mock_cost.call_args + assert call_args[1]['model'] == "fast/us/claude-opus-4-6" + assert call_args[1]['custom_llm_provider'] == "anthropic" + + +def test_fast_mode_parameter_in_supported_params(): + """ + Test that 'speed' is in the list of supported OpenAI params. + """ + config = AnthropicConfig() + + supported_params = config.get_supported_openai_params(model="claude-opus-4-6") + + assert "speed" in supported_params + + +def test_fast_mode_parameter_mapping(): + """ + Test that speed parameter is correctly mapped in map_openai_params. + """ + config = AnthropicConfig() + + non_default_params = {"speed": "fast"} + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-opus-4-6", + drop_params=False + ) + + assert "speed" in result + assert result["speed"] == "fast" From c3b1c0a59068ec618e9f3f3af6faaf5c3b7c2754 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 11:36:10 +0530 Subject: [PATCH 035/117] Add fast mode for other providers --- litellm/anthropic_beta_headers_config.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 193091c0176..1e99e35aabe 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -13,7 +13,8 @@ "web-fetch-2025-09-10", "code-execution-2025-08-25", "skills-2025-10-02", - "files-api-2025-04-14" + "files-api-2025-04-14", + "fast-mode-2026-02-01" ], "bedrock": [ "advanced-tool-use-2025-11-20", @@ -22,7 +23,8 @@ "web-fetch-2025-09-10", "code-execution-2025-08-25", "skills-2025-10-02", - "files-api-2025-04-14" + "files-api-2025-04-14", + "fast-mode-2026-02-01" ], "vertex_ai": [ "prompt-caching-scope-2026-01-05" From 319453d059e69e46117f34d309a5a0acb19bdcb3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 11:39:35 +0530 Subject: [PATCH 036/117] Add documentation for Fast Mode --- docs/my-website/blog/claude_opus_4_6/index.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index 78411b90ba7..3fd70661543 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -619,3 +619,93 @@ LiteLLM will automatically apply the 1.1× pricing multiplier for US-only infere + +### Fast Mode + +:::info +Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock. +::: + +**Pricing:** +- Standard: $5 input / $25 output per MTok +- Fast: $30 input / $150 output per MTok (6× premium) + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-6", + "messages": [ + { + "role": "user", + "content": "Refactor this module..." + } + ], + "max_tokens": 4096, + "speed": "fast" +}' +``` + +**Using OpenAI SDK:** + +```python +import openai + +client = openai.OpenAI( + api_key="your-litellm-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Refactor this module..."}], + max_tokens=4096, + extra_body={"speed": "fast"} +) +``` + +**Using LiteLLM SDK:** + +```python +from litellm import completion + +response = completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "Refactor this module..."}], + max_tokens=4096, + speed="fast" +) +``` + +LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations. + + + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-6", + "max_tokens": 4096, + "speed": "fast", + "messages": [ + { + "role": "user", + "content": "Refactor this module..." + } + ] +}' +``` + +LiteLLM automatically: +- Adds the `fast-mode-2026-02-01` beta header +- Tracks the 6× premium pricing in cost calculations + + + From 248fe6573635078a4f59196cab0afa377ae57719 Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Fri, 30 Jan 2026 12:01:14 +0900 Subject: [PATCH 037/117] add missing indexes on VerificationToken table --- .../migration.sql | 8 ++++++++ .../litellm_proxy_extras/schema.prisma | 10 ++++++++++ litellm/proxy/schema.prisma | 10 ++++++++++ schema.prisma | 10 ++++++++++ 4 files changed, 38 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql new file mode 100644 index 00000000000..572eea9b529 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql @@ -0,0 +1,8 @@ +-- CreateIndex +CREATE INDEX "LiteLLM_VerificationToken_user_id_team_id_idx" ON "LiteLLM_VerificationToken"("user_id", "team_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_VerificationToken_team_id_idx" ON "LiteLLM_VerificationToken"("team_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_VerificationToken_budget_reset_at_expires_idx" ON "LiteLLM_VerificationToken"("budget_reset_at", "expires"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c2a599c178f..b1ca1f71c9e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -310,6 +310,16 @@ model LiteLLM_VerificationToken { litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) + + // SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub" + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2 + @@index([user_id, team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2 + @@index([team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3 + @@index([budget_reset_at, expires]) } // Audit table for deleted keys - preserves spend and key information for historical tracking diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 279946f78de..1750efed92c 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -308,6 +308,16 @@ model LiteLLM_VerificationToken { litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) + + // SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub" + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2 + @@index([user_id, team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2 + @@index([team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3 + @@index([budget_reset_at, expires]) } // Audit table for deleted keys - preserves spend and key information for historical tracking diff --git a/schema.prisma b/schema.prisma index ecf4e06ef6e..9a87a491cf7 100644 --- a/schema.prisma +++ b/schema.prisma @@ -310,6 +310,16 @@ model LiteLLM_VerificationToken { litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) + + // SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub" + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2 + @@index([user_id, team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2 + @@index([team_id]) + + // SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3 + @@index([budget_reset_at, expires]) } // Audit table for deleted keys - preserves spend and key information for historical tracking From 3d49388d8e2f5ef1fe42bf4548ffcb687c8e826f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 13:36:01 +0530 Subject: [PATCH 038/117] Fix structured response of tool call --- litellm/integrations/websearch_interception/transformation.py | 4 ++-- litellm/llms/openai/openai.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 0884d408c84..3201d99d69a 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -3,7 +3,7 @@ WebSearch Tool Transformation Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. """ - +import json from typing import Any, Dict, List, Tuple from litellm._logging import verbose_logger @@ -301,7 +301,7 @@ class WebSearchTransformation: "type": "function", "function": { "name": tc["name"], - "arguments": str(tc["input"]), + "arguments": json.dumps(tc["input"]) if isinstance(tc["input"], dict) else str(tc["input"]), }, } for tc in tool_calls diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index c6f502d3a25..da87852dff5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -926,7 +926,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj=logging_obj, ) stringified_response = response.model_dump() - print(f"🔥stringified_response: {stringified_response}") logging_obj.post_call( input=data["messages"], api_key=api_key, From 4e94ecb08d7fdeaaaff14d239b7bc38d23f53466 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 13:41:29 +0530 Subject: [PATCH 039/117] Add tests for WebSearch interception with chat completions API --- test_websearch_chat_completion.py | 136 ------ .../test_websearch_chat_completion.py | 398 ++++++++++++++++++ 2 files changed, 398 insertions(+), 136 deletions(-) delete mode 100644 test_websearch_chat_completion.py create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py diff --git a/test_websearch_chat_completion.py b/test_websearch_chat_completion.py deleted file mode 100644 index e572e4d860c..00000000000 --- a/test_websearch_chat_completion.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Test script for WebSearch interception with chat completions API. - -This script demonstrates how to use the websearch_interception callback -with litellm.acompletion() for transparent server-side web search execution. -""" -import asyncio -import litellm - -# Enable verbose logging to see what's happening -litellm.set_verbose = True - - -async def test_websearch_chat_completion(): - """Test websearch interception with chat completions API.""" - - # Configure WebSearch interception - litellm.callbacks = ["websearch_interception"] - - print("\n" + "="*80) - print("Testing WebSearch Interception with Chat Completions API") - print("="*80 + "\n") - - # User makes a simple completion call with tools - print("Making request to GPT-4o with litellm_web_search tool...") - print("Question: What's the weather in San Francisco today?") - print("\nExpected behavior:") - print("1. Model calls litellm_web_search tool") - print("2. Server executes web search automatically") - print("3. Server makes follow-up request with search results") - print("4. User gets final answer\n") - - response = await litellm.acompletion( - model="gpt-4o", - messages=[ - {"role": "user", "content": "What's the weather in San Francisco today?"} - ], - tools=[ - { - "type": "function", - "function": { - "name": "litellm_web_search", - "description": "Search the web for information", - "parameters": { - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search query"} - }, - "required": ["query"] - } - } - } - ] - ) - - print("\n" + "-"*80) - print("FINAL RESPONSE:") - print("-"*80) - print(f"\nContent: {response.choices[0].message.content}") - print(f"\nFinish reason: {response.choices[0].finish_reason}") - - # Check if we got tool_calls (should NOT if agentic loop worked) - if hasattr(response.choices[0].message, 'tool_calls') and response.choices[0].message.tool_calls: - print("\n⚠️ WARNING: Got tool_calls in response!") - print("This means the agentic loop did NOT execute automatically.") - print(f"Tool calls: {response.choices[0].message.tool_calls}") - else: - print("\n✅ SUCCESS: No tool_calls in response!") - print("The agentic loop executed automatically and returned the final answer.") - - print("\n" + "="*80 + "\n") - - -async def test_streaming_websearch(): - """Test websearch interception with streaming.""" - - # Configure WebSearch interception - litellm.callbacks = ["websearch_interception"] - - print("\n" + "="*80) - print("Testing WebSearch Interception with STREAMING") - print("="*80 + "\n") - - print("Making STREAMING request to GPT-4o with litellm_web_search tool...") - print("Question: What are the latest AI news?") - - response = await litellm.acompletion( - model="gpt-4o", - messages=[ - {"role": "user", "content": "What are the latest AI news from today?"} - ], - tools=[ - { - "type": "function", - "function": { - "name": "litellm_web_search", - "description": "Search the web for information", - "parameters": { - "type": "object", - "properties": { - "query": {"type": "string"} - } - } - } - } - ], - stream=True - ) - - print("\n" + "-"*80) - print("STREAMING RESPONSE:") - print("-"*80 + "\n") - - full_content = "" - async for chunk in response: - if hasattr(chunk.choices[0].delta, 'content') and chunk.choices[0].delta.content: - content = chunk.choices[0].delta.content - print(content, end="", flush=True) - full_content += content - - print("\n\n✅ Streaming completed successfully!") - print(f"Total content length: {len(full_content)} chars") - print("\n" + "="*80 + "\n") - - -if __name__ == "__main__": - print("\nWebSearch Interception Test Suite") - print("==================================\n") - print("This test demonstrates transparent server-side web search execution.") - print("The agentic loop happens automatically - user just gets the final answer.\n") - - # Run tests - asyncio.run(test_websearch_chat_completion()) - - # Uncomment to test streaming - # asyncio.run(test_streaming_websearch()) diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py new file mode 100644 index 00000000000..1b53633484d --- /dev/null +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py @@ -0,0 +1,398 @@ +""" +Integration tests for WebSearch interception with chat completions API. + +Tests the end-to-end flow of websearch_interception callback with +litellm.acompletion() for transparent server-side web search execution. +""" +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm +from litellm.integrations.websearch_interception.handler import ( + WebSearchInterceptionLogger, +) +from litellm.types.utils import LlmProviders, ModelResponse + + +@pytest.fixture +def mock_search_response(): + """Mock search response from litellm.asearch()""" + mock_response = MagicMock() + mock_response.results = [ + MagicMock( + title="Weather in San Francisco", + url="https://weather.com/sf", + snippet="Current weather: 65°F, partly cloudy", + ) + ] + return mock_response + + +@pytest.fixture +def websearch_logger(): + """Create a WebSearchInterceptionLogger instance""" + return WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.OPENAI, LlmProviders.MINIMAX] + ) + + +@pytest.mark.asyncio +@pytest.mark.skipif( + os.environ.get("OPENAI_API_KEY") is None, + reason="OPENAI_API_KEY not set", +) +async def test_websearch_chat_completion_with_openai(): + """Test websearch interception with OpenAI chat completions API. + + This test verifies that: + 1. Model calls litellm_web_search tool + 2. Server executes web search automatically + 3. Server makes follow-up request with search results + 4. User gets final answer without tool_calls + """ + # Configure WebSearch interception + original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] + websearch_logger = WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.OPENAI] + ) + litellm.callbacks = [websearch_logger] + + try: + response = await litellm.acompletion( + model="gpt-4o-mini", # Use cheaper model for testing + messages=[ + {"role": "user", "content": "What's the weather in San Francisco today?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query", + } + }, + "required": ["query"], + }, + }, + } + ], + ) + + # Verify response structure + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + # If agentic loop worked, we should NOT have tool_calls in final response + # (they should have been executed and replaced with final answer) + if hasattr(response.choices[0].message, "tool_calls"): + # If tool_calls exist, it means agentic loop didn't run + # This could happen if search tool is not configured + pytest.skip( + "Agentic loop did not execute - search tool may not be configured" + ) + + # Verify we got a meaningful response + assert response.choices[0].finish_reason in ["stop", "end_turn"] + + finally: + # Restore original callbacks + litellm.callbacks = original_callbacks + + +@pytest.mark.asyncio +async def test_websearch_chat_completion_hook_detection(): + """Test that websearch hook correctly detects tool calls in response.""" + from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ) + + websearch_logger = WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.OPENAI] + ) + + # Mock response with litellm_web_search tool call + mock_response = ModelResponse( + id="test-123", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_123", + type="function", + function=Function( + name="litellm_web_search", + arguments='{"query": "weather in SF"}', + ), + ) + ], + ) + ) + ], + model="gpt-4o", + object="chat.completion", + created=1234567890, + ) + + # Test should_run_chat_completion_agentic_loop + should_run, tools_dict = ( + await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[ + { + "type": "function", + "function": {"name": "litellm_web_search"}, + } + ], + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + ) + + # Verify hook detected the tool call + assert should_run is True + assert "tool_calls" in tools_dict + assert len(tools_dict["tool_calls"]) == 1 + assert tools_dict["tool_calls"][0]["name"] == "litellm_web_search" + assert tools_dict["response_format"] == "openai" + + +@pytest.mark.asyncio +async def test_websearch_not_triggered_without_tool(): + """Test that websearch hook is NOT triggered when no web search tool in request.""" + from litellm.types.utils import Choices, Message + + websearch_logger = WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.OPENAI] + ) + + mock_response = ModelResponse( + id="test-123", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + role="assistant", + content="Here's the answer", + tool_calls=None, + ) + ) + ], + model="gpt-4o", + object="chat.completion", + created=1234567890, + ) + + # Test without web search tool + should_run, tools_dict = ( + await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "Hello"}], + tools=[ + { + "type": "function", + "function": {"name": "some_other_tool"}, + } + ], + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + ) + + # Verify hook did NOT trigger + assert should_run is False + assert tools_dict == {} + + +@pytest.mark.asyncio +async def test_websearch_not_triggered_for_disabled_provider(): + """Test that websearch hook is NOT triggered for providers not in enabled_providers.""" + from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ) + + # Only enable bedrock + websearch_logger = WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.BEDROCK] + ) + + mock_response = ModelResponse( + id="test-123", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_123", + type="function", + function=Function( + name="litellm_web_search", + arguments='{"query": "test"}', + ), + ) + ], + ) + ) + ], + model="gpt-4o", + object="chat.completion", + created=1234567890, + ) + + # Test with OpenAI provider (not enabled) + should_run, tools_dict = ( + await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "test"}], + tools=[ + { + "type": "function", + "function": {"name": "litellm_web_search"}, + } + ], + stream=False, + custom_llm_provider="openai", # Not in enabled_providers + kwargs={}, + ) + ) + + # Verify hook did NOT trigger + assert should_run is False + assert tools_dict == {} + + +@pytest.mark.asyncio +async def test_websearch_json_serialization_fix(): + """Test that tool call arguments are properly JSON serialized. + + Regression test for the bug where arguments were converted to Python + string representation instead of proper JSON, causing providers like + MiniMax to reject requests with 'invalid function arguments json string'. + """ + from litellm.integrations.websearch_interception.transformation import ( + WebSearchTransformation, + ) + + # Mock tool calls with dict input + tool_calls = [ + { + "id": "call_123", + "name": "litellm_web_search", + "input": {"query": "weather in SF"}, # Dict input + } + ] + + search_results = ["Weather: 65°F, partly cloudy"] + + # Transform to OpenAI format + assistant_message, tool_messages = WebSearchTransformation.transform_response( + tool_calls=tool_calls, + search_results=search_results, + response_format="openai", + ) + + # Verify arguments are properly JSON serialized + import json + + arguments_str = assistant_message["tool_calls"][0]["function"]["arguments"] + + # Should be valid JSON + parsed_args = json.loads(arguments_str) + assert parsed_args == {"query": "weather in SF"} + + # Should NOT be Python string representation like "{'query': 'weather in SF'}" + assert arguments_str == '{"query": "weather in SF"}' + assert arguments_str != "{'query': 'weather in SF'}" + + +@pytest.mark.asyncio +@pytest.mark.skipif( + os.environ.get("OPENAI_API_KEY") is None + or os.environ.get("PERPLEXITY_API_KEY") is None, + reason="OPENAI_API_KEY or PERPLEXITY_API_KEY not set", +) +async def test_websearch_streaming_conversion(): + """Test that streaming requests are converted to non-streaming for web search. + + When stream=True is passed with web search tools, the handler should: + 1. Convert stream=True to stream=False for initial request + 2. Execute web search + 3. Convert final response back to streaming + """ + websearch_logger = WebSearchInterceptionLogger( + enabled_providers=[LlmProviders.OPENAI], search_tool_name="perplexity-search" + ) + litellm.callbacks = [websearch_logger] + + try: + response = await litellm.acompletion( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": "What's the latest AI news?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ], + stream=True, + ) + + # Response should be a streaming iterator + chunks = [] + async for chunk in response: + chunks.append(chunk) + + # Verify we got streaming chunks + assert len(chunks) > 0 + + # Verify chunks have expected structure + for chunk in chunks: + assert hasattr(chunk, "choices") + assert len(chunk.choices) > 0 + + finally: + litellm.callbacks = [] + + +if __name__ == "__main__": + # Run with: pytest test_websearch_chat_completion.py -v -s + pytest.main([__file__, "-v", "-s"]) From 7fa4d090ece9278eb4247bbbb896cabacb93eeac Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 13:51:26 +0530 Subject: [PATCH 040/117] Add doc for chat completion web search --- .../integrations/websearch_interception.md | 411 ++++++++++++++++++ docs/my-website/sidebars.js | 5 + 2 files changed, 416 insertions(+) create mode 100644 docs/my-website/docs/integrations/websearch_interception.md diff --git a/docs/my-website/docs/integrations/websearch_interception.md b/docs/my-website/docs/integrations/websearch_interception.md new file mode 100644 index 00000000000..0c5d8927013 --- /dev/null +++ b/docs/my-website/docs/integrations/websearch_interception.md @@ -0,0 +1,411 @@ +# Web Search Integration + +Enable transparent server-side web search execution for any LLM provider. LiteLLM automatically intercepts web search tool calls and executes them using your configured search provider (Perplexity, Tavily, etc.). + +## Quick Start + +### 1. Configure Web Search Interception + +Add to your `config.yaml`: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + callbacks: + - websearch_interception: + enabled_providers: + - openai + - minimax + - anthropic + search_tool_name: perplexity-search # Optional + +search_tools: + - search_tool_name: perplexity-search + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY +``` + +### 2. Use with Any Provider + +```python +import litellm + +response = await litellm.acompletion( + model="gpt-4o", + messages=[ + {"role": "user", "content": "What's the weather in San Francisco today?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"] + } + } + } + ] +) + +# Response includes search results automatically! +print(response.choices[0].message.content) +``` + +## How It Works + +When a model makes a web search tool call, LiteLLM: + +1. **Detects** the `litellm_web_search` tool call in the response +2. **Executes** the search using your configured search provider +3. **Makes a follow-up request** with the search results +4. **Returns** the final answer to the user + +```mermaid +sequenceDiagram + participant User + participant LiteLLM + participant LLM as LLM Provider + participant Search as Search Provider + + User->>LiteLLM: Request with web_search tool + LiteLLM->>LLM: Forward request + LLM-->>LiteLLM: Response with tool_call + Note over LiteLLM: Detect web search
tool call + LiteLLM->>Search: Execute search + Search-->>LiteLLM: Search results + LiteLLM->>LLM: Follow-up with results + LLM-->>LiteLLM: Final answer + LiteLLM-->>User: Final answer with search results +``` + +**Result**: One API call from user → Complete answer with search results + +## Supported Providers + +Web search integration works with **all providers** that use: +- ✅ **Base HTTP Handler** (`BaseLLMHTTPHandler`) +- ✅ **OpenAI Completion Handler** (`OpenAIChatCompletion`) + +### Providers Using Base HTTP Handler + +| Provider | Status | Notes | +|----------|--------|-------| +| **OpenAI** | ✅ Supported | GPT-4, GPT-3.5, etc. | +| **Anthropic** | ✅ Supported | Claude models via HTTP handler | +| **MiniMax** | ✅ Supported | All MiniMax models | +| **Mistral** | ✅ Supported | Mistral AI models | +| **Cohere** | ✅ Supported | Command models | +| **Fireworks AI** | ✅ Supported | All Fireworks models | +| **Together AI** | ✅ Supported | All Together AI models | +| **Groq** | ✅ Supported | All Groq models | +| **Perplexity** | ✅ Supported | Perplexity models | +| **DeepSeek** | ✅ Supported | DeepSeek models | +| **xAI** | ✅ Supported | Grok models | +| **Hugging Face** | ✅ Supported | Inference API models | +| **OCI** | ✅ Supported | Oracle Cloud models | +| **Vertex AI** | ✅ Supported | Google Vertex AI models | +| **Bedrock** | ✅ Supported | AWS Bedrock models (converse_like route) | +| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI models | +| **Sagemaker** | ✅ Supported | AWS Sagemaker models | +| **Databricks** | ✅ Supported | Databricks models | +| **DataRobot** | ✅ Supported | DataRobot models | +| **Hosted VLLM** | ✅ Supported | Self-hosted VLLM | +| **Heroku** | ✅ Supported | Heroku-hosted models | +| **RAGFlow** | ✅ Supported | RAGFlow models | +| **Compactif** | ✅ Supported | Compactif models | +| **Cometapi** | ✅ Supported | Comet API models | +| **A2A** | ✅ Supported | Agent-to-Agent models | +| **Bytez** | ✅ Supported | Bytez models | + +### Providers Using OpenAI Handler + +| Provider | Status | Notes | +|----------|--------|-------| +| **OpenAI** | ✅ Supported | Native OpenAI API | +| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI | +| **OpenAI-Compatible** | ✅ Supported | Any OpenAI-compatible API | + +## Configuration + +### WebSearch Interception Parameters + +| Parameter | Type | Required | Description | Example | +|-----------|------|----------|-------------|---------| +| `enabled_providers` | List[String] | Yes | List of providers to enable web search for | `[openai, minimax, anthropic]` | +| `search_tool_name` | String | No | Specific search tool from `search_tools` config. If not set, uses first available. | `perplexity-search` | + +### Provider Values + +Use these values in `enabled_providers`: + +| Provider | Value | Provider | Value | +|----------|-------|----------|-------| +| OpenAI | `openai` | Anthropic | `anthropic` | +| MiniMax | `minimax` | Mistral | `mistral` | +| Cohere | `cohere` | Fireworks AI | `fireworks_ai` | +| Together AI | `together_ai` | Groq | `groq` | +| Perplexity | `perplexity` | DeepSeek | `deepseek` | +| xAI | `xai` | Hugging Face | `huggingface` | +| OCI | `oci` | Vertex AI | `vertex_ai` | +| Bedrock | `bedrock` | Azure | `azure` | +| Sagemaker | `sagemaker_chat` | Databricks | `databricks` | +| DataRobot | `datarobot` | VLLM | `hosted_vllm` | +| Heroku | `heroku` | RAGFlow | `ragflow` | +| Compactif | `compactif` | Cometapi | `cometapi` | +| A2A | `a2a` | Bytez | `bytez` | + +## Search Providers + +Configure which search provider to use. LiteLLM supports multiple search providers: + +| Provider | `search_provider` Value | Environment Variable | +|----------|------------------------|----------------------| +| **Perplexity AI** | `perplexity` | `PERPLEXITYAI_API_KEY` | +| **Tavily** | `tavily` | `TAVILY_API_KEY` | +| **Exa AI** | `exa_ai` | `EXA_API_KEY` | +| **Parallel AI** | `parallel_ai` | `PARALLEL_AI_API_KEY` | +| **Google PSE** | `google_pse` | `GOOGLE_PSE_API_KEY`, `GOOGLE_PSE_ENGINE_ID` | +| **DataForSEO** | `dataforseo` | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` | +| **Firecrawl** | `firecrawl` | `FIRECRAWL_API_KEY` | +| **SearXNG** | `searxng` | `SEARXNG_API_BASE` (required) | +| **Linkup** | `linkup` | `LINKUP_API_KEY` | + +See [Search Providers Documentation](../search/index.md) for detailed setup instructions. + +## Complete Configuration Example + +```yaml +model_list: + # OpenAI + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + # MiniMax + - model_name: minimax + litellm_params: + model: minimax/MiniMax-M2.1 + api_key: os.environ/MINIMAX_API_KEY + + # Anthropic + - model_name: claude + litellm_params: + model: anthropic/claude-sonnet-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + + # Azure OpenAI + - model_name: azure-gpt4 + litellm_params: + model: azure/gpt-4 + api_base: https://my-azure.openai.azure.com + api_key: os.environ/AZURE_API_KEY + +litellm_settings: + callbacks: + - websearch_interception: + enabled_providers: + - openai + - minimax + - anthropic + - azure + search_tool_name: perplexity-search + +search_tools: + - search_tool_name: perplexity-search + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY + + - search_tool_name: tavily-search + litellm_params: + search_provider: tavily + api_key: os.environ/TAVILY_API_KEY +``` + +## Usage Examples + +### Python SDK + +```python +import litellm + +# Configure callbacks +litellm.callbacks = ["websearch_interception"] + +# Make completion with web search tool +response = await litellm.acompletion( + model="gpt-4o", + messages=[ + {"role": "user", "content": "What are the latest AI news?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web for current information", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query" + } + }, + "required": ["query"] + } + } + } + ] +) + +print(response.choices[0].message.content) +``` + +### Proxy Server + +```bash +# Start proxy with config +litellm --config config.yaml + +# Make request +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "What is the weather in San Francisco?"} + ], + "tools": [ + { + "type": "function", + "function": { + "name": "litellm_web_search", + "description": "Search the web", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + } + } + ] + }' +``` + +## How Search Tool Selection Works + +1. **If `search_tool_name` is specified** → Uses that specific search tool +2. **If `search_tool_name` is not specified** → Uses first search tool in `search_tools` list + +```yaml +search_tools: + - search_tool_name: perplexity-search # ← This will be used if no search_tool_name specified + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY + + - search_tool_name: tavily-search + litellm_params: + search_provider: tavily + api_key: os.environ/TAVILY_API_KEY +``` + +## Troubleshooting + +### Web Search Not Working + +1. **Check provider is enabled**: + ```yaml + enabled_providers: + - openai # Make sure your provider is in this list + ``` + +2. **Verify search tool is configured**: + ```yaml + search_tools: + - search_tool_name: perplexity-search + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY + ``` + +3. **Check API keys are set**: + ```bash + export PERPLEXITY_API_KEY=your-key + ``` + +4. **Enable debug logging**: + ```python + litellm.set_verbose = True + ``` + +### Common Issues + +**Issue**: Model returns tool_calls instead of final answer +- **Cause**: Provider not in `enabled_providers` list +- **Solution**: Add provider to `enabled_providers` + +**Issue**: "No search tool configured" error +- **Cause**: No search tools in `search_tools` config +- **Solution**: Add at least one search tool configuration + +**Issue**: "Invalid function arguments json string" error (MiniMax) +- **Cause**: Fixed in latest version - arguments weren't properly JSON serialized +- **Solution**: Update to latest LiteLLM version + +## Related Documentation + +- [Search Providers](../search/index.md) - Detailed search provider setup +- [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code +- [Tool Calling](../completion/function_call.md) - General tool calling documentation +- [Callbacks](./custom_callback.md) - Custom callback documentation + +## Technical Details + +### Architecture + +Web search integration is implemented as a custom callback (`WebSearchInterceptionLogger`) that: + +1. **Pre-request Hook**: Converts native web search tools to LiteLLM standard format +2. **Post-response Hook**: Detects web search tool calls in responses +3. **Agentic Loop**: Executes searches and makes follow-up requests automatically + +### Supported APIs + +- ✅ **Chat Completions API** (OpenAI format) +- ✅ **Anthropic Messages API** (Anthropic format) +- ✅ **Streaming** (automatically converted) +- ✅ **Non-streaming** + +### Response Format Detection + +The handler automatically detects response format: +- **OpenAI format**: `tool_calls` in assistant message +- **Anthropic format**: `tool_use` blocks in content + +### Performance + +- **Latency**: Adds one additional LLM call (follow-up request with search results) +- **Caching**: Search results can be cached (depends on search provider) +- **Parallel Searches**: Multiple search queries executed in parallel + +## Contributing + +Found a bug or want to add support for a new provider? See our [Contributing Guide](https://github.com/BerriAI/litellm/blob/main/CONTRIBUTING.md). diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index fda0e3be4e4..9bedb228393 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -96,6 +96,11 @@ const sidebars = { "proxy/prometheus" ] }, + { + type: "doc", + id: "integrations/websearch_interception", + label: "Web Search Integration" + }, { type: "category", label: "[Beta] Prompt Management", From 1f04115fb0ff939f42952d631504ac9aa96b5811 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 13:59:05 +0530 Subject: [PATCH 041/117] Fix: is_web_search_tool_chat_completion --- .../websearch_interception/handler.py | 7 +-- .../websearch_interception/tools.py | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 1e109dc9e39..7e8fa66c493 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -17,6 +17,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.tools import ( get_litellm_web_search_tool, is_web_search_tool, + is_web_search_tool_chat_completion, ) from litellm.integrations.websearch_interception.transformation import ( WebSearchTransformation, @@ -325,11 +326,11 @@ class WebSearchInterceptionLogger(CustomLogger): ) return False, {} - # Check if tools include any web search tool - has_websearch_tool = any(is_web_search_tool(t) for t in (tools or [])) + # Check if tools include any web search tool (strict check for chat completions) + has_websearch_tool = any(is_web_search_tool_chat_completion(t) for t in (tools or [])) if not has_websearch_tool: verbose_logger.debug( - "WebSearchInterception: No web search tool in request" + "WebSearchInterception: No litellm_web_search tool in request" ) return False, {} diff --git a/litellm/integrations/websearch_interception/tools.py b/litellm/integrations/websearch_interception/tools.py index be8808622da..c39d150fb19 100644 --- a/litellm/integrations/websearch_interception/tools.py +++ b/litellm/integrations/websearch_interception/tools.py @@ -49,6 +49,50 @@ def get_litellm_web_search_tool() -> Dict[str, Any]: } +def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool: + """ + Check if a tool is a web search tool for Chat Completions API (strict check). + + This is a stricter version that ONLY checks for the exact LiteLLM web search tool name. + Use this for Chat Completions API to avoid false positives with user-defined tools. + + Detects ONLY: + - LiteLLM standard: name == "litellm_web_search" (Anthropic format) + - OpenAI format: type == "function" with function.name == "litellm_web_search" + + Args: + tool: Tool dictionary to check + + Returns: + True if tool is exactly the LiteLLM web search tool + + Example: + >>> is_web_search_tool_chat_completion({"name": "litellm_web_search"}) + True + >>> is_web_search_tool_chat_completion({"type": "function", "function": {"name": "litellm_web_search"}}) + True + >>> is_web_search_tool_chat_completion({"name": "web_search"}) + False + >>> is_web_search_tool_chat_completion({"name": "WebSearch"}) + False + """ + tool_name = tool.get("name", "") + tool_type = tool.get("type", "") + + # Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}} + if tool_type == "function" and "function" in tool: + function_def = tool.get("function", {}) + function_name = function_def.get("name", "") + if function_name == LITELLM_WEB_SEARCH_TOOL_NAME: + return True + + # Check for LiteLLM standard tool (Anthropic format) + if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME: + return True + + return False + + def is_web_search_tool(tool: Dict[str, Any]) -> bool: """ Check if a tool is a web search tool (native or LiteLLM standard). From c48986ba8d2d845b782e8ed923a6405263d2b9ee Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 14:21:43 +0530 Subject: [PATCH 042/117] Fix double json import --- litellm/integrations/websearch_interception/transformation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 3201d99d69a..ad92f5d03d0 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -190,7 +190,6 @@ class WebSearchTransformation: LITELLM_WEB_SEARCH_TOOL_NAME, "WebSearch", "web_search" ): # Parse arguments (might be JSON string) - import json if isinstance(function_arguments, str): try: arguments = json.loads(function_arguments) From 6c4d6bb15e67a8d05a78376e020a6622ef85dc5c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 16:00:01 +0530 Subject: [PATCH 043/117] Add new vercel ai anthropic models --- ...odel_prices_and_context_window_backup.json | 187 ++++++++++++++++++ model_prices_and_context_window.json | 187 ++++++++++++++++++ 2 files changed, 374 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5cad0db241f..259fb656457 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28567,6 +28567,193 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "vercel_ai_gateway/anthropic/claude-3-5-sonnet": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-3-5-sonnet-20241022": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-3-7-sonnet": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-haiku-4.5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "input_cost_per_token": 1.5e-05, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.1": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "input_cost_per_token": 1.5e-05, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-sonnet-4": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, "litellm_provider": "vercel_ai_gateway", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5cad0db241f..259fb656457 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28567,6 +28567,193 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "vercel_ai_gateway/anthropic/claude-3-5-sonnet": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-3-5-sonnet-20241022": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-3-7-sonnet": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-haiku-4.5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "input_cost_per_token": 1.5e-05, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.1": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "input_cost_per_token": 1.5e-05, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-sonnet-4": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "vercel_ai_gateway", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/cohere/command-a": { "input_cost_per_token": 2.5e-06, "litellm_provider": "vercel_ai_gateway", From d35691aa0ce3e40bb4555fed715d9e45ec64e361 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 16:25:46 +0530 Subject: [PATCH 044/117] Fix: base_model name for body and deplyment name in URL --- litellm/llms/azure/azure.py | 18 +- .../test_azure_image_generation_init.py | 205 ++++++++++++++++++ 2 files changed, 214 insertions(+), 9 deletions(-) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index cb9fe0aeb30..95afa53b625 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -1060,6 +1060,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): headers: dict, client=None, timeout=None, + model: Optional[str] = None, ) -> ImageResponse: response: Optional[dict] = None @@ -1071,8 +1072,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if api_base.endswith("/"): api_base = api_base.rstrip("/") api_version: str = azure_client_params.get("api_version", "") + # Use the deployment name (model) for URL construction, not the base_model from data img_gen_api_base = self.create_azure_base_url( - azure_client_params=azure_client_params, model=data.get("model", "") + azure_client_params=azure_client_params, model=model or data.get("model", "") ) ## LOGGING @@ -1159,21 +1161,18 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): model = model else: model = None - ## BASE MODEL CHECK if ( model_response is not None - and optional_params.get("base_model", None) is not None + and litellm_params.get("base_model", None) is not None ): - model_response._hidden_params["model"] = optional_params.pop( - "base_model" - ) + model_response._hidden_params["model"] = litellm_params.get("base_model", None) # Azure image generation API doesn't support extra_body parameter extra_body = optional_params.pop("extra_body", {}) flattened_params = {**optional_params, **extra_body} - data = {"model": model, "prompt": prompt, **flattened_params} + data = {"model": litellm_params.get("base_model", None) or model, "prompt": prompt, **flattened_params} max_retries = data.pop("max_retries", 2) if not isinstance(max_retries, int): raise AzureOpenAIError( @@ -1196,10 +1195,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): is_async=False, ) if aimg_generation is True: - return self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout, headers=headers) # type: ignore + return self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout, headers=headers, model=model) # type: ignore + # Use the deployment name (model) for URL construction, not the base_model from data img_gen_api_base = self.create_azure_base_url( - azure_client_params=azure_client_params, model=data.get("model", "") + azure_client_params=azure_client_params, model=model ) ## LOGGING diff --git a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py index 987eb5bf998..bf8165c6908 100644 --- a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py +++ b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py @@ -251,3 +251,208 @@ def test_azure_image_generation_drop_params_false_raises_error(): # Verify the error message mentions the unsupported parameter assert "response_format" in str(exc_info.value) + + +def test_azure_image_generation_base_model_vs_deployment_name(): + """ + Test that Azure image generation correctly uses base_model in request body + but deployment name in the URL. + + When base_model is specified in litellm_params, the request should: + 1. Use base_model (e.g., "gpt-image-1.5") in the JSON request body + 2. Use the deployment name (e.g., "gpt-image-15") in the URL path + + This is important because Azure expects: + - URL: /openai/deployments/{deployment_name}/images/generations + - Body: {"model": "{base_model}", ...} + + Example config: + model: azure/gpt-image-15 # deployment name + base_model: gpt-image-1.5 # actual model name + """ + from unittest.mock import MagicMock + + # Setup test parameters + azure_chat_completion = AzureChatCompletion() + + prompt = "A beautiful image of a cat" + model = "gpt-image-15" # This is the deployment name + base_model = "gpt-image-1.5" # This is the actual model name + api_base = "https://openai-gpt-image-1-5-test-v-1.openai.azure.com/" + api_version = "2024-07-01-preview" + api_key = "test-api-key" + + litellm_params = { + "base_model": base_model, + "api_base": api_base, + "api_version": api_version, + } + + optional_params = { + "n": 1, + "size": "1024x1024" + } + + # Mock the HTTP request to capture what gets sent + with patch.object( + azure_chat_completion, + "make_sync_azure_httpx_request", + return_value=MagicMock( + json=lambda: { + "created": 1234567890, + "data": [ + { + "url": "https://example.com/image.png", + "revised_prompt": prompt + } + ] + } + ) + ) as mock_request: + # Mock logging object + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + logging_obj.post_call = MagicMock() + + # Call the image_generation method + try: + response = azure_chat_completion.image_generation( + prompt=prompt, + timeout=60.0, + optional_params=optional_params, + logging_obj=logging_obj, + headers={}, + model=model, + api_key=api_key, + api_base=api_base, + api_version=api_version, + litellm_params=litellm_params, + ) + except Exception as e: + # If there's an error, we still want to check the mock calls + pass + + # Verify the mock was called + assert mock_request.called, "HTTP request should have been made" + + # Get the call arguments + call_kwargs = mock_request.call_args.kwargs + + # Verify the URL uses the deployment name (not base_model) + api_base_used = call_kwargs.get("api_base", "") + assert model in api_base_used, ( + f"URL should contain deployment name '{model}', " + f"but got: {api_base_used}" + ) + assert base_model not in api_base_used or base_model == model, ( + f"URL should NOT contain base_model '{base_model}' when it differs from deployment name, " + f"but got: {api_base_used}" + ) + + # Verify the request body uses base_model (not deployment name) + request_data = call_kwargs.get("data", {}) + assert request_data.get("model") == base_model, ( + f"Request body 'model' field should be base_model '{base_model}', " + f"but got: {request_data.get('model')}" + ) + + # Verify other fields are correct + assert request_data.get("prompt") == prompt + assert request_data.get("n") == 1 + assert request_data.get("size") == "1024x1024" + + +@pytest.mark.asyncio +async def test_azure_aimage_generation_base_model_vs_deployment_name(): + """ + Test that Azure async image generation correctly uses base_model in request body + but deployment name in the URL. + + This is the async version of test_azure_image_generation_base_model_vs_deployment_name. + """ + from unittest.mock import MagicMock + + # Setup test parameters + azure_chat_completion = AzureChatCompletion() + + prompt = "A beautiful image of a cat" + model = "gpt-image-15" # This is the deployment name + base_model = "gpt-image-1.5" # This is the actual model name + api_base = "https://openai-gpt-image-1-5-test-v-1.openai.azure.com/" + api_version = "2024-07-01-preview" + api_key = "test-api-key" + + data = { + "model": base_model, + "prompt": prompt, + "n": 1, + "size": "1024x1024" + } + + azure_client_params = { + "api_base": api_base, + "api_version": api_version, + } + + # Mock the HTTP request to capture what gets sent + with patch.object( + azure_chat_completion, + "make_async_azure_httpx_request", + new_callable=AsyncMock, + return_value=MagicMock( + json=lambda: { + "created": 1234567890, + "data": [ + { + "url": "https://example.com/image.png", + "revised_prompt": prompt + } + ] + } + ) + ) as mock_request: + # Mock logging object + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + logging_obj.post_call = MagicMock() + + # Call the aimage_generation method + try: + response = await azure_chat_completion.aimage_generation( + data=data, + model_response=None, + azure_client_params=azure_client_params, + api_key=api_key, + input=[], + logging_obj=logging_obj, + headers={}, + model=model, # Pass the deployment name + timeout=60.0, + ) + except Exception as e: + # If there's an error, we still want to check the mock calls + pass + + # Verify the mock was called + assert mock_request.called, "HTTP request should have been made" + + # Get the call arguments + call_kwargs = mock_request.call_args.kwargs + + # Verify the URL uses the deployment name (not base_model) + api_base_used = call_kwargs.get("api_base", "") + assert model in api_base_used, ( + f"URL should contain deployment name '{model}', " + f"but got: {api_base_used}" + ) + assert base_model not in api_base_used or base_model == model, ( + f"URL should NOT contain base_model '{base_model}' when it differs from deployment name, " + f"but got: {api_base_used}" + ) + + # Verify the request body uses base_model (not deployment name) + request_data = call_kwargs.get("data", {}) + assert request_data.get("model") == base_model, ( + f"Request body 'model' field should be base_model '{base_model}', " + f"but got: {request_data.get('model')}" + ) From 1b2278951d04aeef568714571fba69de56785f6a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 16:39:21 +0530 Subject: [PATCH 045/117] Add output_config as supported param --- .../experimental_pass_through/messages/transformation.py | 1 + litellm/types/llms/anthropic.py | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index bb40f9df266..a48d1622155 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -46,6 +46,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "thinking", "context_management", "output_format", + "output_config", # TODO: Add Anthropic `metadata` support # "metadata", ] diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index fedf419efd6..6c47a544739 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -360,6 +360,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): context_management: Optional[Dict[str, Any]] container: Optional[Dict[str, Any]] # Container config with skills for code execution output_format: Optional[AnthropicOutputSchema] # Structured outputs support + output_config: Optional[AnthropicOutputConfig] # Configuration for Claude's output behavior class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): From 23088f86bd19c7b868a0e90a522a54438aca59a9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:07:36 +0530 Subject: [PATCH 046/117] Add response schema for vercel ai sonnet 4.5 --- litellm/model_prices_and_context_window_backup.json | 3 ++- model_prices_and_context_window.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 259fb656457..6cc7f737ce4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28763,7 +28763,8 @@ "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "vercel_ai_gateway/cohere/command-r": { "input_cost_per_token": 1.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 259fb656457..6cc7f737ce4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28763,7 +28763,8 @@ "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true }, "vercel_ai_gateway/cohere/command-r": { "input_cost_per_token": 1.5e-07, From 30d17c29e4bcff4c5843fd1befb2dfb9d3832fa0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:13:39 +0530 Subject: [PATCH 047/117] handle when litellm_parrams might be none --- litellm/llms/azure/azure.py | 4 +- .../test_azure_image_generation_init.py | 54 ++++++++----------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 95afa53b625..76fa713ca8c 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -1164,6 +1164,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): ## BASE MODEL CHECK if ( model_response is not None + and litellm_params is not None and litellm_params.get("base_model", None) is not None ): model_response._hidden_params["model"] = litellm_params.get("base_model", None) @@ -1172,7 +1173,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): extra_body = optional_params.pop("extra_body", {}) flattened_params = {**optional_params, **extra_body} - data = {"model": litellm_params.get("base_model", None) or model, "prompt": prompt, **flattened_params} + base_model = litellm_params.get("base_model", None) if litellm_params else None + data = {"model": base_model or model, "prompt": prompt, **flattened_params} max_retries = data.pop("max_retries", 2) if not isinstance(max_retries, int): raise AzureOpenAIError( diff --git a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py index bf8165c6908..44bcc9f954a 100644 --- a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py +++ b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py @@ -315,22 +315,18 @@ def test_azure_image_generation_base_model_vs_deployment_name(): logging_obj.post_call = MagicMock() # Call the image_generation method - try: - response = azure_chat_completion.image_generation( - prompt=prompt, - timeout=60.0, - optional_params=optional_params, - logging_obj=logging_obj, - headers={}, - model=model, - api_key=api_key, - api_base=api_base, - api_version=api_version, - litellm_params=litellm_params, - ) - except Exception as e: - # If there's an error, we still want to check the mock calls - pass + response = azure_chat_completion.image_generation( + prompt=prompt, + timeout=60.0, + optional_params=optional_params, + logging_obj=logging_obj, + headers={}, + model=model, + api_key=api_key, + api_base=api_base, + api_version=api_version, + litellm_params=litellm_params, + ) # Verify the mock was called assert mock_request.called, "HTTP request should have been made" @@ -417,21 +413,17 @@ async def test_azure_aimage_generation_base_model_vs_deployment_name(): logging_obj.post_call = MagicMock() # Call the aimage_generation method - try: - response = await azure_chat_completion.aimage_generation( - data=data, - model_response=None, - azure_client_params=azure_client_params, - api_key=api_key, - input=[], - logging_obj=logging_obj, - headers={}, - model=model, # Pass the deployment name - timeout=60.0, - ) - except Exception as e: - # If there's an error, we still want to check the mock calls - pass + response = await azure_chat_completion.aimage_generation( + data=data, + model_response=None, + azure_client_params=azure_client_params, + api_key=api_key, + input=[], + logging_obj=logging_obj, + headers={}, + model=model, # Pass the deployment name + timeout=60.0, + ) # Verify the mock was called assert mock_request.called, "HTTP request should have been made" From 56119742285265889eaf785ab688e265aeac8a76 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:17:07 +0530 Subject: [PATCH 048/117] Fix : litellm/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py --- litellm/anthropic_beta_headers_config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 1e99e35aabe..4ebb5ddb609 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -24,7 +24,8 @@ "code-execution-2025-08-25", "skills-2025-10-02", "files-api-2025-04-14", - "fast-mode-2026-02-01" + "fast-mode-2026-02-01", + "mcp-servers-2025-12-04" ], "vertex_ai": [ "prompt-caching-scope-2026-01-05" From 0b5cb47c03ab590eb5f80e35695bc2071af318a7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:34:11 +0530 Subject: [PATCH 049/117] fix: Missing return statement for async streaming --- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3907ff7abf7..95db8ec64b3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -438,7 +438,7 @@ class BaseLLMHTTPHandler: provider_config=provider_config, fake_stream=fake_stream, ) - response = self.acompletion_stream_function( + return self.acompletion_stream_function( model=model, messages=messages, api_base=api_base, From 5702cc7e13b8cd8122cb570c24b1adeae02c3890 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:38:03 +0530 Subject: [PATCH 050/117] Fix: get_supported_anthropic_messages_params --- .../experimental_pass_through/messages/transformation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 7d93e184090..043a70f3c67 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -46,12 +46,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "thinking", "context_management", "output_format", -<<<<<<< litellm_v1_messages_claude_4_6 "inference_geo", "speed", -======= "output_config", ->>>>>>> main # TODO: Add Anthropic `metadata` support # "metadata", ] From 125e11d36e46de24081da3d271a3fa8290d021cb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:43:52 +0530 Subject: [PATCH 051/117] Fix mypy issues --- litellm/integrations/websearch_interception/handler.py | 7 ++++--- .../integrations/websearch_interception/transformation.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 7e8fa66c493..82d91d811ef 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -490,7 +490,8 @@ class WebSearchInterceptionLogger(CustomLogger): ) # Make follow-up request with search results - follow_up_messages = messages + [assistant_message, user_message] + # Type cast: user_message is a Dict for Anthropic format (default response_format) + follow_up_messages = messages + [assistant_message, cast(Dict, user_message)] verbose_logger.debug( "WebSearchInterception: Making follow-up request with search results" @@ -702,10 +703,10 @@ class WebSearchInterceptionLogger(CustomLogger): # Make follow-up request with search results # For OpenAI format, tool_messages_or_user is a list of tool messages if response_format == "openai": - follow_up_messages = messages + [assistant_message] + tool_messages_or_user + follow_up_messages = messages + [assistant_message] + cast(List[Dict], tool_messages_or_user) else: # For Anthropic format (shouldn't happen in this method, but handle it) - follow_up_messages = messages + [assistant_message, tool_messages_or_user] + follow_up_messages = messages + [assistant_message, cast(Dict, tool_messages_or_user)] verbose_logger.debug( "WebSearchInterception: Making follow-up chat completion request with search results" diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index ad92f5d03d0..e44ec35c3a2 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -4,7 +4,7 @@ WebSearch Tool Transformation Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. """ import json -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Union from litellm._logging import verbose_logger from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME @@ -224,7 +224,7 @@ class WebSearchTransformation: tool_calls: List[Dict], search_results: List[str], response_format: str = "anthropic", - ) -> Tuple[Dict, Dict]: + ) -> Tuple[Dict, Union[Dict, List[Dict]]]: """ Transform LiteLLM search results to Anthropic/OpenAI tool_result format. From 2d18ae4f9e92478d627973650dd95f69adec351e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 9 Feb 2026 17:44:39 +0530 Subject: [PATCH 052/117] Fix mypy issues --- litellm/integrations/websearch_interception/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 82d91d811ef..1277cac51d7 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -630,7 +630,7 @@ class WebSearchInterceptionLogger(CustomLogger): ) raise - async def _execute_chat_completion_agentic_loop( + async def _execute_chat_completion_agentic_loop( # noqa: PLR0915 self, model: str, messages: List[Dict], From 9532ad0fab15087a883a89cee1c6a264be6e2100 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Feb 2026 10:03:43 -0800 Subject: [PATCH 053/117] docs fix (#20768) --- .../docs/proxy/forward_client_headers.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/my-website/docs/proxy/forward_client_headers.md b/docs/my-website/docs/proxy/forward_client_headers.md index 5477ffe87aa..2155a7517be 100644 --- a/docs/my-website/docs/proxy/forward_client_headers.md +++ b/docs/my-website/docs/proxy/forward_client_headers.md @@ -6,6 +6,52 @@ Control which model groups can forward client headers to the underlying LLM prov By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting. +## How it Works + +LiteLLM does **not** forward all client headers to the LLM provider. Instead, it uses an **allowlist** approach — only headers matching specific rules are forwarded. This ensures sensitive headers (like your LiteLLM API key) are never accidentally sent to upstream providers. + +```mermaid +sequenceDiagram + participant Client as Client (SDK / curl) + participant Proxy as LiteLLM Proxy + participant Filter as Header Filter (Allowlist) + participant LLM as LLM Provider (OpenAI, Anthropic, etc.) + + Client->>Proxy: Request with all headers
(Authorization, x-trace-id,
x-custom-header, anthropic-beta, etc.) + + Proxy->>Filter: Check forward_client_headers_to_llm_api
setting for this model group + + Note over Filter: Allowlist rules:
1. Headers starting with "x-" ✅
2. "anthropic-beta" ✅
3. "x-stainless-*" ❌ (blocked)
4. All other headers ❌ (blocked) + + Filter-->>Proxy: Return only allowed headers + + Proxy->>LLM: Request with filtered headers
(x-trace-id, x-custom-header,
anthropic-beta) + + LLM-->>Proxy: Response + Proxy-->>Client: Response +``` + +### Header Allowlist Rules + +The following rules determine which headers are forwarded (see [`_get_forwardable_headers`](https://github.com/litellm/litellm/blob/main/litellm/proxy/litellm_pre_call_utils.py) in `litellm/proxy/litellm_pre_call_utils.py`): + +| Rule | Example | Forwarded? | +|---|---|---| +| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes | +| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes | +| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) | +| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No | +| Other provider headers | `Accept`, `User-Agent` | ❌ No | + +### Additional Header Mechanisms + +| Mechanism | Description | Reference | +|---|---|---| +| **`x-pass-` prefix** | Headers prefixed with `x-pass-` are always forwarded with the prefix stripped, regardless of settings. E.g., `x-pass-anthropic-beta: value` → `anthropic-beta: value`. Works for all pass-through endpoints. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/passthrough/utils.py) | +| **`openai-organization`** | Forwarded only when `forward_openai_org_id: true` is set in `general_settings`. | [Forward OpenAI Org ID](#enable-globally) | +| **User information headers** | When `add_user_information_to_llm_headers: true`, LiteLLM adds `x-litellm-user-id`, `x-litellm-org-id`, etc. | [User Information Headers](#user-information-headers-optional) | +| **Vertex AI pass-through** | Uses a separate, stricter allowlist: only `anthropic-beta` and `content-type`. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/constants.py) | + ## Configuration ## Enable Globally From f2ba120c4391e3e4aa799914fea011b3ac5912be Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 9 Feb 2026 10:59:57 -0800 Subject: [PATCH 054/117] docs fix --- docs/my-website/release_notes/v1.81.3-stable/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/release_notes/v1.81.3-stable/index.md b/docs/my-website/release_notes/v1.81.3-stable/index.md index 22b6f43deef..c4b9013590c 100644 --- a/docs/my-website/release_notes/v1.81.3-stable/index.md +++ b/docs/my-website/release_notes/v1.81.3-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -docker.litellm.ai/berriai/litellm:v1.81.3.rc.2 +docker.litellm.ai/berriai/litellm:v1.81.3-stable ``` From a7ed3f240ccdb00f3b5ddb5fa1bab8aacb020208 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Feb 2026 11:08:28 -0800 Subject: [PATCH 055/117] Show predefined error codes in UI with user adjustable fallback --- .../src/components/molecules/filter.test.tsx | 408 +++++++++++++++++- .../src/components/molecules/filter.tsx | 2 +- .../src/components/view_logs/constants.ts | 21 + .../src/components/view_logs/index.tsx | 40 +- 4 files changed, 437 insertions(+), 34 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/constants.ts diff --git a/ui/litellm-dashboard/src/components/molecules/filter.test.tsx b/ui/litellm-dashboard/src/components/molecules/filter.test.tsx index bd06d110e95..1a90c4a069f 100644 --- a/ui/litellm-dashboard/src/components/molecules/filter.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/filter.test.tsx @@ -1,4 +1,4 @@ -import { screen, waitFor } from "@testing-library/react"; +import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; @@ -58,8 +58,34 @@ describe("FilterComponent", () => { expect(screen.getByRole("button", { name: "Custom Filters" })).toBeInTheDocument(); }); + it("should toggle filters visibility when filter button is clicked", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + expect(screen.queryByPlaceholderText("Enter User ID...")).not.toBeInTheDocument(); + + await user.click(filterButton); + + await waitFor(() => { + expect(screen.getByPlaceholderText("Enter User ID...")).toBeInTheDocument(); + }); + + await user.click(filterButton); + + await waitFor(() => { + expect(screen.queryByPlaceholderText("Enter User ID...")).not.toBeInTheDocument(); + }); + }); + it("should call onResetFilters when reset button is clicked", async () => { - const user = userEvent.setup(); + const user = userEvent.setup({ delay: null }); renderWithProviders( { }); it("should render filters in correct order", async () => { - const user = userEvent.setup(); + const user = userEvent.setup({ delay: null }); const options: FilterOption[] = [ { name: "model", label: "Model" }, { name: "teamId", label: "Team ID" }, @@ -105,7 +131,7 @@ describe("FilterComponent", () => { }); it("should handle input filter changes", async () => { - const user = userEvent.setup(); + const user = userEvent.setup({ delay: null }); renderWithProviders( { expect(mockOnApplyFilters).toHaveBeenCalledWith({ userId: "user123" }); }); }); + + it("should display initial values in filters", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + const userIdInput = screen.getByPlaceholderText("Enter User ID...") as HTMLInputElement; + expect(userIdInput.value).toBe("user123"); + }); + }); + + it("should handle select dropdown filter changes", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + const teamIdLabel = screen.getByText("Team ID"); + const teamIdSection = teamIdLabel.closest("div"); + const teamIdSelect = within(teamIdSection!).getByRole("combobox"); + + await user.click(teamIdSelect); + + await waitFor(() => { + expect(screen.getByText("Team 1")).toBeInTheDocument(); + }); + + await user.click(screen.getByText("Team 1")); + + await waitFor(() => { + expect(mockOnApplyFilters).toHaveBeenCalledWith({ teamId: "team1" }); + }); + }); + + it("should handle searchable filter with search function", async () => { + const user = userEvent.setup({ delay: null }); + const mockSearchFn = vi.fn().mockResolvedValue([ + { label: "Result 1", value: "result1" }, + { label: "Result 2", value: "result2" }, + ]); + + const options: FilterOption[] = [ + { + name: "model", + label: "Model", + isSearchable: true, + searchFn: mockSearchFn, + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(mockSearchFn).toHaveBeenCalledWith(""); + }); + + const modelLabel = screen.getByText("Model"); + const modelSection = modelLabel.closest("div"); + const modelSelect = within(modelSection!).getByRole("combobox"); + await user.click(modelSelect); + + await waitFor(() => { + expect(screen.getByText("Result 1")).toBeInTheDocument(); + expect(screen.getByText("Result 2")).toBeInTheDocument(); + }); + }); + + it("should debounce search input for searchable filters", async () => { + const user = userEvent.setup({ delay: null }); + const mockSearchFn = vi.fn().mockResolvedValue([ + { label: "Result", value: "result" }, + ]); + + const options: FilterOption[] = [ + { + name: "model", + label: "Model", + isSearchable: true, + searchFn: mockSearchFn, + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(mockSearchFn).toHaveBeenCalledWith(""); + }); + + vi.clearAllMocks(); + + const modelLabel = screen.getByText("Model"); + const modelSection = modelLabel.closest("div"); + const modelSelect = within(modelSection!).getByRole("combobox"); + await user.click(modelSelect); + await user.type(modelSelect, "test"); + + expect(mockSearchFn).not.toHaveBeenCalled(); + + await waitFor( + () => { + expect(mockSearchFn).toHaveBeenCalledWith("test"); + }, + { timeout: 500 }, + ); + }); + + it("should show loading state when searching", async () => { + const user = userEvent.setup({ delay: null }); + let resolveSearch: (value: Array<{ label: string; value: string }>) => void; + const mockSearchFn = vi.fn().mockImplementation( + () => + new Promise>((resolve) => { + resolveSearch = resolve; + }), + ); + + const options: FilterOption[] = [ + { + name: "model", + label: "Model", + isSearchable: true, + searchFn: mockSearchFn, + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(mockSearchFn).toHaveBeenCalledWith(""); + }); + + const modelLabel = screen.getByText("Model"); + const modelSection = modelLabel.closest("div"); + const modelSelect = within(modelSection!).getByRole("combobox"); + await user.click(modelSelect); + await user.type(modelSelect, "test"); + + await waitFor( + () => { + expect(screen.getByText("Loading...")).toBeInTheDocument(); + }, + { timeout: 500 }, + ); + + resolveSearch!([{ label: "Result", value: "result" }]); + + await waitFor(() => { + expect(screen.queryByText("Loading...")).not.toBeInTheDocument(); + }); + }); + + it("should handle search errors gracefully", async () => { + const user = userEvent.setup({ delay: null }); + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const mockSearchFn = vi.fn().mockRejectedValue(new Error("Search failed")); + + const options: FilterOption[] = [ + { + name: "model", + label: "Model", + isSearchable: true, + searchFn: mockSearchFn, + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(mockSearchFn).toHaveBeenCalledWith(""); + }); + + const modelLabel = screen.getByText("Model"); + const modelSection = modelLabel.closest("div"); + const modelSelect = within(modelSection!).getByRole("combobox"); + await user.click(modelSelect); + await user.type(modelSelect, "test"); + + await waitFor( + () => { + expect(consoleErrorSpy).toHaveBeenCalledWith("Error searching:", expect.any(Error)); + expect(screen.getByText("No results found")).toBeInTheDocument(); + }, + { timeout: 500 }, + ); + + consoleErrorSpy.mockRestore(); + }); + + it("should load initial options when dropdown opens for searchable filter", async () => { + const user = userEvent.setup({ delay: null }); + const mockSearchFn = vi.fn().mockResolvedValue([ + { label: "Initial Result", value: "initial" }, + ]); + + const options: FilterOption[] = [ + { + name: "model", + label: "Model", + isSearchable: true, + searchFn: mockSearchFn, + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(mockSearchFn).toHaveBeenCalledWith(""); + }); + + vi.clearAllMocks(); + + const modelLabel = screen.getByText("Model"); + const modelSection = modelLabel.closest("div"); + const modelSelect = within(modelSection!).getByRole("combobox"); + await user.click(modelSelect); + + await waitFor(() => { + expect(screen.getByText("Initial Result")).toBeInTheDocument(); + }); + }); + + it("should not render filters that are not in orderedFilters list", async () => { + const user = userEvent.setup({ delay: null }); + const options: FilterOption[] = [ + { + name: "unknownFilter", + label: "Unknown Filter", + }, + ]; + + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + expect(screen.queryByText("Unknown Filter")).not.toBeInTheDocument(); + }); + }); + + it("should call onApplyFilters with updated values when multiple filters change", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + const userIdInput = screen.getByPlaceholderText("Enter User ID..."); + await user.type(userIdInput, "user123"); + + await waitFor(() => { + expect(mockOnApplyFilters).toHaveBeenCalledWith({ userId: "user123" }); + }); + + const teamIdLabel = screen.getByText("Team ID"); + const teamIdSection = teamIdLabel.closest("div"); + const teamIdSelect = within(teamIdSection!).getByRole("combobox"); + await user.click(teamIdSelect); + + await waitFor(() => { + expect(screen.getByText("Team 1")).toBeInTheDocument(); + }); + + await user.click(screen.getByText("Team 1")); + + await waitFor(() => { + expect(mockOnApplyFilters).toHaveBeenCalledWith({ + userId: "user123", + teamId: "team1", + }); + }); + }); + + it("should reset all filter values when reset button is clicked", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders( + , + ); + + const filterButton = screen.getByRole("button", { name: "Filters" }); + await user.click(filterButton); + + await waitFor(() => { + const userIdInput = screen.getByPlaceholderText("Enter User ID...") as HTMLInputElement; + expect(userIdInput.value).toBe("user123"); + }); + + const resetButton = screen.getByRole("button", { name: "Reset Filters" }); + await user.click(resetButton); + + await waitFor(() => { + const userIdInput = screen.getByPlaceholderText("Enter User ID...") as HTMLInputElement; + expect(userIdInput.value).toBe(""); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/molecules/filter.tsx b/ui/litellm-dashboard/src/components/molecules/filter.tsx index 6c46392410d..a3a12fdf759 100644 --- a/ui/litellm-dashboard/src/components/molecules/filter.tsx +++ b/ui/litellm-dashboard/src/components/molecules/filter.tsx @@ -164,7 +164,7 @@ const FilterComponent: React.FC = ({ placeholder={`Search ${option.label || option.name}...`} value={tempValues[option.name] || undefined} onChange={(value) => handleFilterChange(option.name, value)} - onDropdownVisibleChange={(open) => handleDropdownVisibleChange(open, option)} + onOpenChange={(open) => handleDropdownVisibleChange(open, option)} onSearch={(value) => { setSearchInputValueMap((prev) => ({ ...prev, diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts new file mode 100644 index 00000000000..84862fa4632 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -0,0 +1,21 @@ +export const ERROR_CODE_OPTIONS: { label: string; value: string }[] = [ + { label: "400 - Bad Request", value: "400" }, + { label: "401 - Invalid Authentication", value: "401" }, + { label: "403 - Permission Denied", value: "403" }, + { label: "404 - Not Found", value: "404" }, + { label: "408 - Request Timeout", value: "408" }, + { label: "422 - Unprocessable Entity", value: "422" }, + { label: "429 - Rate Limited", value: "429" }, + { label: "500 - Internal Server Error", value: "500" }, + { label: "502 - Bad Gateway", value: "502" }, + { label: "503 - Service Unavailable", value: "503" }, + { label: "529 - Overloaded", value: "529" }, +]; + +export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ + { label: "Last 15 Minutes", value: 15, unit: "minutes" }, + { label: "Last Hour", value: 1, unit: "hours" }, + { label: "Last 4 Hours", value: 4, unit: "hours" }, + { label: "Last 24 Hours", value: 24, unit: "hours" }, + { label: "Last 7 Days", value: 7, unit: "days" }, +]; diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 87e11e00c7c..b791663cee3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -25,6 +25,7 @@ import { ErrorViewer } from "./ErrorViewer"; import { useLogFilterLogic } from "./log_filter_logic"; import { getTimeRangeDisplay } from "./logs_utils"; import { prefetchLogDetails } from "./prefetch"; +import { ERROR_CODE_OPTIONS, QUICK_SELECT_OPTIONS } from "./constants"; import { RequestResponsePanel } from "./RequestResponsePanel"; import { SessionView } from "./SessionView"; import SpendLogsSettingsModal from "./SpendLogsSettingsModal/SpendLogsSettingsModal"; @@ -372,24 +373,6 @@ export default function SpendLogsTable({ setSelectedLog(log); }; - // Function to extract unique error codes from logs - const extractErrorCodes = (logs: LogEntry[], searchText: string = "") => { - const errorCodes = new Set(); - logs.forEach((log) => { - const metadata = log.metadata || {}; - if (metadata.status === "failure" && metadata.error_information) { - const errorCode = metadata.error_information.error_code; - if (errorCode && (!searchText || errorCode.toLowerCase().includes(searchText.toLowerCase()))) { - errorCodes.add(errorCode); - } - } - }); - return Array.from(errorCodes).map((code) => ({ - label: code, - value: code, - })); - }; - const logFilterOptions: FilterOption[] = [ { name: "Team ID", @@ -455,7 +438,14 @@ export default function SpendLogsTable({ label: "Error Code", isSearchable: true, searchFn: async (searchText: string) => { - return extractErrorCodes(logsData.data, searchText); + if (!searchText) return ERROR_CODE_OPTIONS; + const lower = searchText.toLowerCase(); + const filtered = ERROR_CODE_OPTIONS.filter((opt) => opt.label.toLowerCase().includes(lower)); + const isExactValue = ERROR_CODE_OPTIONS.some((opt) => opt.value === searchText.trim()); + if (!isExactValue && searchText.trim()) { + filtered.push({ label: `Use custom code: ${searchText.trim()}`, value: searchText.trim() }); + } + return filtered; }, }, { @@ -492,15 +482,7 @@ export default function SpendLogsTable({ return unit; }; - const quickSelectOptions = [ - { label: "Last 15 Minutes", value: 15, unit: "minutes" }, - { label: "Last Hour", value: 1, unit: "hours" }, - { label: "Last 4 Hours", value: 4, unit: "hours" }, - { label: "Last 24 Hours", value: 24, unit: "hours" }, - { label: "Last 7 Days", value: 7, unit: "days" }, - ]; - - const selectedOption = quickSelectOptions.find( + const selectedOption = QUICK_SELECT_OPTIONS.find( (option) => option.value === selectedTimeInterval.value && option.unit === selectedTimeInterval.unit, ); @@ -617,7 +599,7 @@ export default function SpendLogsTable({ {quickSelectOpen && (
- {quickSelectOptions.map((option) => ( + {QUICK_SELECT_OPTIONS.map((option) => ( - {/* Display message if the URL is saved */} - {isUrlSaved && ( -
- - -

Proxy Admin UI (Save this URL): {clickableUrl}

- - -

- Get Started with Proxy Admin UI 👉 - - {clickableUrl} - -

- - -
- )} - -
- ); -}; - -export default EnterProxyUrl; diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/team_search_fn.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/team_search_fn.tsx deleted file mode 100644 index 7ff4417ea3f..00000000000 --- a/ui/litellm-dashboard/src/components/key_team_helpers/team_search_fn.tsx +++ /dev/null @@ -1,19 +0,0 @@ -import { Team } from "./key_list"; - -export const createTeamSearchFunction = (teams: Team[] | null) => { - return async (searchText: string): Promise> => { - // Return empty array if teams is null or searchText is empty - if (!teams || !searchText.trim()) { - return []; - } - - // Filter teams where team_alias contains the search text (case insensitive) - const filteredTeams = teams.filter((team) => team.team_alias.toLowerCase().includes(searchText.toLowerCase())); - - // Map filtered teams to the required format - return filteredTeams.map((team) => ({ - label: `${team.team_alias} (${team.team_id.substring(0, 8)}...)`, - value: team.team_id, - })); - }; -}; diff --git a/ui/litellm-dashboard/src/components/mcp_connection_test.tsx b/ui/litellm-dashboard/src/components/mcp_connection_test.tsx deleted file mode 100644 index 35f8bd7aa09..00000000000 --- a/ui/litellm-dashboard/src/components/mcp_connection_test.tsx +++ /dev/null @@ -1,279 +0,0 @@ -import React from "react"; -import { Typography, Space, Button, Divider } from "antd"; -import { WarningOutlined, InfoCircleOutlined, CopyOutlined } from "@ant-design/icons"; -import { testMCPConnectionRequest } from "./networking"; -import NotificationsManager from "./molecules/notifications_manager"; - -const { Text } = Typography; - -interface MCPConnectionTestProps { - formValues: Record; - accessToken: string; - serverName?: string; - onClose?: () => void; - onTestComplete?: () => void; -} - -const MCPConnectionTest: React.FC = ({ - formValues, - accessToken, - serverName = "this MCP server", - onClose, - onTestComplete, -}) => { - const [connectionError, setConnectionError] = React.useState(null); - const [rawRequest, setRawRequest] = React.useState(null); - const [rawResponse, setRawResponse] = React.useState(null); - const [isLoading, setIsLoading] = React.useState(true); - const [connectionSuccess, setConnectionSuccess] = React.useState(false); - const [showDetails, setShowDetails] = React.useState(false); - - const testMCPConnection = async () => { - setIsLoading(true); - setShowDetails(false); - setConnectionError(null); - setRawRequest(null); - setRawResponse(null); - setConnectionSuccess(false); - - // Add a small delay to ensure form values are fully populated - await new Promise((resolve) => setTimeout(resolve, 100)); - - try { - console.log("Testing MCP connection with form values:", formValues); - - // Prepare the MCP server config from form values - const mcpServerConfig = { - server_id: formValues.server_id || "", - alias: formValues.alias || "", - url: formValues.url, - transport: formValues.transport, - auth_type: formValues.auth_type, - mcp_info: formValues.mcp_info, - }; - - setRawRequest(mcpServerConfig); - - // Test connection - const connectionResponse = await testMCPConnectionRequest(accessToken, mcpServerConfig); - console.log("Connection test response:", connectionResponse); - - if (connectionResponse.status === "ok") { - setConnectionError(null); - setConnectionSuccess(true); - } else { - const errorMessage = connectionResponse.message || "Unknown connection error"; - setConnectionError(errorMessage); - setRawResponse(connectionResponse); - } - } catch (error) { - console.error("MCP connection test error:", error); - setConnectionError(error instanceof Error ? error.message : String(error)); - } finally { - setIsLoading(false); - if (onTestComplete) onTestComplete(); - } - }; - - React.useEffect(() => { - // Run the test once when component mounts - // Add a small timeout to ensure form values are ready - const timer = setTimeout(() => { - testMCPConnection(); - }, 200); - - return () => clearTimeout(timer); - }, []); // Empty dependency array means this runs once on mount - - const getCleanErrorMessage = (errorMsg: string) => { - if (!errorMsg) return "Unknown error"; - - const mainError = errorMsg.split("stack trace:")[0].trim(); - - const cleanedError = mainError.replace(/^(.*?)Error: /, ""); - - return cleanedError; - }; - - const connectionErrorMessage = - typeof connectionError === "string" - ? getCleanErrorMessage(connectionError) - : connectionError?.message - ? getCleanErrorMessage(connectionError.message) - : "Unknown error"; - - const formatMCPRequest = (mcpConfig: Record) => { - return JSON.stringify(mcpConfig, null, 2); - }; - - const isOverallSuccess = connectionSuccess && !connectionError; - - return ( -
- {isLoading ? ( -
-
- {/* Simple CSS spinner */} -
-
- Testing connection to {serverName}... - -
- ) : isOverallSuccess ? ( -
-
-
- -
- - Connection to {serverName} successful! - -
-
- ) : ( - <> -
-
- - - Connection to {serverName} failed - -
- -
- - Error:{" "} - - - {connectionErrorMessage} - - - {connectionError && ( -
- -
- )} -
- - {showDetails && ( -
- - Troubleshooting Details - - 
-                  {typeof connectionError === "string" ? connectionError : JSON.stringify(connectionError, null, 2)}
-
-
- )} - -
- - MCP Server Configuration - - 
-                {formatMCPRequest(rawRequest || {})}
-
- -
-
- - )} - -
- - - - - -
-
- ); -}; - -export default MCPConnectionTest; diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialDeleteModal.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialDeleteModal.tsx deleted file mode 100644 index 0ef889b9da4..00000000000 --- a/ui/litellm-dashboard/src/components/model_add/CredentialDeleteModal.tsx +++ /dev/null @@ -1,90 +0,0 @@ -import React, { useState } from "react"; -import { Modal } from "antd"; -import { Button as TremorButton } from "@tremor/react"; -import { ExclamationIcon } from "@heroicons/react/outline"; - -interface CredentialDeleteModalProps { - isVisible: boolean; - onCancel: () => void; - onConfirm: () => void; - credentialName: string; -} - -const CredentialDeleteModal: React.FC = ({ - isVisible, - onCancel, - onConfirm, - credentialName, -}) => { - const [deleteConfirmInput, setDeleteConfirmInput] = useState(""); - const isValid = deleteConfirmInput === credentialName; - - const handleCancel = () => { - setDeleteConfirmInput(""); - onCancel(); - }; - - const handleConfirm = () => { - if (isValid) { - setDeleteConfirmInput(""); - onConfirm(); - } - }; - - return ( - - - Delete Credential -
- } - open={isVisible} - footer={null} - onCancel={handleCancel} - closable={true} - destroyOnHidden={true} - maskClosable={false} - > -
-
-
- -
-
-

- This action cannot be undone and may break existing integrations. -

-
-
- -
- - setDeleteConfirmInput(e.target.value)} - placeholder="Enter credential name exactly" - className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" - autoFocus - /> -
- -
- - Cancel - - - Delete Credential - -
-
- - ); -}; - -export default CredentialDeleteModal; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 0d543952efc..023c88c5e83 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -83,7 +83,7 @@ const defaultServerRootPath = "/"; export let serverRootPath = defaultServerRootPath; export let proxyBaseUrl = defaultProxyBaseUrl; if (isLocal != true) { - console.log = function () {}; + console.log = function () { }; } const getWindowLocation = () => { @@ -136,8 +136,6 @@ const HTTP_REQUEST = { DELETE: "DELETE", }; -export const DEFAULT_ORGANIZATION = "default_organization"; - export interface Model { model_name: string; litellm_params: object; @@ -533,38 +531,6 @@ export const modelCreateCall = async (accessToken: string, formValues: Model) => } }; -export const modelSettingsCall = async (accessToken: string) => { - /** - * Get all configurable params for setting a model - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/settings` : `/model/settings`; - - //NotificationsManager.info("Requesting model data"); - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - //NotificationsManager.info("Received model data"); - return data; - // Handle success - you might want to update some state or UI based on the created key - } catch (error: any) { - console.error("Failed to get model settings:", error); - } -}; - export const modelDeleteCall = async (accessToken: string, model_id: string) => { console.log(`model_id in model delete call: ${model_id}`); try { @@ -2301,178 +2267,6 @@ export const deleteAllowedIP = async (accessToken: string, ip: string) => { } }; -export const modelMetricsCall = async ( - accessToken: string, - userID: string, - userRole: string, - modelGroup: string | null, - startTime: string | undefined, - endTime: string | undefined, - apiKey: string | null, - customer: string | null, -) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`; - if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; - } - // NotificationsManager.info("Requesting model data"); - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - // NotificationsManager.info("Received model data"); - return data; - // Handle success - you might want to update some state or UI based on the created key - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; -export const streamingModelMetricsCall = async ( - accessToken: string, - modelGroup: string | null, - startTime: string | undefined, - endTime: string | undefined, -) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/streaming_metrics` : `/model/streaming_metrics`; - if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}`; - } - // NotificationsManager.info("Requesting model data"); - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - // NotificationsManager.info("Received model data"); - return data; - // Handle success - you might want to update some state or UI based on the created key - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const modelMetricsSlowResponsesCall = async ( - accessToken: string, - userID: string, - userRole: string, - modelGroup: string | null, - startTime: string | undefined, - endTime: string | undefined, - apiKey: string | null, - customer: string | null, -) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`; - if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; - } - - // NotificationsManager.info("Requesting model data"); - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - // NotificationsManager.info("Received model data"); - return data; - // Handle success - you might want to update some state or UI based on the created key - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const modelExceptionsCall = async ( - accessToken: string, - userID: string, - userRole: string, - modelGroup: string | null, - startTime: string | undefined, - endTime: string | undefined, - apiKey: string | null, - customer: string | null, -) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/exceptions` : `/model/metrics/exceptions`; - - if (modelGroup) { - url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}&customer=${customer}`; - } - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - // NotificationsManager.info("Received model data"); - return data; - // Handle success - you might want to update some state or UI based on the created key - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const updateUsefulLinksCall = async ( accessToken: string, useful_links: Record, diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/types.ts b/ui/litellm-dashboard/src/components/playground/chat_ui/types.ts index 4e43e6ec77d..07855827886 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/types.ts +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/types.ts @@ -1,61 +1,3 @@ -export interface Delta { - content?: string; - reasoning_content?: string; - role?: string; - function_call?: any; - tool_calls?: any; - audio?: any; - refusal?: any; - provider_specific_fields?: any; - image?: { - url: string; - detail: string; - }; -} - -export interface CompletionTokensDetails { - accepted_prediction_tokens?: number; - audio_tokens?: number; - reasoning_tokens?: number; - rejected_prediction_tokens?: number; - text_tokens?: number | null; -} - -export interface PromptTokensDetails { - audio_tokens?: number; - cached_tokens?: number; - text_tokens?: number; - image_tokens?: number; -} - -export interface Usage { - completion_tokens: number; - prompt_tokens: number; - total_tokens: number; - completion_tokens_details?: CompletionTokensDetails; - prompt_tokens_details?: PromptTokensDetails; -} - -export interface StreamingChoices { - finish_reason?: string | null; - index: number; - delta: Delta; - logprobs?: any; -} - -export interface StreamingResponse { - id: string; - created: number; - model: string; - object: string; - system_fingerprint?: string; - choices: StreamingChoices[]; - provider_specific_fields?: any; - stream_options?: any; - citations?: any; - usage?: Usage; -} - export interface VectorStoreSearchResult { score: number; content: Array<{ text: string; type: string }>; diff --git a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx deleted file mode 100644 index 6f3ce27567c..00000000000 --- a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx +++ /dev/null @@ -1,1753 +0,0 @@ -import { - Card, - Col, - Grid, - Subtitle, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Text, - Title, -} from "@tremor/react"; -import React, { useEffect, useMemo, useRef, useState } from "react"; -import { CredentialItem, credentialListCall, CredentialsResponse } from "../networking"; - -import { handleAddModelSubmit } from "../add_model/handle_add_model_submit"; - -import CredentialsPanel from "@/components/model_add/credentials"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { FilterIcon, RefreshIcon } from "@heroicons/react/outline"; -import { - AreaChart, - BarChart, - Button, - DateRangePickerValue, - Icon, - Select, - SelectItem, - Tab, - TabGroup, - TabList, - TabPanel, - TabPanels, -} from "@tremor/react"; -import type { UploadProps } from "antd"; -import { Form, InputNumber, Popover, Typography } from "antd"; -import AddModelTab from "../add_model/add_model_tab"; -import { Team } from "../key_team_helpers/key_list"; -import ModelInfoView from "../model_info_view"; -import TimeToFirstToken from "../model_metrics/time_to_first_token"; -import { - adminGlobalActivityExceptions, - adminGlobalActivityExceptionsPerDeployment, - allEndUsersCall, - getCallbacksCall, - healthCheckCall, - modelCostMap, - modelExceptionsCall, - modelInfoCall, - modelMetricsCall, - modelMetricsSlowResponsesCall, - modelSettingsCall, - setCallbacksCall, - streamingModelMetricsCall, -} from "../networking"; -import { getPlaceholder, getProviderModels, provider_map, Providers } from "../provider_info_helpers"; -import UsageDatePicker from "../shared/usage_date_picker"; -import TeamInfoView from "../team/team_info"; -import { getDisplayModelName } from "../view_model/model_name_display"; - -import { all_admin_roles } from "@/utils/roles"; -import { PaginationState } from "@tanstack/react-table"; -import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; -import { ModelDataTable } from "../model_dashboard/table"; -import ModelGroupAliasSettings from "../model_group_alias_settings"; -import { columns } from "../molecules/models/columns"; -import NotificationsManager from "../molecules/notifications_manager"; -import PassThroughSettings from "../pass_through_settings"; -import PriceDataReload from "../price_data_reload"; - -interface ModelDashboardProps { - accessToken: string | null; - token: string | null; - userRole: string | null; - userID: string | null; - modelData: any; - keys: any[] | null; - setModelData: any; - premiumUser: boolean; - teams: Team[] | null; -} - -interface RetryPolicyObject { - [key: string]: { [retryPolicyKey: string]: number } | undefined; -} - -interface GlobalRetryPolicyObject { - [retryPolicyKey: string]: number; -} - -interface GlobalExceptionActivityData { - sum_num_rate_limit_exceptions: number; - daily_data: { date: string; num_rate_limit_exceptions: number }[]; -} - -//["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"] - -interface ProviderFields { - field_name: string; - field_type: string; - field_description: string; - field_value: string; -} - -interface ProviderSettings { - name: string; - fields: ProviderFields[]; -} - -const retry_policy_map: Record = { - "BadRequestError (400)": "BadRequestErrorRetries", - "AuthenticationError (401)": "AuthenticationErrorRetries", - "TimeoutError (408)": "TimeoutErrorRetries", - "RateLimitError (429)": "RateLimitErrorRetries", - "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", - "InternalServerError (500)": "InternalServerErrorRetries", -}; - -const OldModelDashboard: React.FC = ({ - accessToken, - token, - userRole, - userID, - modelData = { data: [] }, - keys, - setModelData, - premiumUser, - teams, -}) => { - const [addModelForm] = Form.useForm(); - const [autoRouterForm] = Form.useForm(); - const [modelMap, setModelMap] = useState(null); - const [lastRefreshed, setLastRefreshed] = useState(""); - - const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array - - const [providerSettings, setProviderSettings] = useState([]); - const [selectedProvider, setSelectedProvider] = useState(Providers.OpenAI); - const [healthCheckResponse, setHealthCheckResponse] = useState(null); - const [isHealthCheckLoading, setIsHealthCheckLoading] = useState(false); - const [editModalVisible, setEditModalVisible] = useState(false); - - const [selectedModel, setSelectedModel] = useState(null); - const [availableModelGroups, setAvailableModelGroups] = useState>([]); - const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState>([]); - const [selectedModelGroup, setSelectedModelGroup] = useState(null); - const [modelMetrics, setModelMetrics] = useState([]); - const [modelMetricsCategories, setModelMetricsCategories] = useState([]); - const [streamingModelMetrics, setStreamingModelMetrics] = useState([]); - const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = useState([]); - const [modelExceptions, setModelExceptions] = useState([]); - const [allExceptions, setAllExceptions] = useState([]); - const [slowResponsesData, setSlowResponsesData] = useState([]); - const [dateValue, setDateValue] = useState({ - from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), - to: new Date(), - }); - - const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); - const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null); - const [defaultRetry, setDefaultRetry] = useState(0); - - const [globalExceptionData, setGlobalExceptionData] = useState( - {} as GlobalExceptionActivityData, - ); - const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState([]); - - const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); - const [selectedAPIKey, setSelectedAPIKey] = useState(null); - const [selectedCustomer, setSelectedCustomer] = useState(null); - - const [allEndUsers, setAllEndUsers] = useState([]); - - const [credentialsList, setCredentialsList] = useState([]); - - // Model Group Alias state - const [modelGroupAlias, setModelGroupAlias] = useState<{ [key: string]: string }>({}); - - // Add state for advanced settings visibility - const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); - - // Add these state variables - const [selectedModelId, setSelectedModelId] = useState(null); - const [editModel, setEditModel] = useState(false); - - const [selectedTeamId, setSelectedTeamId] = useState(null); - const [selectedTeam, setSelectedTeam] = useState(null); - - const [selectedTeamFilter, setSelectedTeamFilter] = useState(null); - const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState(null); - - const [modelNameSearch, setModelNameSearch] = useState(""); - - // Add new state for current team and model view mode - const [currentTeam, setCurrentTeam] = useState("personal"); // 'personal' or team_id - const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">("current_team"); - - // Add state for showing/hiding filters - const [showFilters, setShowFilters] = useState(false); - - const [showColumnDropdown, setShowColumnDropdown] = useState(false); - - const [isDropdownOpen, setIsDropdownOpen] = useState(false); - const [expandedRows, setExpandedRows] = useState>(new Set()); - const dropdownRef = useRef(null); - - // Pagination state - const [pagination, setPagination] = useState({ - pageIndex: 0, - pageSize: 50, - }); - const [selectedTabIndex, setSelectedTabIndex] = useState(0); - - const handleCreateNewModelClick = () => { - if (selectedModelId) { - setSelectedModelId(null); - } - setSelectedTabIndex(1); - }; - - const resetFilters = () => { - setModelNameSearch(""); - setSelectedModelGroup("all"); - setSelectedModelAccessGroupFilter(null); - setCurrentTeam("personal"); - setModelViewMode("current_team"); - setPagination({ pageIndex: 0, pageSize: 50 }); - }; - - // Memoize filtered data to prevent unnecessary re-calculations - const filteredData = useMemo(() => { - if (!modelData || !modelData.data || modelData.data.length === 0) { - return []; - } - - return modelData.data.filter((model: any) => { - const searchMatch = - modelNameSearch === "" || model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); - - const modelNameMatch = - selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - !selectedModelGroup || - (selectedModelGroup === "wildcard" && model.model_name?.includes("*")); - - const accessGroupMatch = - selectedModelAccessGroupFilter === "all" || - model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || - !selectedModelAccessGroupFilter; - - let teamAccessMatch = true; - if (modelViewMode === "current_team") { - if (currentTeam === "personal") { - teamAccessMatch = model.model_info?.direct_access === true; - } else { - teamAccessMatch = model.model_info?.access_via_team_ids?.includes(currentTeam) === true; - } - } - - return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch; - }); - }, [modelData, modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); - - // Memoize paginated data - const paginatedData = useMemo(() => { - const startIndex = pagination.pageIndex * pagination.pageSize; - const endIndex = startIndex + pagination.pageSize; - return filteredData.slice(startIndex, endIndex); - }, [filteredData, pagination.pageIndex, pagination.pageSize]); - - // Reset pagination when filters change - useEffect(() => { - setPagination((prev) => ({ ...prev, pageIndex: 0 })); - }, [modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); - - const setProviderModelsFn = (provider: Providers) => { - const _providerModels = getProviderModels(provider, modelMap); - setProviderModels(_providerModels); - console.log(`providerModels: ${_providerModels}`); - }; - - const updateModelMetrics = async ( - modelGroup: string | null, - startTime: Date | undefined, - endTime: Date | undefined, - ) => { - console.log("Updating model metrics for group:", modelGroup); - if (!accessToken || !userID || !userRole || !startTime || !endTime) { - return; - } - console.log("inside updateModelMetrics - startTime:", startTime, "endTime:", endTime); - setSelectedModelGroup(modelGroup); - - let selected_token = selectedAPIKey?.token; - if (selected_token === undefined) { - selected_token = null; - } - - let selected_customer = selectedCustomer; - if (selected_customer === undefined) { - selected_customer = null; - } - - try { - const modelMetricsResponse = await modelMetricsCall( - accessToken, - userID, - userRole, - modelGroup, - startTime.toISOString(), - endTime.toISOString(), - selected_token, - selected_customer, - ); - console.log("Model metrics response:", modelMetricsResponse); - - // Assuming modelMetricsResponse now contains the metric data for the specified model group - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); - - const streamingModelMetricsResponse = await streamingModelMetricsCall( - accessToken, - modelGroup, - startTime.toISOString(), - endTime.toISOString(), - ); - - // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases); - - const modelExceptionsResponse = await modelExceptionsCall( - accessToken, - userID, - userRole, - modelGroup, - startTime.toISOString(), - endTime.toISOString(), - selected_token, - selected_customer, - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); - - const slowResponses = await modelMetricsSlowResponsesCall( - accessToken, - userID, - userRole, - modelGroup, - startTime.toISOString(), - endTime.toISOString(), - selected_token, - selected_customer, - ); - - console.log("slowResponses:", slowResponses); - - setSlowResponsesData(slowResponses); - - if (modelGroup) { - const dailyExceptions = await adminGlobalActivityExceptions( - accessToken, - startTime?.toISOString().split("T")[0], - endTime?.toISOString().split("T")[0], - modelGroup, - ); - - setGlobalExceptionData(dailyExceptions); - - const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( - accessToken, - startTime?.toISOString().split("T")[0], - endTime?.toISOString().split("T")[0], - modelGroup, - ); - - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); - } - } catch (error) { - console.error("Failed to fetch model metrics", error); - } - }; - - const fetchCredentials = async (accessToken: string) => { - try { - const response: CredentialsResponse = await credentialListCall(accessToken); - console.log(`credentials: ${JSON.stringify(response)}`); - setCredentialsList(response.credentials); - } catch (error) { - console.error("Error fetching credentials:", error); - } - }; - - useEffect(() => { - updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to); - }, [selectedAPIKey, selectedCustomer, selectedTeam]); - - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { - setIsDropdownOpen(false); - } - }; - - document.addEventListener("mousedown", handleClickOutside); - return () => document.removeEventListener("mousedown", handleClickOutside); - }, []); - - function formatCreatedAt(createdAt: string | null) { - if (createdAt) { - const date = new Date(createdAt); - const options = { month: "long", day: "numeric", year: "numeric" }; - return date.toLocaleDateString("en-US"); - } - return null; - } - - const handleEditClick = (model: any) => { - setSelectedModel(model); - setEditModalVisible(true); - }; - - const handleEditCancel = () => { - setEditModalVisible(false); - setSelectedModel(null); - }; - - const uploadProps: UploadProps = { - name: "file", - accept: ".json", - beforeUpload: (file) => { - if (file.type === "application/json") { - const reader = new FileReader(); - reader.onload = (e) => { - if (e.target) { - const jsonStr = e.target.result as string; - console.log(`Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}`); - addModelForm.setFieldsValue({ vertex_credentials: jsonStr }); - console.log("Form values right after setting:", addModelForm.getFieldsValue()); - } - }; - reader.readAsText(file); - } - // Prevent upload - return false; - }, - onChange(info) { - console.log("Upload onChange triggered with values:", info); - console.log("Current form values:", addModelForm.getFieldsValue()); - - if (info.file.status !== "uploading") { - console.log(info.file, info.fileList); - } - if (info.file.status === "done") { - NotificationsManager.success(`${info.file.name} file uploaded successfully`); - } else if (info.file.status === "error") { - NotificationsManager.fromBackend(`${info.file.name} file upload failed.`); - } - }, - }; - - const handleRefreshClick = () => { - // Update the 'lastRefreshed' state to the current date and time - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); - }; - - const handleSaveRetrySettings = async () => { - if (!accessToken) { - console.error("Access token is missing"); - return; - } - - try { - const payload: any = { - router_settings: {}, - }; - - if (selectedModelGroup === "global") { - // Only update global retry policy - console.log("Saving global retry policy:", globalRetryPolicy); - if (globalRetryPolicy) { - payload.router_settings.retry_policy = globalRetryPolicy; - } - NotificationsManager.success("Global retry settings saved successfully"); - } else { - // Only update model group retry policy - console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy); - if (modelGroupRetryPolicy) { - payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy; - } - NotificationsManager.success(`Retry settings saved successfully for ${selectedModelGroup}`); - } - - await setCallbacksCall(accessToken, payload); - } catch (error) { - console.error("Failed to save retry settings:", error); - NotificationsManager.fromBackend("Failed to save retry settings"); - } - }; - - useEffect(() => { - if (!accessToken || !token || !userRole || !userID) { - return; - } - const fetchData = async () => { - try { - // Replace with your actual API call for model data - const modelDataResponse = await modelInfoCall(accessToken, userID, userRole); - console.log("Model data response:", modelDataResponse.data); - setModelData(modelDataResponse); - const _providerSettings = await modelSettingsCall(accessToken); - if (_providerSettings) { - setProviderSettings(_providerSettings); - } - - // loop through modelDataResponse and get all`model_name` values - let all_model_groups: Set = new Set(); - for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - all_model_groups.add(model.model_name); - } - console.log("all_model_groups:", all_model_groups); - let _array_model_groups = Array.from(all_model_groups); - // sort _array_model_groups alphabetically - _array_model_groups = _array_model_groups.sort(); - - setAvailableModelGroups(_array_model_groups); - - let all_model_access_groups: Set = new Set(); - for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - let model_info: any | null = model.model_info; - if (model_info) { - let access_groups = model_info.access_groups; - if (access_groups) { - for (let j = 0; j < access_groups.length; j++) { - all_model_access_groups.add(access_groups[j]); - } - } - } - } - - setAvailableModelAccessGroups(Array.from(all_model_access_groups)); - - console.log("array_model_groups:", _array_model_groups); - let _initial_model_group = "all"; - if (_array_model_groups.length > 0) { - // set selectedModelGroup to the last model group - _initial_model_group = _array_model_groups[_array_model_groups.length - 1]; - console.log("_initial_model_group:", _initial_model_group); - //setSelectedModelGroup(_initial_model_group); - } - - console.log("selectedModelGroup:", selectedModelGroup); - - const modelMetricsResponse = await modelMetricsCall( - accessToken, - userID, - userRole, - _initial_model_group, - dateValue.from?.toISOString(), - dateValue.to?.toISOString(), - selectedAPIKey?.token, - selectedCustomer, - ); - - console.log("Model metrics response:", modelMetricsResponse); - // Sort by latency (avg_latency_per_token) - - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); - - const streamingModelMetricsResponse = await streamingModelMetricsCall( - accessToken, - _initial_model_group, - dateValue.from?.toISOString(), - dateValue.to?.toISOString(), - ); - - // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases); - - const modelExceptionsResponse = await modelExceptionsCall( - accessToken, - userID, - userRole, - _initial_model_group, - dateValue.from?.toISOString(), - dateValue.to?.toISOString(), - selectedAPIKey?.token, - selectedCustomer, - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); - - const slowResponses = await modelMetricsSlowResponsesCall( - accessToken, - userID, - userRole, - _initial_model_group, - dateValue.from?.toISOString(), - dateValue.to?.toISOString(), - selectedAPIKey?.token, - selectedCustomer, - ); - - const dailyExceptions = await adminGlobalActivityExceptions( - accessToken, - dateValue.from?.toISOString().split("T")[0], - dateValue.to?.toISOString().split("T")[0], - _initial_model_group, - ); - - setGlobalExceptionData(dailyExceptions); - - const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( - accessToken, - dateValue.from?.toISOString().split("T")[0], - dateValue.to?.toISOString().split("T")[0], - _initial_model_group, - ); - - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); - - console.log("dailyExceptions:", dailyExceptions); - - console.log("dailyExceptionsPerDeplyment:", dailyExceptionsPerDeplyment); - - console.log("slowResponses:", slowResponses); - - setSlowResponsesData(slowResponses); - - let all_end_users_data = await allEndUsersCall(accessToken); - - setAllEndUsers(all_end_users_data?.map((u: any) => u.user_id)); - - const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole); - - let router_settings = routerSettingsInfo.router_settings; - - console.log("routerSettingsInfo:", router_settings); - - let model_group_retry_policy = router_settings.model_group_retry_policy; - let default_retries = router_settings.num_retries; - - console.log("model_group_retry_policy:", model_group_retry_policy); - console.log("default_retries:", default_retries); - setModelGroupRetryPolicy(model_group_retry_policy); - setGlobalRetryPolicy(router_settings.retry_policy); - setDefaultRetry(default_retries); - - // Set model group alias - const model_group_alias = router_settings.model_group_alias || {}; - setModelGroupAlias(model_group_alias); - } catch (error) { - console.error("There was an error fetching the model data", error); - } - }; - - if (accessToken && token && userRole && userID) { - fetchData(); - } - - const fetchModelMap = async () => { - const data = await modelCostMap(); - console.log(`received model cost map data: ${Object.keys(data)}`); - setModelMap(data); - }; - if (modelMap == null) { - fetchModelMap(); - } - - handleRefreshClick(); - }, [accessToken, token, userRole, userID, modelMap, lastRefreshed, selectedTeam]); - - if (!modelData) { - return
Loading...
; - } - - if (!accessToken || !token || !userRole || !userID) { - return
Loading...
; - } - let all_models_on_proxy: any[] = []; - let all_providers: string[] = []; - - // loop through model data and edit each row - for (let i = 0; i < modelData.data.length; i++) { - let curr_model = modelData.data[i]; - let litellm_model_name = curr_model?.litellm_params?.model; - let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider; - let model_info = curr_model?.model_info; - - let defaultProvider = "openai"; - let provider = ""; - let input_cost = "Undefined"; - let output_cost = "Undefined"; - let max_tokens = "Undefined"; - let max_input_tokens = "Undefined"; - let cleanedLitellmParams = {}; - - const getProviderFromModel = (model: string) => { - /** - * Use model map - * - check if model in model map - * - return it's litellm_provider, if so - */ - console.log(`GET PROVIDER CALLED! - ${modelMap}`); - if (modelMap !== null && modelMap !== undefined) { - if (typeof modelMap == "object" && model in modelMap) { - return modelMap[model]["litellm_provider"]; - } - } - return "openai"; - }; - - // Check if litellm_model_name is null or undefined - if (litellm_model_name) { - // Split litellm_model_name based on "/" - let splitModel = litellm_model_name.split("/"); - - // Get the first element in the split - let firstElement = splitModel[0]; - - // If there is only one element, default provider to openai - provider = custom_llm_provider; - if (!provider) { - provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement; - } - } else { - // litellm_model_name is null or undefined, default provider to openai - provider = "-"; - } - - if (model_info) { - input_cost = model_info?.input_cost_per_token; - output_cost = model_info?.output_cost_per_token; - max_tokens = model_info?.max_tokens; - max_input_tokens = model_info?.max_input_tokens; - } - - if (curr_model?.litellm_params) { - cleanedLitellmParams = Object.fromEntries( - Object.entries(curr_model?.litellm_params).filter(([key]) => key !== "model" && key !== "api_base"), - ); - } - - modelData.data[i].provider = provider; - modelData.data[i].input_cost = input_cost; - modelData.data[i].output_cost = output_cost; - modelData.data[i].litellm_model_name = litellm_model_name; - all_providers.push(provider); - - // Convert Cost in terms of Cost per 1M tokens - if (modelData.data[i].input_cost) { - modelData.data[i].input_cost = (Number(modelData.data[i].input_cost) * 1000000).toFixed(2); - } - - if (modelData.data[i].output_cost) { - modelData.data[i].output_cost = (Number(modelData.data[i].output_cost) * 1000000).toFixed(2); - } - - modelData.data[i].max_tokens = max_tokens; - modelData.data[i].max_input_tokens = max_input_tokens; - modelData.data[i].api_base = curr_model?.litellm_params?.api_base; - modelData.data[i].cleanedLitellmParams = cleanedLitellmParams; - - all_models_on_proxy.push(curr_model.model_name); - - console.log(modelData.data[i]); - } - // when users click request access show pop up to allow them to request access - - if (userRole && userRole == "Admin Viewer") { - const { Title, Paragraph } = Typography; - return ( -
- Access Denied - Ask your proxy admin for access to view all models -
- ); - } - - const runHealthCheck = async () => { - try { - NotificationsManager.info("Running health check..."); - setIsHealthCheckLoading(true); - setHealthCheckResponse(null); - const response = await healthCheckCall(accessToken); - setHealthCheckResponse(response); - } catch (error) { - console.error("Error running health check:", error); - setHealthCheckResponse("Error running health check"); - } finally { - setIsHealthCheckLoading(false); - } - }; - - const FilterByContent = ( -
- Select API Key Name - - {premiumUser ? ( -
- - - Select Customer Name - - - - Select Team - - -
- ) : ( -
- {/* ... existing non-premium user content ... */} - Select Team - - -
- )} -
- ); - - const customTooltip = (props: any) => { - const { payload, active } = props; - if (!active || !payload) return null; - - // Extract the date from the first item in the payload array - const date = payload[0]?.payload?.date; - - // Sort the payload array by category.value in descending order - let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); - - // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown - if (sortedPayload.length > 5) { - let remainingItems = sortedPayload.length - 5; - sortedPayload = sortedPayload.slice(0, 5); - sortedPayload.push({ - dataKey: `${remainingItems} other deployments`, - value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), - color: "gray", - }); - } - - return ( -
- {date &&

Date: {date}

} - {sortedPayload.map((category: any, idx: number) => { - const roundedValue = parseFloat(category.value.toFixed(5)); - const displayValue = roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5); - return ( -
-
-
-

{category.dataKey}

-
-

{displayValue}

-
- ); - })} -
- ); - }; - - const handleOk = async () => { - console.log("🚀 handleOk called from model dashboard!"); - console.log("Current form values:", addModelForm.getFieldsValue()); - - addModelForm - .validateFields() - .then((values: any) => { - console.log("✅ Validation passed, submitting:", values); - handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick); - }) - .catch((error: any) => { - console.error("❌ Validation failed:", error); - console.error("Form errors:", error.errorFields); - const errorMessages = - error.errorFields - ?.map((field: any) => { - return `${field.name.join(".")}: ${field.errors.join(", ")}`; - }) - .join(" | ") || "Unknown validation error"; - NotificationsManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`); - }); - }; - - console.log(`selectedProvider: ${selectedProvider}`); - console.log(`providerModels.length: ${providerModels.length}`); - - const providerKey = Object.keys(Providers).find( - (key) => (Providers as { [index: string]: any })[key] === selectedProvider, - ); - - let dynamicProviderForm: ProviderSettings | undefined = undefined; - if (providerKey && providerSettings) { - dynamicProviderForm = providerSettings.find((provider) => provider.name === provider_map[providerKey]); - } - - // If a team is selected, render TeamInfoView in full page layout - if (selectedTeamId) { - return ( -
- setSelectedTeamId(null)} - accessToken={accessToken} - is_team_admin={userRole === "Admin"} - is_proxy_admin={userRole === "Proxy Admin"} - userModels={all_models_on_proxy} - editTeam={false} - onUpdate={handleRefreshClick} - /> -
- ); - } - - return ( -
- - - {/* Model Management Header */} -
-
-

Model Management

- {!all_admin_roles.includes(userRole) ? ( -

Add models for teams you are an admin for.

- ) : ( -

Add and manage models for the proxy

- )} -
-
- {selectedModelId ? ( - { - setSelectedModelId(null); - setEditModel(false); - }} - accessToken={accessToken} - userID={userID} - userRole={userRole} - onModelUpdate={(updatedModel) => { - // Update the model in the modelData.data array - const updatedModelData = { - ...modelData, - data: modelData.data.map((model: any) => - model.model_info.id === updatedModel.model_info.id ? updatedModel : model, - ), - }; - setModelData(updatedModelData); - // Trigger a refresh to update UI - handleRefreshClick(); - }} - modelAccessGroups={availableModelAccessGroups} - /> - ) : ( - - -
- {all_admin_roles.includes(userRole) ? All Models : Your Models} - Add Model - {all_admin_roles.includes(userRole) && LLM Credentials} - {all_admin_roles.includes(userRole) && Pass-Through Endpoints} - {all_admin_roles.includes(userRole) && Health Status} - {all_admin_roles.includes(userRole) && Model Analytics} - {all_admin_roles.includes(userRole) && Model Retry Settings} - {all_admin_roles.includes(userRole) && Model Group Alias} - {all_admin_roles.includes(userRole) && Price Data Reload} -
- -
- {lastRefreshed && Last Refreshed: {lastRefreshed}} - -
- - - - -
-
- {/* Current Team and View Mode Selector - Prominent Section */} -
-
-
- Current Team: - -
- -
- View: - -
-
- - {modelViewMode === "current_team" && ( -
- -
- {currentTeam === "personal" ? ( - - To access these models: Create a Virtual Key without selecting a team on the{" "} - - Virtual Keys page - - - ) : ( - - To access these models: Create a Virtual Key and select Team as " - {currentTeam}" on the{" "} - - Virtual Keys page - - - )} -
-
- )} -
- - {/* Search and Filter Controls */} -
-
- {/* Search and Filter Controls */} -
- {/* Model Name Search */} -
- setModelNameSearch(e.target.value)} - /> - - - -
- - {/* Filter Button */} - - - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* Model Name Filter */} -
- -
- - {/* Model Access Group Filter */} -
- -
-
- )} - - {/* Results Count and Pagination Controls */} -
- - {filteredData.length > 0 - ? `Showing ${pagination.pageIndex * pagination.pageSize + 1} - ${Math.min( - (pagination.pageIndex + 1) * pagination.pageSize, - filteredData.length, - )} of ${filteredData.length} results` - : "Showing 0 results"} - - - {/* Pagination Controls */} - {filteredData.length > pagination.pageSize && ( -
- - - -
- )} -
-
-
- - -
-
- - - - - - - - - - - - - - - - - - { - setDateValue(value); - updateModelMetrics(selectedModelGroup, value.from, value.to); - }} - /> - - - Select Model Group - - - - - - - - - - - - - - - Avg. Latency per Token - Time to first token - - - -

(seconds/token)

- - average Latency for successfull requests divided by the total tokens - - {modelMetrics && modelMetricsCategories && ( - - )} - - - - - - - - - - - - - - Deployment - Success Responses - - Slow Responses

Success Responses taking 600+s

- - - - - {slowResponsesData.map((metric, idx) => ( - - {metric.api_base} - {metric.total_count} - {metric.slow_count} - - ))} - -
- - - - - - All Exceptions for {selectedModelGroup} - - - - - - - - All Up Rate Limit Errors (429) for {selectedModelGroup} - - - - Num Rate Limit Errors {globalExceptionData.sum_num_rate_limit_exceptions} - - console.log(v)} - /> - - - - - - {premiumUser ? ( - <> - {globalExceptionPerDeployment.map((globalActivity, index) => ( - - {globalActivity.api_base ? globalActivity.api_base : "Unknown API Base"} - - - - Num Rate Limit Errors (429) {globalActivity.sum_num_rate_limit_exceptions} - - console.log(v)} - /> - - - - ))} - - ) : ( - <> - {globalExceptionPerDeployment && - globalExceptionPerDeployment.length > 0 && - globalExceptionPerDeployment.slice(0, 1).map((globalActivity, index) => ( - - ✨ Rate Limit Errors by Deployment -

- Upgrade to see exceptions for all deployments -

- - - {globalActivity.api_base} - - - - Num Rate Limit Errors {globalActivity.sum_num_rate_limit_exceptions} - - console.log(v)} - /> - - - - - ))} - - )} - - - -
-
- Retry Policy Scope: - -
-
- - {selectedModelGroup === "global" ? ( - <> - Global Retry Policy - Default retry settings applied to all model groups unless overridden - - ) : ( - <> - Retry Policy for {selectedModelGroup} - - Model-specific retry settings. Falls back to global defaults if not set. - - - )} - {retry_policy_map && ( - - - {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => { - let retryCount: number; - - if (selectedModelGroup === "global") { - // Show global policy values - retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; - } else { - // Show model-group specific values with fallback to global - const modelSpecificCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey]; - if (modelSpecificCount != null) { - retryCount = modelSpecificCount; - } else { - // Fall back to global policy, then default - retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; - } - } - - return ( - - - - - ); - })} - -
- {exceptionType} - {selectedModelGroup !== "global" && ( - - (Global: {globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry}) - - )} - - { - if (selectedModelGroup === "global") { - // Update global policy - setGlobalRetryPolicy((prevGlobalRetryPolicy) => { - if (value == null) return prevGlobalRetryPolicy; - return { - ...(prevGlobalRetryPolicy ?? {}), - [retryPolicyKey]: value, - }; - }); - } else { - // Update model-group specific policy - setModelGroupRetryPolicy((prevModelGroupRetryPolicy) => { - const prevRetryPolicy = prevModelGroupRetryPolicy?.[selectedModelGroup!] ?? {}; - return { - ...(prevModelGroupRetryPolicy ?? {}), - [selectedModelGroup!]: { - ...prevRetryPolicy, - [retryPolicyKey!]: value, - }, - } as RetryPolicyObject; - }); - } - }} - /> -
- )} - - - - - - -
-
- Price Data Management - - Manage model pricing data and configure automatic reload schedules - -
- { - // Refresh the model map after successful reload - const fetchModelMap = async () => { - const data = await modelCostMap(); - setModelMap(data); - }; - fetchModelMap(); - }} - buttonText="Reload Price Data" - size="middle" - type="primary" - className="w-full" - /> -
- - - - )} - - -
- ); -}; - -export default OldModelDashboard; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/constants.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/constants.ts index 3186cc27b42..67c274a6d29 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/constants.ts @@ -29,7 +29,6 @@ export const FONT_SIZE_HEADER = 16; // Colors export const COLOR_BORDER = "#f0f0f0"; export const COLOR_BACKGROUND = "#fff"; -export const COLOR_SECONDARY = "#8c8c8c"; export const COLOR_BG_LIGHT = "#fafafa"; // Spacing @@ -38,5 +37,3 @@ export const SPACING_MEDIUM = 8; export const SPACING_LARGE = 12; export const SPACING_XLARGE = 16; export const SPACING_XXLARGE = 24; - -// Messages (kept for backwards compatibility if needed elsewhere) diff --git a/ui/litellm-dashboard/src/components/view_logs/country_cell.tsx b/ui/litellm-dashboard/src/components/view_logs/country_cell.tsx deleted file mode 100644 index b4cae556fd0..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/country_cell.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import * as React from "react"; -import { getCountryFromIP } from "./ip_lookup"; - -interface CountryCellProps { - ipAddress: string | null; -} - -export const CountryCell: React.FC = ({ ipAddress }) => { - const [country, setCountry] = React.useState("-"); - - React.useEffect(() => { - if (!ipAddress) return; - - let mounted = true; - getCountryFromIP(ipAddress) - .then((result) => { - if (mounted) setCountry(result); - }) - .catch(() => { - if (mounted) setCountry("-"); - }); - - return () => { - mounted = false; - }; - }, [ipAddress]); - - return {country}; -}; diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 0e875bc4fd1..628a2ecfd57 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -8,7 +8,7 @@ import { debounce } from "lodash"; import { defaultPageSize } from "../constants"; import { PaginatedResponse } from "."; -export const FILTER_KEYS = { +const FILTER_KEYS = { TEAM_ID: "Team ID", KEY_HASH: "Key Hash", REQUEST_ID: "Request ID", From ff5a3acc1c8b8cd3fb2a856597b982dae24b7af9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Feb 2026 12:07:53 -0800 Subject: [PATCH 057/117] addressing feedback around tests --- .../mcpServers/useMCPAccessGroups.test.ts | 38 ++++++++++++++-- .../hooks/mcpServers/useMCPServers.test.ts | 44 ++++++++++++++++--- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.test.ts index b57f5d182d5..9c555ff1234 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.test.ts @@ -31,6 +31,7 @@ const wrapper = ({ children }: { children: React.ReactNode }) => { return React.createElement(QueryClientProvider, { client: queryClient }, children); }; +const mockAccessToken = "test-token-456"; const mockAccessGroups = ["group-1", "group-2", "group-3"]; describe("useMCPAccessGroups", () => { @@ -38,10 +39,22 @@ describe("useMCPAccessGroups", () => { vi.clearAllMocks(); const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); vi.mocked(useAuthorizedModule.default).mockReturnValue({ - accessToken: "test-token-456", + accessToken: mockAccessToken, } as any); }); + it("should return hook result without errors", () => { + vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); + + const { result } = renderHook(() => useMCPAccessGroups(), { wrapper }); + + expect(result.current).toBeDefined(); + expect(result.current).toHaveProperty("data"); + expect(result.current).toHaveProperty("isSuccess"); + expect(result.current).toHaveProperty("isError"); + expect(result.current).toHaveProperty("status"); + }); + it("should return MCP access groups when access token is present", async () => { vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue(mockAccessGroups); @@ -51,11 +64,11 @@ describe("useMCPAccessGroups", () => { expect(result.current.isSuccess).toBe(true); }); - expect(networking.fetchMCPAccessGroups).toHaveBeenCalledWith("test-token-456"); + expect(networking.fetchMCPAccessGroups).toHaveBeenCalledWith(mockAccessToken); expect(result.current.data).toEqual(mockAccessGroups); }); - it("should not fetch when access token is not available", async () => { + it("should not fetch when access token is null", async () => { const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); vi.mocked(useAuthorizedModule.default).mockReturnValue({ accessToken: null, @@ -63,7 +76,23 @@ describe("useMCPAccessGroups", () => { const { result } = renderHook(() => useMCPAccessGroups(), { wrapper }); - expect(result.current.status).toBe("pending"); + expect(result.current.isFetching).toBe(false); + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(networking.fetchMCPAccessGroups).not.toHaveBeenCalled(); + }); + + it("should not fetch when access token is empty string", async () => { + const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); + vi.mocked(useAuthorizedModule.default).mockReturnValue({ + accessToken: "", + } as any); + + const { result } = renderHook(() => useMCPAccessGroups(), { wrapper }); + + expect(result.current.isFetching).toBe(false); + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); expect(networking.fetchMCPAccessGroups).not.toHaveBeenCalled(); }); @@ -78,6 +107,7 @@ describe("useMCPAccessGroups", () => { }); expect(result.current.error).toEqual(mockError); + expect(result.current.data).toBeUndefined(); }); it("should return empty array when API returns no groups", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.test.ts index b30591cf1cc..3681ffc7475 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.test.ts @@ -10,7 +10,7 @@ vi.mock("@/components/networking", () => ({ fetchMCPServers: vi.fn(), })); -vi.mock("../useAuthorized", () => ({ +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: vi.fn(() => ({ accessToken: "test-token-123", })), @@ -31,6 +31,7 @@ const wrapper = ({ children }: { children: React.ReactNode }) => { return React.createElement(QueryClientProvider, { client: queryClient }, children); }; +const mockAccessToken = "test-token-123"; const mockServers = [ { server_id: "server-1", @@ -46,12 +47,24 @@ const mockServers = [ describe("useMCPServers", () => { beforeEach(async () => { vi.clearAllMocks(); - const useAuthorizedModule = await import("../useAuthorized"); + const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); vi.mocked(useAuthorizedModule.default).mockReturnValue({ - accessToken: "test-token-123", + accessToken: mockAccessToken, } as any); }); + it("should return hook result without errors", () => { + vi.mocked(networking.fetchMCPServers).mockResolvedValue([]); + + const { result } = renderHook(() => useMCPServers(), { wrapper }); + + expect(result.current).toBeDefined(); + expect(result.current).toHaveProperty("data"); + expect(result.current).toHaveProperty("isSuccess"); + expect(result.current).toHaveProperty("isError"); + expect(result.current).toHaveProperty("status"); + }); + it("should return MCP servers when access token is present", async () => { vi.mocked(networking.fetchMCPServers).mockResolvedValue(mockServers); @@ -61,19 +74,35 @@ describe("useMCPServers", () => { expect(result.current.isSuccess).toBe(true); }); - expect(networking.fetchMCPServers).toHaveBeenCalledWith("test-token-123"); + expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken); expect(result.current.data).toEqual(mockServers); }); - it("should not fetch when access token is not available", async () => { - const useAuthorizedModule = await import("../useAuthorized"); + it("should not fetch when access token is null", async () => { + const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); vi.mocked(useAuthorizedModule.default).mockReturnValue({ accessToken: null, } as any); const { result } = renderHook(() => useMCPServers(), { wrapper }); - expect(result.current.status).toBe("pending"); + expect(result.current.isFetching).toBe(false); + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(networking.fetchMCPServers).not.toHaveBeenCalled(); + }); + + it("should not fetch when access token is empty string", async () => { + const useAuthorizedModule = await import("@/app/(dashboard)/hooks/useAuthorized"); + vi.mocked(useAuthorizedModule.default).mockReturnValue({ + accessToken: "", + } as any); + + const { result } = renderHook(() => useMCPServers(), { wrapper }); + + expect(result.current.isFetching).toBe(false); + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); expect(networking.fetchMCPServers).not.toHaveBeenCalled(); }); @@ -88,6 +117,7 @@ describe("useMCPServers", () => { }); expect(result.current.error).toEqual(mockError); + expect(result.current.data).toBeUndefined(); }); it("should return empty array when API returns empty list", async () => { From 4555ed37c5b4594acf4c0a4eafbe1dea9b6dfd35 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Feb 2026 12:11:32 -0800 Subject: [PATCH 058/117] fix(callbacks): allow MAX_CALLBACKS override via env var (#20781) * fix(callbacks): allow MAX_CALLBACKS override via env var (#20778) * fix(callbacks): allow MAX_CALLBACKS override via env var - Move MAX_CALLBACKS from logging_callback_manager.py to constants.py - Add LITELLM_MAX_CALLBACKS env var override (default: 30) - Add troubleshooting doc explaining the limit and override Fixes issue where large deployments with 60+ teams using guardrails would hit the hardcoded MAX_CALLBACKS=30 limit and fail to start. * docs: add max_callbacks to sidebar navigation --------- Co-authored-by: shin-bot-litellm * fix callbacks issue --------- Co-authored-by: shin-bot-litellm Co-authored-by: shin-bot-litellm --- .../docs/troubleshoot/max_callbacks.md | 68 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/constants.py | 7 ++ litellm/litellm_core_utils/env_utils.py | 21 ++++++ .../logging_callback_manager.py | 8 +-- 5 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 docs/my-website/docs/troubleshoot/max_callbacks.md create mode 100644 litellm/litellm_core_utils/env_utils.py diff --git a/docs/my-website/docs/troubleshoot/max_callbacks.md b/docs/my-website/docs/troubleshoot/max_callbacks.md new file mode 100644 index 00000000000..4b0f3e24b73 --- /dev/null +++ b/docs/my-website/docs/troubleshoot/max_callbacks.md @@ -0,0 +1,68 @@ +# MAX_CALLBACKS Limit + +## Error Message + +``` +Cannot add callback - would exceed MAX_CALLBACKS limit of 30. Current callbacks: 30 +``` + +## What This Means + +LiteLLM limits the number of callbacks that can be registered to prevent performance degradation. Each callback runs on every LLM request, so having too many callbacks can cause exponential CPU usage and slow down your proxy. + +The default limit is **30 callbacks**. + +## When You Might Hit This Limit + +- **Large enterprise deployments** with many teams, each having their own guardrails +- **Multiple logging integrations** combined with custom callbacks +- **Per-team callback configurations** that add up across your organization + +## How to Override + +Set the `LITELLM_MAX_CALLBACKS` environment variable to increase the limit: + +```bash +# Docker +docker run -e LITELLM_MAX_CALLBACKS=100 ... + +# Docker Compose +environment: + - LITELLM_MAX_CALLBACKS=100 + +# Kubernetes +env: + - name: LITELLM_MAX_CALLBACKS + value: "100" + +# Direct +export LITELLM_MAX_CALLBACKS=100 +litellm --config config.yaml +``` + +## Recommendations + +1. **Start conservative** - Only increase as much as you need. If you have 60 teams with guardrails, try `LITELLM_MAX_CALLBACKS=75` to leave headroom. + +2. **Monitor performance** - More callbacks means more processing per request. Watch your CPU usage and response latency after increasing the limit. + +3. **Consolidate where possible** - If multiple teams use identical guardrails, consider using shared callback configurations rather than per-team duplicates. + +## Example: Large Enterprise Setup + +For an organization with 60+ teams, each with a guardrail callback: + +```yaml +# config.yaml +litellm_settings: + callbacks: ["prometheus", "langfuse"] # 2 global callbacks + +# Each team adds 1 guardrail callback = 60+ callbacks +# Total: 62+ callbacks needed +``` + +Set the environment variable: + +```bash +export LITELLM_MAX_CALLBACKS=100 +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 37392ff5c18..0d95bf7e545 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1081,6 +1081,7 @@ const sidebars = { "troubleshoot/cpu_issues", "troubleshoot/memory_issues", "troubleshoot/spend_queue_warnings", + "troubleshoot/max_callbacks", ], }, ], diff --git a/litellm/constants.py b/litellm/constants.py index 3c618723d64..f5f2c7a49e4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2,6 +2,8 @@ import os import sys from typing import List, Literal +from litellm.litellm_core_utils.env_utils import get_env_int + DEFAULT_HEALTH_CHECK_PROMPT = str( os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm") ) @@ -99,6 +101,11 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int( ) ) +# Maximum number of callbacks that can be registered +# This prevents callbacks from exponentially growing and consuming CPU resources +# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails) +MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 30) + # Generic fallback for unknown models DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int( os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128) diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py new file mode 100644 index 00000000000..34c65275331 --- /dev/null +++ b/litellm/litellm_core_utils/env_utils.py @@ -0,0 +1,21 @@ +""" +Utility helpers for reading and parsing environment variables. +""" + +import os + + +def get_env_int(env_var: str, default: int) -> int: + """Parse an environment variable as an integer, falling back to default on invalid values. + + Handles empty strings, whitespace, and non-numeric values gracefully + so that misconfiguration doesn't crash the process at import time. + """ + raw = os.getenv(env_var) + if raw is None: + return default + raw = raw.strip() + try: + return int(raw) + except (ValueError, TypeError): + return default diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 435ae078a65..34d25817378 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Set, Type, Uni import litellm from litellm._logging import verbose_logger +from litellm.constants import MAX_CALLBACKS from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger @@ -24,9 +25,6 @@ class LoggingCallbackManager: - Keep a reasonable MAX_CALLBACKS limit (this ensures callbacks don't exponentially grow and consume CPU Resources) """ - # healthy maximum number of callbacks - unlikely someone needs more than 20 - MAX_CALLBACKS = 30 - def add_litellm_input_callback(self, callback: Union[CustomLogger, str]): """ Add a input callback to litellm.input_callback @@ -155,9 +153,9 @@ class LoggingCallbackManager: Check if adding another callback would exceed MAX_CALLBACKS Returns True if safe to add, False if would exceed limit """ - if len(parent_list) >= self.MAX_CALLBACKS: + if len(parent_list) >= MAX_CALLBACKS: verbose_logger.warning( - f"Cannot add callback - would exceed MAX_CALLBACKS limit of {self.MAX_CALLBACKS}. Current callbacks: {len(parent_list)}" + f"Cannot add callback - would exceed MAX_CALLBACKS limit of {MAX_CALLBACKS}. Current callbacks: {len(parent_list)}" ) return False return True From c2536ee82ac3cd3847454bf453262039f0668ee8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Feb 2026 14:05:05 -0800 Subject: [PATCH 059/117] refactor antd tabs and table --- .../ModelsAndEndpointsView.tsx | 2 +- .../src/app/(dashboard)/teams/TeamsView.tsx | 2 +- .../src/components/OldTeams.test.tsx | 2 +- .../src/components/OldTeams.tsx | 2 +- .../{team_info.test.tsx => TeamInfo.test.tsx} | 71 +- .../src/components/team/TeamInfo.tsx | 1294 ++++++++++++++++ .../components/team/TeamMemberTab.test.tsx | 386 +++++ .../src/components/team/TeamMemberTab.tsx | 233 +++ .../team/tabVisibilityUtils.test.ts | 74 + .../src/components/team/tabVisibilityUtils.ts | 59 + .../src/components/team/team_info.tsx | 1296 ----------------- .../components/team/team_member_view.test.tsx | 193 --- .../src/components/team/team_member_view.tsx | 193 --- 13 files changed, 2104 insertions(+), 1703 deletions(-) rename ui/litellm-dashboard/src/components/team/{team_info.test.tsx => TeamInfo.test.tsx} (88%) create mode 100644 ui/litellm-dashboard/src/components/team/TeamInfo.tsx create mode 100644 ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx create mode 100644 ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx create mode 100644 ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts create mode 100644 ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts delete mode 100644 ui/litellm-dashboard/src/components/team/team_info.tsx delete mode 100644 ui/litellm-dashboard/src/components/team/team_member_view.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/team/team_member_view.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 8bfbaa8d3a6..9d77774cb4c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -26,7 +26,7 @@ import ModelGroupAliasSettings from "../../../components/model_group_alias_setti import ModelInfoView from "../../../components/model_info_view"; import NotificationsManager from "../../../components/molecules/notifications_manager"; import PassThroughSettings from "../../../components/pass_through_settings"; -import TeamInfoView from "../../../components/team/team_info"; +import TeamInfoView from "../../../components/team/TeamInfo"; import useAuthorized from "../hooks/useAuthorized"; interface ModelDashboardProps { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx index 10616e95523..88bdf3cdda0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/TeamsView.tsx @@ -2,7 +2,7 @@ import React, { useState, useEffect } from "react"; import { teamDeleteCall, Organization } from "@/components/networking"; import { fetchTeams } from "@/components/common_components/fetch_teams"; import { Form } from "antd"; -import TeamInfoView from "@/components/team/team_info"; +import TeamInfoView from "@/components/team/TeamInfo"; import TeamSSOSettings from "@/components/TeamSSOSettings"; import { isAdminRole } from "@/utils/roles"; import { Card, Button, Col, Text, Grid, TabPanel } from "@tremor/react"; diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index 91b428c8c98..0ca37cc82d7 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -52,7 +52,7 @@ vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ }), })); -vi.mock("@/components/team/team_info", () => ({ +vi.mock("@/components/team/TeamInfo", () => ({ __esModule: true, default: (props: any) => { mockTeamInfoView(props); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index 4eab197db14..ecc6a624be0 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -1,6 +1,6 @@ import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import AvailableTeamsPanel from "@/components/team/available_teams"; -import TeamInfoView from "@/components/team/team_info"; +import TeamInfoView from "@/components/team/TeamInfo"; import TeamSSOSettings from "@/components/TeamSSOSettings"; import { isProxyAdminRole } from "@/utils/roles"; import { InfoCircleOutlined } from "@ant-design/icons"; diff --git a/ui/litellm-dashboard/src/components/team/team_info.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx similarity index 88% rename from ui/litellm-dashboard/src/components/team/team_info.test.tsx rename to ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index fa1b13fbd2d..7a15a6e3f22 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -3,7 +3,7 @@ import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; -import TeamInfoView from "./team_info"; +import TeamInfoView from "./TeamInfo"; vi.mock("@/components/networking", () => ({ teamInfoCall: vi.fn(), @@ -40,7 +40,7 @@ vi.mock("@/app/(dashboard)/hooks/users/useCurrentUser", () => ({ useCurrentUser: vi.fn(), })); -vi.mock("@/components/team/team_member_view", () => ({ +vi.mock("@/components/team/TeamMemberTab", () => ({ default: vi.fn(({ setIsAddMemberModalVisible }) => (
@@ -89,10 +89,6 @@ vi.mock("@/components/team/member_permissions", () => ({ default: vi.fn(() =>
Member Permissions
), })); -vi.mock("@/components/team/member_permissions", () => ({ - default: vi.fn(() =>
Member Permissions
), -})); - import { useAllProxyModels } from "@/app/(dashboard)/hooks/models/useModels"; import { useOrganization } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import { useTeam } from "@/app/(dashboard)/hooks/teams/useTeams"; @@ -335,10 +331,10 @@ describe("TeamInfoView", () => { await user.click(settingsTab); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); + const editButton = screen.getByRole("button", { name: /edit settings/i }); await user.click(editButton); await waitFor(() => { @@ -361,17 +357,17 @@ describe("TeamInfoView", () => { await user.click(settingsTab); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); + const editButton = screen.getByRole("button", { name: /edit settings/i }); await user.click(editButton); await waitFor(() => { expect(screen.getByLabelText("Team Name")).toBeInTheDocument(); }); - const cancelButton = screen.getByRole("button", { name: "Cancel" }); + const cancelButton = screen.getByRole("button", { name: /cancel/i }); await user.click(cancelButton); await waitFor(() => { @@ -438,10 +434,10 @@ describe("TeamInfoView", () => { await user.click(settingsTab); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); + const editButton = screen.getByRole("button", { name: /edit settings/i }); await user.click(editButton); const secretField = await screen.findByPlaceholderText( @@ -472,10 +468,10 @@ describe("TeamInfoView", () => { await user.click(settingsTab); await waitFor(() => { - expect(screen.getByRole("button", { name: "Edit Settings" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); }); - const editButton = screen.getByRole("button", { name: "Edit Settings" }); + const editButton = screen.getByRole("button", { name: /edit settings/i }); await user.click(editButton); const secretField = await screen.findByPlaceholderText( @@ -502,10 +498,10 @@ describe("TeamInfoView", () => { await user.click(membersTab); await waitFor(() => { - expect(screen.getByRole("button", { name: "Add Member" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add member/i })).toBeInTheDocument(); }); - const addButton = screen.getByRole("button", { name: "Add Member" }); + const addButton = screen.getByRole("button", { name: /add member/i }); await user.click(addButton); await waitFor(() => { @@ -603,6 +599,47 @@ describe("TeamInfoView", () => { }); }); + it("should open Settings tab by default when editTeam is true and user can edit", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData()); + + renderWithProviders(); + + await waitFor(() => { + const teamNameElements = screen.queryAllByText("Test Team"); + expect(teamNameElements.length).toBeGreaterThan(0); + }); + + expect(screen.getByText("Team Settings")).toBeInTheDocument(); + }); + + it("should open Overview tab by default when editTeam is false", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData()); + + renderWithProviders(); + + await waitFor(() => { + const teamNameElements = screen.queryAllByText("Test Team"); + expect(teamNameElements.length).toBeGreaterThan(0); + }); + + expect(screen.getByText("Budget Status")).toBeInTheDocument(); + }); + + it("should open Overview tab by default when editTeam is true but user cannot edit", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData()); + + renderWithProviders( + + ); + + await waitFor(() => { + const teamNameElements = screen.queryAllByText("Test Team"); + expect(teamNameElements.length).toBeGreaterThan(0); + }); + + expect(screen.getByText("Budget Status")).toBeInTheDocument(); + }); + it("should display soft budget alerting emails in settings view when present", async () => { const user = userEvent.setup(); vi.mocked(networking.teamInfoCall).mockResolvedValue( diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx new file mode 100644 index 00000000000..d081538fb64 --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -0,0 +1,1294 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import UserSearchModal from "@/components/common_components/user_search_modal"; +import { + getGuardrailsList, + getPoliciesList, + getPolicyInfoWithGuardrails, + Member, + Organization, + organizationInfoCall, + teamInfoCall, + teamMemberAddCall, + teamMemberDeleteCall, + teamMemberUpdateCall, + teamUpdateCall, +} from "@/components/networking"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; +import { isProxyAdminRole } from "@/utils/roles"; +import { EditOutlined, InfoCircleOutlined, SaveOutlined } from "@ant-design/icons"; +import { ArrowLeftIcon } from "@heroicons/react/outline"; +import { Badge, Card, Grid, Text, TextInput, Title } from "@tremor/react"; +import { Button, Form, Input, message, Select, Switch, Tabs, Tooltip } from "antd"; +import { CheckIcon, CopyIcon } from "lucide-react"; +import React, { useEffect, useMemo, useState } from "react"; +import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; +import AgentSelector from "../agent_management/AgentSelector"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import DurationSelect from "../common_components/DurationSelect"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; +import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; +import LoggingSettingsView from "../logging_settings_view"; +import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; +import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; +import { ModelSelect } from "../ModelSelect/ModelSelect"; +import NotificationsManager from "../molecules/notifications_manager"; +import { fetchMCPAccessGroups } from "../networking"; +import ObjectPermissionsView from "../object_permissions_view"; +import NumericalInput from "../shared/numerical_input"; +import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import EditLoggingSettings from "./EditLoggingSettings"; +import MemberModal from "./EditMembership"; +import MemberPermissions from "./member_permissions"; +import { + getTeamInfoDefaultTab, + getTeamInfoVisibleTabs, + TEAM_INFO_TAB_KEYS, + TEAM_INFO_TAB_LABELS, +} from "./tabVisibilityUtils"; +import TeamMembersComponent from "./TeamMemberTab"; + +export interface TeamMembership { + user_id: string; + team_id: string; + budget_id: string; + spend: number; + litellm_budget_table: { + budget_id: string; + soft_budget: number | null; + max_budget: number | null; + max_parallel_requests: number | null; + tpm_limit: number | null; + rpm_limit: number | null; + model_max_budget: Record | null; + budget_duration: string | null; + }; +} + +export interface TeamData { + team_id: string; + team_info: { + team_alias: string; + team_id: string; + organization_id: string | null; + admins: string[]; + members: string[]; + members_with_roles: Member[]; + metadata: Record; + tpm_limit: number | null; + rpm_limit: number | null; + max_budget: number | null; + soft_budget?: number | null; + budget_duration: string | null; + models: string[]; + blocked: boolean; + spend: number; + max_parallel_requests: number | null; + budget_reset_at: string | null; + model_id: string | null; + litellm_model_table: { + model_aliases: Record; + } | null; + created_at: string; + guardrails?: string[]; + policies?: string[]; + object_permission?: { + object_permission_id: string; + mcp_servers: string[]; + mcp_access_groups?: string[]; + mcp_tool_permissions?: Record; + vector_stores: string[]; + agents?: string[]; + agent_access_groups?: string[]; + }; + team_member_budget_table: { + max_budget: number; + budget_duration: string; + tpm_limit: number | null; + rpm_limit: number | null; + } | null; + }; + keys: any[]; + team_memberships: TeamMembership[]; +} + +export interface TeamInfoProps { + teamId: string; + onUpdate: (data: any) => void; + onClose: () => void; + accessToken: string | null; + is_team_admin: boolean; + is_proxy_admin: boolean; + userModels: string[]; + editTeam: boolean; + premiumUser?: boolean; +} + +const getOrganizationModels = (organization: Organization | null, userModels: string[]) => { + let tempModelsToPick = []; + + if (organization) { + // Check if organization has "all-proxy-models" in its models array + if (organization.models.includes("all-proxy-models")) { + // Treat as all-proxy-models (use userModels) + tempModelsToPick = userModels; + } else if (organization.models.length > 0) { + // Organization has specific models + tempModelsToPick = organization.models; + } else { + // Empty array [] is treated as all-proxy-models + tempModelsToPick = userModels; + } + } else { + // No organization, show all available models + tempModelsToPick = userModels; + } + + return unfurlWildcardModelsInList(tempModelsToPick, userModels); +}; + +const TeamInfoView: React.FC = ({ + teamId, + onClose, + accessToken, + is_team_admin, + is_proxy_admin, + userModels, + editTeam, + premiumUser = false, + onUpdate, +}) => { + const [teamData, setTeamData] = useState(null); + const [loading, setLoading] = useState(true); + const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false); + const [form] = Form.useForm(); + const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); + const [selectedEditMember, setSelectedEditMember] = useState(null); + const [isEditing, setIsEditing] = useState(false); + const [mcpAccessGroups, setMcpAccessGroups] = useState([]); + const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); + const [copiedStates, setCopiedStates] = useState>({}); + const [guardrailsList, setGuardrailsList] = useState([]); + const [policiesList, setPoliciesList] = useState([]); + const [policyGuardrails, setPolicyGuardrails] = useState>({}); + const [loadingPolicies, setLoadingPolicies] = useState(false); + const [memberToDelete, setMemberToDelete] = useState(null); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); + const [isTeamSaving, setIsTeamSaving] = useState(false); + const [organization, setOrganization] = useState(null); + const { userRole } = useAuthorized(); + + const canEditTeam = is_team_admin || is_proxy_admin; + const visibleTabs = useMemo(() => getTeamInfoVisibleTabs(canEditTeam), [canEditTeam]); + const defaultTabKey = useMemo( + () => getTeamInfoDefaultTab(editTeam, canEditTeam), + [editTeam, canEditTeam] + ); + + const fetchTeamInfo = async () => { + try { + setLoading(true); + if (!accessToken) return; + const response = await teamInfoCall(accessToken, teamId); + setTeamData(response); + } catch (error) { + NotificationsManager.fromBackend("Failed to load team information"); + console.error("Error fetching team info:", error); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchTeamInfo(); + }, [teamId, accessToken]); + + // Fetch organization data when team has organization_id + useEffect(() => { + const fetchOrganization = async () => { + if (!accessToken || !teamData?.team_info?.organization_id) { + setOrganization(null); + return; + } + + try { + const orgData = await organizationInfoCall(accessToken, teamData.team_info.organization_id); + setOrganization(orgData); + } catch (error) { + console.error("Error fetching organization info:", error); + setOrganization(null); + } + }; + + fetchOrganization(); + }, [accessToken, teamData?.team_info?.organization_id]); + + // Compute modelsToPick based on organization and userModels + const modelsToPick = useMemo(() => { + return getOrganizationModels(organization, userModels); + }, [organization, userModels]); + + const fetchMcpAccessGroups = async () => { + if (!accessToken) return; + if (mcpAccessGroupsLoaded) return; + try { + const groups = await fetchMCPAccessGroups(accessToken); + setMcpAccessGroups(groups); + setMcpAccessGroupsLoaded(true); + } catch (error) { + console.error("Failed to fetch MCP access groups:", error); + } + }; + + useEffect(() => { + const fetchGuardrails = async () => { + try { + if (!accessToken) return; + const response = await getGuardrailsList(accessToken); + const guardrailNames = response.guardrails.map((g: { guardrail_name: string }) => g.guardrail_name); + setGuardrailsList(guardrailNames); + } catch (error) { + console.error("Failed to fetch guardrails:", error); + } + }; + + const fetchPolicies = async () => { + try { + if (!accessToken) return; + const response = await getPoliciesList(accessToken); + const policyNames = response.policies.map((p: { policy_name: string }) => p.policy_name); + setPoliciesList(policyNames); + } catch (error) { + console.error("Failed to fetch policies:", error); + } + }; + + fetchGuardrails(); + fetchPolicies(); + }, [accessToken]); + + // Fetch resolved guardrails for all policies + useEffect(() => { + const fetchPolicyGuardrails = async () => { + if (!accessToken || !teamData?.team_info?.policies || teamData.team_info.policies.length === 0) { + return; + } + + setLoadingPolicies(true); + const guardrailsMap: Record = {}; + + try { + await Promise.all( + teamData.team_info.policies.map(async (policyName: string) => { + try { + const policyInfo = await getPolicyInfoWithGuardrails(accessToken, policyName); + guardrailsMap[policyName] = policyInfo.resolved_guardrails || []; + } catch (error) { + console.error(`Failed to fetch guardrails for policy ${policyName}:`, error); + guardrailsMap[policyName] = []; + } + }) + ); + setPolicyGuardrails(guardrailsMap); + } catch (error) { + console.error("Failed to fetch policy guardrails:", error); + } finally { + setLoadingPolicies(false); + } + }; + + fetchPolicyGuardrails(); + }, [accessToken, teamData?.team_info?.policies]); + + const handleMemberCreate = async (values: any) => { + try { + if (accessToken == null) return; + + const member: Member = { + user_email: values.user_email, + user_id: values.user_id, + role: values.role, + }; + + await teamMemberAddCall(accessToken, teamId, member); + + NotificationsManager.success("Team member added successfully"); + setIsAddMemberModalVisible(false); + form.resetFields(); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error: any) { + let errMsg = "Failed to add team member"; + + if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + + NotificationsManager.fromBackend(errMsg); + console.error("Error adding team member:", error); + } + }; + + const handleMemberUpdate = async (values: any) => { + try { + if (accessToken == null) { + return; + } + + const member: Member = { + user_email: values.user_email, + user_id: values.user_id, + role: values.role, + max_budget_in_team: values.max_budget_in_team, + tpm_limit: values.tpm_limit, + rpm_limit: values.rpm_limit, + }; + message.destroy(); // Remove all existing toasts + + await teamMemberUpdateCall(accessToken, teamId, member); + + NotificationsManager.success("Team member updated successfully"); + setIsEditMemberModalVisible(false); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error: any) { + let errMsg = "Failed to update team member"; + if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + setIsEditMemberModalVisible(false); + + message.destroy(); // Remove all existing toasts + + NotificationsManager.fromBackend(errMsg); + console.error("Error updating team member:", error); + } + }; + + const handleMemberDelete = (member: Member) => { + setMemberToDelete(member); + setIsDeleteModalOpen(true); + }; + + const handleDeleteConfirm = async () => { + if (!memberToDelete || !accessToken) return; + + setIsDeleting(true); + try { + await teamMemberDeleteCall(accessToken, teamId, memberToDelete); + + NotificationsManager.success("Team member removed successfully"); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error) { + NotificationsManager.fromBackend("Failed to remove team member"); + console.error("Error removing team member:", error); + } finally { + setIsDeleting(false); + setIsDeleteModalOpen(false); + setMemberToDelete(null); + } + }; + + const handleDeleteCancel = () => { + setIsDeleteModalOpen(false); + setMemberToDelete(null); + }; + + const handleTeamUpdate = async (values: any) => { + try { + if (!accessToken) return; + setIsTeamSaving(true); + + let parsedMetadata = {}; + try { + const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; + // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately + const { soft_budget_alerting_emails, ...rest } = rawMetadata; + parsedMetadata = rest; + } catch (e) { + NotificationsManager.fromBackend("Invalid JSON in metadata field"); + return; + } + + let secretManagerSettings: Record | undefined; + if (typeof values.secret_manager_settings === "string") { + const trimmedSecretConfig = values.secret_manager_settings.trim(); + if (trimmedSecretConfig.length > 0) { + try { + secretManagerSettings = JSON.parse(values.secret_manager_settings); + } catch (e) { + NotificationsManager.fromBackend("Invalid JSON in secret manager settings"); + return; + } + } + } + + const sanitizeNumeric = (v: any) => { + if (v === null || v === undefined) return null; + if (typeof v === "string" && v.trim() === "") return null; + if (typeof v === "number" && Number.isNaN(v)) return null; + return v; + }; + + const updateData: any = { + team_id: teamId, + team_alias: values.team_alias, + models: values.models, + tpm_limit: sanitizeNumeric(values.tpm_limit), + rpm_limit: sanitizeNumeric(values.rpm_limit), + max_budget: values.max_budget, + soft_budget: sanitizeNumeric(values.soft_budget), + budget_duration: values.budget_duration, + metadata: { + ...parsedMetadata, + ...(values.guardrails?.length > 0 ? { guardrails: values.guardrails } : {}), + ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), + disable_global_guardrails: values.disable_global_guardrails || false, + soft_budget_alerting_emails: + typeof values.soft_budget_alerting_emails === "string" + ? values.soft_budget_alerting_emails + .split(",") + .map((email: string) => email.trim()) + .filter((email: string) => email.length > 0) + : values.soft_budget_alerting_emails || [], + ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), + }, + ...(values.policies?.length > 0 ? { policies: values.policies } : {}), + organization_id: values.organization_id, + }; + + updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); + updateData.team_member_budget_duration = values.team_member_budget_duration; + + if (values.team_member_budget !== undefined) { + updateData.team_member_budget = Number(values.team_member_budget); + } + + if (values.team_member_key_duration !== undefined) { + updateData.team_member_key_duration = values.team_member_key_duration; + } + + if (values.team_member_tpm_limit !== undefined || values.team_member_rpm_limit !== undefined) { + updateData.team_member_tpm_limit = sanitizeNumeric(values.team_member_tpm_limit); + updateData.team_member_rpm_limit = sanitizeNumeric(values.team_member_rpm_limit); + } + + // Handle object_permission updates + const { servers, accessGroups } = values.mcp_servers_and_groups || { + servers: [], + accessGroups: [], + }; + const serverIds = new Set(servers || []); + const mcpToolPermissions = Object.fromEntries( + Object.entries(values.mcp_tool_permissions || {}).filter(([serverId]) => serverIds.has(serverId)), + ); + + updateData.object_permission = {}; + if (servers) { + updateData.object_permission.mcp_servers = servers; + } + if (accessGroups) { + updateData.object_permission.mcp_access_groups = accessGroups; + } + if (mcpToolPermissions) { + updateData.object_permission.mcp_tool_permissions = mcpToolPermissions; + } + delete values.mcp_servers_and_groups; + delete values.mcp_tool_permissions; + + // Handle agent permissions + const { agents, accessGroups: agentAccessGroups } = values.agents_and_groups || { + agents: [], + accessGroups: [], + }; + if (agents && agents.length > 0) { + updateData.object_permission.agents = agents; + } + if (agentAccessGroups && agentAccessGroups.length > 0) { + updateData.object_permission.agent_access_groups = agentAccessGroups; + } + delete values.agents_and_groups; + + // Handle vector stores permissions + if (values.vector_stores && values.vector_stores.length > 0) { + updateData.object_permission.vector_stores = values.vector_stores; + } + + const response = await teamUpdateCall(accessToken, updateData); + + NotificationsManager.success("Team settings updated successfully"); + setIsEditing(false); + fetchTeamInfo(); + } catch (error) { + console.error("Error updating team:", error); + } finally { + setIsTeamSaving(false); + } + }; + + if (loading) { + return
Loading...
; + } + + if (!teamData?.team_info) { + return
Team not found
; + } + + const { team_info: info } = teamData; + + const copyToClipboard = async (text: string, key: string) => { + const success = await utilCopyToClipboard(text); + if (success) { + setCopiedStates((prev) => ({ ...prev, [key]: true })); + setTimeout(() => { + setCopiedStates((prev) => ({ ...prev, [key]: false })); + }, 2000); + } + }; + + return ( +
+
+
+ + {info.team_alias} +
+ {info.team_id} +
+
+
+ + + + Budget Status +
+ ${formatNumberWithCommas(info.spend, 4)} + + of {info.max_budget === null ? "Unlimited" : `$${formatNumberWithCommas(info.max_budget, 4)}`} + + {info.budget_duration && Reset: {info.budget_duration}} +
+ {info.team_member_budget_table && ( + + Team Member Budget: ${formatNumberWithCommas(info.team_member_budget_table.max_budget, 4)} + + )} +
+ + + + Rate Limits +
+ TPM: {info.tpm_limit || "Unlimited"} + RPM: {info.rpm_limit || "Unlimited"} + {info.max_parallel_requests && Max Parallel Requests: {info.max_parallel_requests}} +
+ + + + Models +
+ {info.models.length === 0 ? ( + All proxy models + ) : ( + info.models.map((model, index) => ( + + {model} + + )) + )} +
+ + + + Virtual Keys +
+ User Keys: {teamData.keys.filter((key) => key.user_id).length} + Service Account Keys: {teamData.keys.filter((key) => !key.user_id).length} + Total: {teamData.keys.length} +
+ + + + + + Guardrails + {info.guardrails && info.guardrails.length > 0 ? ( +
+ {info.guardrails.map((guardrail: string, index: number) => ( + + {guardrail} + + ))} +
+ ) : ( + No guardrails configured + )} + {info.metadata?.disable_global_guardrails && ( +
+ Global Guardrails Disabled +
+ )} + + + + Policies + {info.policies && info.policies.length > 0 ? ( +
+ {info.policies.map((policy: string, index: number) => ( +
+
+ {policy} + {loadingPolicies && Loading guardrails...} +
+ {!loadingPolicies && policyGuardrails[policy] && policyGuardrails[policy].length > 0 && ( +
+ Resolved Guardrails: +
+ {policyGuardrails[policy].map((guardrail: string, gIndex: number) => ( + + {guardrail} + + ))} +
+
+ )} +
+ ))} +
+ ) : ( + No policies configured + )} + + + + + ), + }, + { + key: TEAM_INFO_TAB_KEYS.MEMBERS, + label: TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.MEMBERS], + children: ( + + ), + }, + { + key: TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS, + label: TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS], + children: ( + + ), + }, + { + key: TEAM_INFO_TAB_KEYS.SETTINGS, + label: TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.SETTINGS], + children: ( + +
+ Team Settings + {canEditTeam && !isEditing && ( + + )} +
+ + {isEditing ? ( +
rest)(info.metadata), + null, + 2, + ) + : "", + logging_settings: info.metadata?.logging || [], + secret_manager_settings: info.metadata?.secret_manager_settings + ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) + : "", + organization_id: info.organization_id, + vector_stores: info.object_permission?.vector_stores || [], + mcp_servers: info.object_permission?.mcp_servers || [], + mcp_access_groups: info.object_permission?.mcp_access_groups || [], + mcp_servers_and_groups: { + servers: info.object_permission?.mcp_servers || [], + accessGroups: info.object_permission?.mcp_access_groups || [], + }, + mcp_tool_permissions: info.object_permission?.mcp_tool_permissions || {}, + agents_and_groups: { + agents: info.object_permission?.agents || [], + accessGroups: info.object_permission?.agent_access_groups || [], + }, + }} + layout="vertical" + > + + + + + + form.setFieldValue("models", values)} + teamID={teamId} + organizationID={teamData?.team_info?.organization_id || undefined} + options={{ + includeSpecialOptions: true, + includeUserModels: !teamData?.team_info?.organization_id, + showAllProxyModelsOverride: isProxyAdminRole(userRole) && !teamData?.team_info?.organization_id, + }} + context="team" + dataTestId="models-select" + /> + + + + + + + + + + + + + + + + + + + + form.setFieldValue("team_member_budget_duration", value)} + value={form.getFieldValue("team_member_budget_duration")} + /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + Guardrails{" "} + + e.stopPropagation()} + > + + + + + } + name="guardrails" + help="Select existing guardrails or enter new ones" + > + ({ value: name, label: name }))} + /> + + + + form.setFieldValue("vector_stores", values)} + value={form.getFieldValue("vector_stores")} + accessToken={accessToken || ""} + placeholder="Select vector stores" + /> + + + + form.setFieldValue("allowed_passthrough_routes", values)} + value={form.getFieldValue("allowed_passthrough_routes")} + accessToken={accessToken || ""} + placeholder="Select pass through routes" + /> + + + + form.setFieldValue("mcp_servers_and_groups", val)} + value={form.getFieldValue("mcp_servers_and_groups")} + accessToken={accessToken || ""} + placeholder="Select MCP servers or access groups (optional)" + /> + + + {/* Hidden field to register mcp_tool_permissions with the form */} + + + + prevValues.mcp_servers_and_groups !== currentValues.mcp_servers_and_groups || + prevValues.mcp_tool_permissions !== currentValues.mcp_tool_permissions + } + > + {() => ( +
+ form.setFieldsValue({ mcp_tool_permissions: toolPerms })} + /> +
+ )} + + + + form.setFieldValue("agents_and_groups", val)} + value={form.getFieldValue("agents_and_groups")} + accessToken={accessToken || ""} + placeholder="Select agents or access groups (optional)" + /> + + + + + + + + form.setFieldValue("logging_settings", values)} + /> + + + { + if (!value) { + return Promise.resolve(); + } + try { + JSON.parse(value); + return Promise.resolve(); + } catch (error) { + return Promise.reject(new Error("Please enter valid JSON")); + } + }, + }, + ]} + > + + + + + + + +
+
+ + +
+
+
+ ) : ( +
+
+ Team Name +
{info.team_alias}
+
+
+ Team ID +
{info.team_id}
+
+
+ Created At +
{new Date(info.created_at).toLocaleString()}
+
+
+ Models +
+ {info.models.map((model, index) => ( + + {model} + + ))} +
+
+
+ Rate Limits +
TPM: {info.tpm_limit || "Unlimited"}
+
RPM: {info.rpm_limit || "Unlimited"}
+
+
+ Team Budget +
+ Max Budget:{" "} + {info.max_budget !== null ? `$${formatNumberWithCommas(info.max_budget, 4)}` : "No Limit"} +
+
+ Soft Budget:{" "} + {info.soft_budget !== null && info.soft_budget !== undefined + ? `$${formatNumberWithCommas(info.soft_budget, 4)}` + : "No Limit"} +
+
Budget Reset: {info.budget_duration || "Never"}
+ {info.metadata?.soft_budget_alerting_emails && + Array.isArray(info.metadata.soft_budget_alerting_emails) && + info.metadata.soft_budget_alerting_emails.length > 0 && ( +
+ Soft Budget Alerting Emails: {info.metadata.soft_budget_alerting_emails.join(", ")} +
+ )} +
+
+ + Team Member Settings{" "} + + + + +
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
+
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
+
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
+
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
+
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
+
+
+ Organization ID +
{info.organization_id}
+
+
+ Status + {info.blocked ? "Blocked" : "Active"} +
+ +
+ Disable Global Guardrails +
+ {info.metadata?.disable_global_guardrails === true ? ( + Enabled - Global guardrails bypassed + ) : ( + Disabled - Global guardrails active + )} +
+
+ + + + + + {info.metadata?.secret_manager_settings && ( +
+ Secret Manager Settings + 
+                          {JSON.stringify(info.metadata.secret_manager_settings, null, 2)}
+
+
+ )} +
+ )} + + ), + }, + ].filter(tab => visibleTabs.includes(tab.key))} + /> + + setIsEditMemberModalVisible(false)} + onSubmit={handleMemberUpdate} + initialData={selectedEditMember} + mode="edit" + config={{ + title: "Edit Member", + showEmail: true, + showUserId: true, + roleOptions: [ + { label: "Admin", value: "admin" }, + { label: "User", value: "user" }, + ], + additionalFields: [ + { + name: "max_budget_in_team", + label: ( + + Team Member Budget (USD){" "} + + + + + ), + type: "numerical" as const, + step: 0.01, + min: 0, + placeholder: "Budget limit for this member within this team", + }, + { + name: "tpm_limit", + label: ( + + Team Member TPM Limit{" "} + + + + + ), + type: "numerical" as const, + step: 1, + min: 0, + placeholder: "Tokens per minute limit for this member in this team", + }, + { + name: "rpm_limit", + label: ( + + Team Member RPM Limit{" "} + + + + + ), + type: "numerical" as const, + step: 1, + min: 0, + placeholder: "Requests per minute limit for this member in this team", + }, + ], + }} + /> + + setIsAddMemberModalVisible(false)} + onSubmit={handleMemberCreate} + accessToken={accessToken} + /> + + {/* Delete Member Confirmation Modal */} + +
+ ); +}; + +export default TeamInfoView; diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx new file mode 100644 index 00000000000..d494b874e4d --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -0,0 +1,386 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../../tests/test-utils"; +import { TeamData } from "./TeamInfo"; +import TeamMembersComponent from "./TeamMemberTab"; + +vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ + useUISettings: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(), +})); + +vi.mock("@/utils/roles", () => ({ + isUserTeamAdminForSingleTeam: vi.fn(() => false), + isProxyAdminRole: vi.fn(() => false), +})); + +import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; + +const mockHandleMemberDelete = vi.fn(); +const mockSetSelectedEditMember = vi.fn(); +const mockSetIsEditMemberModalVisible = vi.fn(); +const mockSetIsAddMemberModalVisible = vi.fn(); + +const createMockTeamData = (overrides: Partial = {}): TeamData => ({ + team_id: "team-123", + team_info: { + team_alias: "Test Team", + team_id: "team-123", + organization_id: null, + admins: ["admin@test.com"], + members: ["user1@test.com"], + members_with_roles: [ + { + user_id: "user1@test.com", + user_email: "user1@test.com", + role: "member", + }, + { + user_id: "user2@test.com", + user_email: "user2@test.com", + role: "admin", + }, + ], + metadata: {}, + tpm_limit: null, + rpm_limit: null, + max_budget: null, + budget_duration: null, + models: [], + blocked: false, + spend: 0, + max_parallel_requests: null, + budget_reset_at: null, + model_id: null, + litellm_model_table: null, + created_at: "2024-01-01T00:00:00Z", + team_member_budget_table: null, + }, + keys: [], + team_memberships: [ + { + user_id: "user1@test.com", + team_id: "team-123", + budget_id: "budget1", + spend: 100.5, + litellm_budget_table: { + budget_id: "budget1", + soft_budget: null, + max_budget: 1000, + max_parallel_requests: null, + tpm_limit: 10000, + rpm_limit: 100, + model_max_budget: null, + budget_duration: null, + }, + }, + ], + ...overrides, +}); + +describe("TeamMembersComponent", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(useUISettings).mockReturnValue({ + data: { values: { disable_team_admin_delete_team_user: false } }, + isLoading: false, + isError: false, + error: null, + isSuccess: true, + isFetching: false, + refetch: vi.fn(), + } as any); + + vi.mocked(useAuthorized).mockReturnValue({ + isLoading: false, + isAuthorized: true, + userId: "test-user-id", + userRole: "Admin", + accessToken: "test-token", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + }); + + it("should render", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + }); + + it("should render team members table with headers", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("columnheader", { name: /user email/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /user id/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /team role/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /actions/i })).toBeInTheDocument(); + }); + + it("should render team members data", () => { + renderWithProviders( + , + ); + + // user1@test.com appears twice (User ID and User Email columns) + expect(screen.getAllByText("user1@test.com").length).toBeGreaterThanOrEqual(1); + expect(screen.getAllByText("user2@test.com").length).toBeGreaterThanOrEqual(1); + const table = screen.getByRole("table"); + expect(table).toHaveTextContent("member"); + expect(table).toHaveTextContent("admin"); + }); + + it("should render Add Member button", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("Add Member")).toBeInTheDocument(); + }); + + it("should display dash when user email is null", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should display Default Proxy Admin tag for default_user_id", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("Default Proxy Admin")).toBeInTheDocument(); + }); + + it("should display spend and rate limits for member with membership", () => { + renderWithProviders( + , + ); + + expect(screen.getByText(/\$100\.5/)).toBeInTheDocument(); + expect(screen.getByText(/100 RPM/)).toBeInTheDocument(); + expect(screen.getByText(/10000 TPM/)).toBeInTheDocument(); + }); + + it("should display No Limit for budget when member has no budget", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("No Limit")).toBeInTheDocument(); + }); + + it("should display No Limits for rate limits when member has no limits", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("No Limits")).toBeInTheDocument(); + }); + + it("should call setIsEditMemberModalVisible and setSelectedEditMember when edit button is clicked", async () => { + const user = userEvent.setup(); + vi.mocked(isProxyAdminRole).mockReturnValue(true); + vi.mocked(isUserTeamAdminForSingleTeam).mockReturnValue(false); + + renderWithProviders( + , + ); + + const editButtons = screen.getAllByTestId("edit-member"); + await user.click(editButtons[0]); + + expect(mockSetIsEditMemberModalVisible).toHaveBeenCalledWith(true); + expect(mockSetSelectedEditMember).toHaveBeenCalled(); + }); + + it("should call setIsAddMemberModalVisible when Add Member button is clicked", async () => { + const user = userEvent.setup(); + + renderWithProviders( + , + ); + + const addButton = screen.getByRole("button", { name: /add member/i }); + await user.click(addButton); + + expect(mockSetIsAddMemberModalVisible).toHaveBeenCalledWith(true); + }); + + it("should hide delete button when disable_team_admin_delete_team_user is true and user is team admin", () => { + vi.mocked(isProxyAdminRole).mockReturnValue(false); + vi.mocked(isUserTeamAdminForSingleTeam).mockReturnValue(true); + vi.mocked(useUISettings).mockReturnValue({ + data: { values: { disable_team_admin_delete_team_user: true } }, + isLoading: false, + isError: false, + error: null, + isSuccess: true, + isFetching: false, + refetch: vi.fn(), + } as any); + + renderWithProviders( + , + ); + + expect(screen.queryByTestId("delete-member")).not.toBeInTheDocument(); + expect(screen.getAllByTestId("edit-member")).toHaveLength(2); + }); + + it("should show delete button for proxy admin when canEditTeam is true", () => { + vi.mocked(isProxyAdminRole).mockReturnValue(true); + vi.mocked(isUserTeamAdminForSingleTeam).mockReturnValue(false); + + renderWithProviders( + , + ); + + expect(screen.getAllByTestId("delete-member")).toHaveLength(2); + expect(screen.getAllByTestId("edit-member")).toHaveLength(2); + }); + + it("should hide action buttons when canEditTeam is false", () => { + renderWithProviders( + , + ); + + expect(screen.queryByTestId("edit-member")).not.toBeInTheDocument(); + expect(screen.queryByTestId("delete-member")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx new file mode 100644 index 00000000000..09a6b0f468d --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -0,0 +1,233 @@ +import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Member } from "@/components/networking"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; +import { CrownOutlined, InfoCircleOutlined, UserAddOutlined, UserOutlined } from "@ant-design/icons"; +import { Button, Space, Table, Tag, Tooltip, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { TeamData } from "./TeamInfo"; + +const { Text } = Typography; + +interface TeamMemberTabProps { + teamData: TeamData; + canEditTeam: boolean; + handleMemberDelete: (member: Member) => void; + setSelectedEditMember: (member: Member) => void; + setIsEditMemberModalVisible: (visible: boolean) => void; + setIsAddMemberModalVisible: (visible: boolean) => void; +} + +export default function TeamMemberTab({ + teamData, + canEditTeam, + handleMemberDelete, + setSelectedEditMember, + setIsEditMemberModalVisible, + setIsAddMemberModalVisible, +}: TeamMemberTabProps) { + const formatNumber = (value: number | null): string => { + if (value === null || value === undefined) return "0"; + + if (typeof value === "number") { + // Convert scientific notation to normal decimal + const normalNumber = Number(value); + + // If it's a whole number, return it without decimals + if (normalNumber === Math.floor(normalNumber)) { + return normalNumber.toString(); + } + + // For decimal numbers, use toFixed and remove trailing zeros + return formatNumberWithCommas(normalNumber, 8).replace(/\.?0+$/, ""); + } + + return "0"; + }; + + // Helper function to get spend for a user + const getUserSpend = (userId: string | null): number | null => { + if (!userId) return 0; + const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); + return membership?.spend || 0; + }; + + const getUserBudget = (userId: string | null): string | null => { + if (!userId) return null; + const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); + const maxBudget = membership?.litellm_budget_table?.max_budget; + if (maxBudget === null || maxBudget === undefined) { + return null; + } + return formatNumber(maxBudget); + }; + + // Helper function to get rate limits for a user + const getUserRateLimits = (userId: string | null): string => { + if (!userId) return "No Limits"; + const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); + const rpmLimit = membership?.litellm_budget_table?.rpm_limit; + const tpmLimit = membership?.litellm_budget_table?.tpm_limit; + + const rpmText = rpmLimit ? `${formatNumber(rpmLimit)} RPM` : null; + const tpmText = tpmLimit ? `${formatNumber(tpmLimit)} TPM` : null; + + const limits = [rpmText, tpmText].filter(Boolean); + return limits.length > 0 ? limits.join(" / ") : "No Limits"; + }; + + const { data: uiSettingsData } = useUISettings(); + const { userId, userRole } = useAuthorized(); + const disableTeamAdminDeleteTeamUser = Boolean(uiSettingsData?.values?.disable_team_admin_delete_team_user); + const isUserTeamAdmin = isUserTeamAdminForSingleTeam(teamData.team_info.members_with_roles, userId || ""); + const isProxyAdmin = isProxyAdminRole(userRole || ""); + + const columns: ColumnsType = [ + { + title: "User Email", + dataIndex: "user_email", + key: "user_email", + render: (email: string | null) => ( + {email || "-"} + ), + }, + { + title: "User ID", + dataIndex: "user_id", + key: "user_id", + render: (userId: string | null) => + userId === "default_user_id" ? ( + Default Proxy Admin + ) : ( + {userId} + ), + }, + { + title: ( + + Team Role + + + + + ), + dataIndex: "role", + key: "role", + render: (role: string) => ( + + {role?.toLowerCase() === "admin" ? ( + + ) : ( + + )} + {role} + + ), + }, + { + title: ( + + Team Member Spend (USD) + + + + + ), + key: "spend", + render: (_: unknown, record: Member) => ( + + ${formatNumberWithCommas(getUserSpend(record.user_id), 4)} + + ), + }, + { + title: "Team Member Budget (USD)", + key: "budget", + render: (_: unknown, record: Member) => { + const budget = getUserBudget(record.user_id); + return ( + + {budget ? `$${formatNumberWithCommas(Number(budget), 4)}` : "No Limit"} + + ); + }, + }, + { + title: ( + + Team Member Rate Limits + + + + + ), + key: "rate_limits", + render: (_: unknown, record: Member) => ( + {getUserRateLimits(record.user_id)} + ), + }, + { + title: "Actions", + key: "actions", + fixed: "right", + width: 120, + render: (_: unknown, record: Member) => + canEditTeam ? ( +
+ { + const membership = teamData.team_memberships.find( + (tm) => tm.user_id === record.user_id + ); + const enhancedMember = { + ...record, + max_budget_in_team: + membership?.litellm_budget_table?.max_budget || null, + tpm_limit: + membership?.litellm_budget_table?.tpm_limit || null, + rpm_limit: + membership?.litellm_budget_table?.rpm_limit || null, + }; + setSelectedEditMember(enhancedMember); + setIsEditMemberModalVisible(true); + }} + /> + {(isProxyAdmin || + (isUserTeamAdmin && !disableTeamAdminDeleteTeamUser)) && ( + handleMemberDelete(record)} + /> + )} +
+ ) : null, + }, + ]; + + return ( +
+ record.user_id || String(index)} + pagination={false} + size="small" + scroll={{ x: "max-content" }} + /> + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts new file mode 100644 index 00000000000..5a09b4fa36c --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { + getTeamInfoDefaultTab, + getTeamInfoVisibleTabs, + isTeamInfoTabVisible, + TEAM_INFO_TAB_KEYS, + TEAM_INFO_TAB_LABELS, +} from "./tabVisibilityUtils"; + +describe("team_info_tabs", () => { + describe("TEAM_INFO_TAB_LABELS", () => { + it("should have label for every tab key", () => { + expect(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.OVERVIEW]).toBe("Overview"); + expect(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.MEMBERS]).toBe("Members"); + expect(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS]).toBe("Member Permissions"); + expect(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.SETTINGS]).toBe("Settings"); + }); + }); + + describe("getTeamInfoVisibleTabs", () => { + it("returns only overview when user cannot edit team", () => { + const tabs = getTeamInfoVisibleTabs(false); + expect(tabs).toEqual([TEAM_INFO_TAB_KEYS.OVERVIEW]); + }); + + it("returns all tabs when user can edit team", () => { + const tabs = getTeamInfoVisibleTabs(true); + expect(tabs).toEqual([ + TEAM_INFO_TAB_KEYS.OVERVIEW, + TEAM_INFO_TAB_KEYS.MEMBERS, + TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS, + TEAM_INFO_TAB_KEYS.SETTINGS, + ]); + }); + }); + + describe("getTeamInfoDefaultTab", () => { + it("returns overview when editTeam is false", () => { + expect(getTeamInfoDefaultTab(false, true)).toBe(TEAM_INFO_TAB_KEYS.OVERVIEW); + expect(getTeamInfoDefaultTab(false, false)).toBe(TEAM_INFO_TAB_KEYS.OVERVIEW); + }); + + it("returns settings when editTeam is true and user can edit", () => { + expect(getTeamInfoDefaultTab(true, true)).toBe(TEAM_INFO_TAB_KEYS.SETTINGS); + }); + + it("returns overview when editTeam is true but user cannot edit", () => { + expect(getTeamInfoDefaultTab(true, false)).toBe(TEAM_INFO_TAB_KEYS.OVERVIEW); + }); + }); + + describe("isTeamInfoTabVisible", () => { + it("always returns true for overview tab", () => { + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.OVERVIEW, false)).toBe(true); + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.OVERVIEW, true)).toBe(true); + }); + + it("returns false for members tab when user cannot edit", () => { + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.MEMBERS, false)).toBe(false); + }); + + it("returns true for members tab when user can edit", () => { + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.MEMBERS, true)).toBe(true); + }); + + it("returns false for settings tab when user cannot edit", () => { + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.SETTINGS, false)).toBe(false); + }); + + it("returns true for settings tab when user can edit", () => { + expect(isTeamInfoTabVisible(TEAM_INFO_TAB_KEYS.SETTINGS, true)).toBe(true); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts new file mode 100644 index 00000000000..d77230ea09b --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts @@ -0,0 +1,59 @@ +/** + * Team info tab configuration and permission logic. + * Extracted for testability - permission rules can be unit tested in isolation. + */ + +export const TEAM_INFO_TAB_KEYS = { + OVERVIEW: "overview", + MEMBERS: "members", + MEMBER_PERMISSIONS: "member-permissions", + SETTINGS: "settings", +} as const; + +export const TEAM_INFO_TAB_LABELS: Record = { + [TEAM_INFO_TAB_KEYS.OVERVIEW]: "Overview", + [TEAM_INFO_TAB_KEYS.MEMBERS]: "Members", + [TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS]: "Member Permissions", + [TEAM_INFO_TAB_KEYS.SETTINGS]: "Settings", +}; + +/** + * Returns the list of tab keys that should be visible based on permissions. + * - Overview: always visible + * - Members, Member Permissions, Settings: only when canEditTeam is true + */ +export function getTeamInfoVisibleTabs(canEditTeam: boolean): readonly string[] { + const baseTabs = [TEAM_INFO_TAB_KEYS.OVERVIEW]; + if (canEditTeam) { + return [ + ...baseTabs, + TEAM_INFO_TAB_KEYS.MEMBERS, + TEAM_INFO_TAB_KEYS.MEMBER_PERMISSIONS, + TEAM_INFO_TAB_KEYS.SETTINGS, + ]; + } + return baseTabs; +} + +/** + * Returns the default active tab key based on permissions and edit intent. + * - When editTeam is true and user can edit: open Settings tab + * - Otherwise: open Overview tab + */ +export function getTeamInfoDefaultTab(editTeam: boolean, canEditTeam: boolean): string { + if (editTeam && canEditTeam) { + return TEAM_INFO_TAB_KEYS.SETTINGS; + } + return TEAM_INFO_TAB_KEYS.OVERVIEW; +} + +/** + * Checks if a specific tab should be visible based on permissions. + */ +export function isTeamInfoTabVisible( + tabKey: string, + canEditTeam: boolean +): boolean { + const visibleTabs = getTeamInfoVisibleTabs(canEditTeam); + return visibleTabs.includes(tabKey); +} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx deleted file mode 100644 index 014f8fb9010..00000000000 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ /dev/null @@ -1,1296 +0,0 @@ -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import UserSearchModal from "@/components/common_components/user_search_modal"; -import { - getGuardrailsList, - getPoliciesList, - getPolicyInfoWithGuardrails, - Member, - Organization, - organizationInfoCall, - teamInfoCall, - teamMemberAddCall, - teamMemberDeleteCall, - teamMemberUpdateCall, - teamUpdateCall, -} from "@/components/networking"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; -import { isProxyAdminRole } from "@/utils/roles"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { ArrowLeftIcon } from "@heroicons/react/outline"; -import { - Badge, - Card, - Grid, - Tab, - TabGroup, - TabList, - TabPanel, - TabPanels, - Text, - TextInput, - Title, - Button as TremorButton, -} from "@tremor/react"; -import { Button, Form, Input, message, Select, Switch, Tooltip } from "antd"; -import { CheckIcon, CopyIcon } from "lucide-react"; -import React, { useEffect, useMemo, useState } from "react"; -import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; -import AgentSelector from "../agent_management/AgentSelector"; -import DeleteResourceModal from "../common_components/DeleteResourceModal"; -import DurationSelect from "../common_components/DurationSelect"; -import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; -import LoggingSettingsView from "../logging_settings_view"; -import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; -import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; -import { ModelSelect } from "../ModelSelect/ModelSelect"; -import NotificationsManager from "../molecules/notifications_manager"; -import { fetchMCPAccessGroups } from "../networking"; -import ObjectPermissionsView from "../object_permissions_view"; -import NumericalInput from "../shared/numerical_input"; -import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; -import EditLoggingSettings from "./EditLoggingSettings"; -import MemberModal from "./EditMembership"; -import MemberPermissions from "./member_permissions"; -import TeamMembersComponent from "./team_member_view"; - -export interface TeamMembership { - user_id: string; - team_id: string; - budget_id: string; - spend: number; - litellm_budget_table: { - budget_id: string; - soft_budget: number | null; - max_budget: number | null; - max_parallel_requests: number | null; - tpm_limit: number | null; - rpm_limit: number | null; - model_max_budget: Record | null; - budget_duration: string | null; - }; -} - -export interface TeamData { - team_id: string; - team_info: { - team_alias: string; - team_id: string; - organization_id: string | null; - admins: string[]; - members: string[]; - members_with_roles: Member[]; - metadata: Record; - tpm_limit: number | null; - rpm_limit: number | null; - max_budget: number | null; - soft_budget?: number | null; - budget_duration: string | null; - models: string[]; - blocked: boolean; - spend: number; - max_parallel_requests: number | null; - budget_reset_at: string | null; - model_id: string | null; - litellm_model_table: { - model_aliases: Record; - } | null; - created_at: string; - guardrails?: string[]; - policies?: string[]; - object_permission?: { - object_permission_id: string; - mcp_servers: string[]; - mcp_access_groups?: string[]; - mcp_tool_permissions?: Record; - vector_stores: string[]; - agents?: string[]; - agent_access_groups?: string[]; - }; - team_member_budget_table: { - max_budget: number; - budget_duration: string; - tpm_limit: number | null; - rpm_limit: number | null; - } | null; - }; - keys: any[]; - team_memberships: TeamMembership[]; -} - -export interface TeamInfoProps { - teamId: string; - onUpdate: (data: any) => void; - onClose: () => void; - accessToken: string | null; - is_team_admin: boolean; - is_proxy_admin: boolean; - userModels: string[]; - editTeam: boolean; - premiumUser?: boolean; -} - -const getOrganizationModels = (organization: Organization | null, userModels: string[]) => { - let tempModelsToPick = []; - - if (organization) { - // Check if organization has "all-proxy-models" in its models array - if (organization.models.includes("all-proxy-models")) { - // Treat as all-proxy-models (use userModels) - tempModelsToPick = userModels; - } else if (organization.models.length > 0) { - // Organization has specific models - tempModelsToPick = organization.models; - } else { - // Empty array [] is treated as all-proxy-models - tempModelsToPick = userModels; - } - } else { - // No organization, show all available models - tempModelsToPick = userModels; - } - - return unfurlWildcardModelsInList(tempModelsToPick, userModels); -}; - -const TeamInfoView: React.FC = ({ - teamId, - onClose, - accessToken, - is_team_admin, - is_proxy_admin, - userModels, - editTeam, - premiumUser = false, - onUpdate, -}) => { - const [teamData, setTeamData] = useState(null); - const [loading, setLoading] = useState(true); - const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false); - const [form] = Form.useForm(); - const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); - const [selectedEditMember, setSelectedEditMember] = useState(null); - const [isEditing, setIsEditing] = useState(false); - const [mcpAccessGroups, setMcpAccessGroups] = useState([]); - const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); - const [copiedStates, setCopiedStates] = useState>({}); - const [guardrailsList, setGuardrailsList] = useState([]); - const [policiesList, setPoliciesList] = useState([]); - const [policyGuardrails, setPolicyGuardrails] = useState>({}); - const [loadingPolicies, setLoadingPolicies] = useState(false); - const [memberToDelete, setMemberToDelete] = useState(null); - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [isDeleting, setIsDeleting] = useState(false); - const [isTeamSaving, setIsTeamSaving] = useState(false); - const [organization, setOrganization] = useState(null); - const { userRole } = useAuthorized(); - - const canEditTeam = is_team_admin || is_proxy_admin; - - const fetchTeamInfo = async () => { - try { - setLoading(true); - if (!accessToken) return; - const response = await teamInfoCall(accessToken, teamId); - setTeamData(response); - } catch (error) { - NotificationsManager.fromBackend("Failed to load team information"); - console.error("Error fetching team info:", error); - } finally { - setLoading(false); - } - }; - - useEffect(() => { - fetchTeamInfo(); - }, [teamId, accessToken]); - - // Fetch organization data when team has organization_id - useEffect(() => { - const fetchOrganization = async () => { - if (!accessToken || !teamData?.team_info?.organization_id) { - setOrganization(null); - return; - } - - try { - const orgData = await organizationInfoCall(accessToken, teamData.team_info.organization_id); - setOrganization(orgData); - } catch (error) { - console.error("Error fetching organization info:", error); - setOrganization(null); - } - }; - - fetchOrganization(); - }, [accessToken, teamData?.team_info?.organization_id]); - - // Compute modelsToPick based on organization and userModels - const modelsToPick = useMemo(() => { - return getOrganizationModels(organization, userModels); - }, [organization, userModels]); - - const fetchMcpAccessGroups = async () => { - if (!accessToken) return; - if (mcpAccessGroupsLoaded) return; - try { - const groups = await fetchMCPAccessGroups(accessToken); - setMcpAccessGroups(groups); - setMcpAccessGroupsLoaded(true); - } catch (error) { - console.error("Failed to fetch MCP access groups:", error); - } - }; - - useEffect(() => { - const fetchGuardrails = async () => { - try { - if (!accessToken) return; - const response = await getGuardrailsList(accessToken); - const guardrailNames = response.guardrails.map((g: { guardrail_name: string }) => g.guardrail_name); - setGuardrailsList(guardrailNames); - } catch (error) { - console.error("Failed to fetch guardrails:", error); - } - }; - - const fetchPolicies = async () => { - try { - if (!accessToken) return; - const response = await getPoliciesList(accessToken); - const policyNames = response.policies.map((p: { policy_name: string }) => p.policy_name); - setPoliciesList(policyNames); - } catch (error) { - console.error("Failed to fetch policies:", error); - } - }; - - fetchGuardrails(); - fetchPolicies(); - }, [accessToken]); - - // Fetch resolved guardrails for all policies - useEffect(() => { - const fetchPolicyGuardrails = async () => { - if (!accessToken || !teamData?.team_info?.policies || teamData.team_info.policies.length === 0) { - return; - } - - setLoadingPolicies(true); - const guardrailsMap: Record = {}; - - try { - await Promise.all( - teamData.team_info.policies.map(async (policyName: string) => { - try { - const policyInfo = await getPolicyInfoWithGuardrails(accessToken, policyName); - guardrailsMap[policyName] = policyInfo.resolved_guardrails || []; - } catch (error) { - console.error(`Failed to fetch guardrails for policy ${policyName}:`, error); - guardrailsMap[policyName] = []; - } - }) - ); - setPolicyGuardrails(guardrailsMap); - } catch (error) { - console.error("Failed to fetch policy guardrails:", error); - } finally { - setLoadingPolicies(false); - } - }; - - fetchPolicyGuardrails(); - }, [accessToken, teamData?.team_info?.policies]); - - const handleMemberCreate = async (values: any) => { - try { - if (accessToken == null) return; - - const member: Member = { - user_email: values.user_email, - user_id: values.user_id, - role: values.role, - }; - - await teamMemberAddCall(accessToken, teamId, member); - - NotificationsManager.success("Team member added successfully"); - setIsAddMemberModalVisible(false); - form.resetFields(); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error: any) { - let errMsg = "Failed to add team member"; - - if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { - errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; - } else if (error?.message) { - errMsg = error.message; - } - - NotificationsManager.fromBackend(errMsg); - console.error("Error adding team member:", error); - } - }; - - const handleMemberUpdate = async (values: any) => { - try { - if (accessToken == null) { - return; - } - - const member: Member = { - user_email: values.user_email, - user_id: values.user_id, - role: values.role, - max_budget_in_team: values.max_budget_in_team, - tpm_limit: values.tpm_limit, - rpm_limit: values.rpm_limit, - }; - console.log("Updating member with values:", member); - message.destroy(); // Remove all existing toasts - - await teamMemberUpdateCall(accessToken, teamId, member); - - NotificationsManager.success("Team member updated successfully"); - setIsEditMemberModalVisible(false); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error: any) { - let errMsg = "Failed to update team member"; - if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { - errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; - } else if (error?.message) { - errMsg = error.message; - } - setIsEditMemberModalVisible(false); - - message.destroy(); // Remove all existing toasts - - NotificationsManager.fromBackend(errMsg); - console.error("Error updating team member:", error); - } - }; - - const handleMemberDelete = (member: Member) => { - setMemberToDelete(member); - setIsDeleteModalOpen(true); - }; - - const handleDeleteConfirm = async () => { - if (!memberToDelete || !accessToken) return; - - setIsDeleting(true); - try { - await teamMemberDeleteCall(accessToken, teamId, memberToDelete); - - NotificationsManager.success("Team member removed successfully"); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error) { - NotificationsManager.fromBackend("Failed to remove team member"); - console.error("Error removing team member:", error); - } finally { - setIsDeleting(false); - setIsDeleteModalOpen(false); - setMemberToDelete(null); - } - }; - - const handleDeleteCancel = () => { - setIsDeleteModalOpen(false); - setMemberToDelete(null); - }; - - const handleTeamUpdate = async (values: any) => { - try { - if (!accessToken) return; - setIsTeamSaving(true); - - let parsedMetadata = {}; - try { - const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; - // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately - const { soft_budget_alerting_emails, ...rest } = rawMetadata; - parsedMetadata = rest; - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in metadata field"); - return; - } - - let secretManagerSettings: Record | undefined; - if (typeof values.secret_manager_settings === "string") { - const trimmedSecretConfig = values.secret_manager_settings.trim(); - if (trimmedSecretConfig.length > 0) { - try { - secretManagerSettings = JSON.parse(values.secret_manager_settings); - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in secret manager settings"); - return; - } - } - } - - const sanitizeNumeric = (v: any) => { - if (v === null || v === undefined) return null; - if (typeof v === "string" && v.trim() === "") return null; - if (typeof v === "number" && Number.isNaN(v)) return null; - return v; - }; - - const updateData: any = { - team_id: teamId, - team_alias: values.team_alias, - models: values.models, - tpm_limit: sanitizeNumeric(values.tpm_limit), - rpm_limit: sanitizeNumeric(values.rpm_limit), - max_budget: values.max_budget, - soft_budget: sanitizeNumeric(values.soft_budget), - budget_duration: values.budget_duration, - metadata: { - ...parsedMetadata, - ...(values.guardrails?.length > 0 ? { guardrails: values.guardrails } : {}), - ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), - disable_global_guardrails: values.disable_global_guardrails || false, - soft_budget_alerting_emails: - typeof values.soft_budget_alerting_emails === "string" - ? values.soft_budget_alerting_emails - .split(",") - .map((email: string) => email.trim()) - .filter((email: string) => email.length > 0) - : values.soft_budget_alerting_emails || [], - ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), - }, - ...(values.policies?.length > 0 ? { policies: values.policies } : {}), - organization_id: values.organization_id, - }; - - updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); - updateData.team_member_budget_duration = values.team_member_budget_duration; - - if (values.team_member_budget !== undefined) { - updateData.team_member_budget = Number(values.team_member_budget); - } - - if (values.team_member_key_duration !== undefined) { - updateData.team_member_key_duration = values.team_member_key_duration; - } - - if (values.team_member_tpm_limit !== undefined || values.team_member_rpm_limit !== undefined) { - updateData.team_member_tpm_limit = sanitizeNumeric(values.team_member_tpm_limit); - updateData.team_member_rpm_limit = sanitizeNumeric(values.team_member_rpm_limit); - } - - // Handle object_permission updates - const { servers, accessGroups } = values.mcp_servers_and_groups || { - servers: [], - accessGroups: [], - }; - const serverIds = new Set(servers || []); - const mcpToolPermissions = Object.fromEntries( - Object.entries(values.mcp_tool_permissions || {}).filter(([serverId]) => serverIds.has(serverId)), - ); - - updateData.object_permission = {}; - if (servers) { - updateData.object_permission.mcp_servers = servers; - } - if (accessGroups) { - updateData.object_permission.mcp_access_groups = accessGroups; - } - if (mcpToolPermissions) { - updateData.object_permission.mcp_tool_permissions = mcpToolPermissions; - } - delete values.mcp_servers_and_groups; - delete values.mcp_tool_permissions; - - // Handle agent permissions - const { agents, accessGroups: agentAccessGroups } = values.agents_and_groups || { - agents: [], - accessGroups: [], - }; - if (agents && agents.length > 0) { - updateData.object_permission.agents = agents; - } - if (agentAccessGroups && agentAccessGroups.length > 0) { - updateData.object_permission.agent_access_groups = agentAccessGroups; - } - delete values.agents_and_groups; - - // Handle vector stores permissions - if (values.vector_stores && values.vector_stores.length > 0) { - updateData.object_permission.vector_stores = values.vector_stores; - } - - const response = await teamUpdateCall(accessToken, updateData); - - NotificationsManager.success("Team settings updated successfully"); - setIsEditing(false); - fetchTeamInfo(); - } catch (error) { - console.error("Error updating team:", error); - } finally { - setIsTeamSaving(false); - } - }; - - if (loading) { - return
Loading...
; - } - - if (!teamData?.team_info) { - return
Team not found
; - } - - const { team_info: info } = teamData; - - const copyToClipboard = async (text: string, key: string) => { - const success = await utilCopyToClipboard(text); - if (success) { - setCopiedStates((prev) => ({ ...prev, [key]: true })); - setTimeout(() => { - setCopiedStates((prev) => ({ ...prev, [key]: false })); - }, 2000); - } - }; - - return ( -
-
-
- - Back to Teams - - {info.team_alias} -
- {info.team_id} -
-
-
- - - - {[ - Overview, - ...(canEditTeam - ? [ - Members, - Member Permissions, - Settings, - ] - : []), - ]} - - - - {/* Overview Panel */} - - - - Budget Status -
- ${formatNumberWithCommas(info.spend, 4)} - - of {info.max_budget === null ? "Unlimited" : `$${formatNumberWithCommas(info.max_budget, 4)}`} - - {info.budget_duration && Reset: {info.budget_duration}} -
- {info.team_member_budget_table && ( - - Team Member Budget: ${formatNumberWithCommas(info.team_member_budget_table.max_budget, 4)} - - )} -
- - - - Rate Limits -
- TPM: {info.tpm_limit || "Unlimited"} - RPM: {info.rpm_limit || "Unlimited"} - {info.max_parallel_requests && Max Parallel Requests: {info.max_parallel_requests}} -
- - - - Models -
- {info.models.length === 0 ? ( - All proxy models - ) : ( - info.models.map((model, index) => ( - - {model} - - )) - )} -
- - - - Virtual Keys -
- User Keys: {teamData.keys.filter((key) => key.user_id).length} - Service Account Keys: {teamData.keys.filter((key) => !key.user_id).length} - Total: {teamData.keys.length} -
- - - - - - Guardrails - {info.guardrails && info.guardrails.length > 0 ? ( -
- {info.guardrails.map((guardrail: string, index: number) => ( - - {guardrail} - - ))} -
- ) : ( - No guardrails configured - )} - {info.metadata?.disable_global_guardrails && ( -
- Global Guardrails Disabled -
- )} - - - - Policies - {info.policies && info.policies.length > 0 ? ( -
- {info.policies.map((policy: string, index: number) => ( -
-
- {policy} - {loadingPolicies && Loading guardrails...} -
- {!loadingPolicies && policyGuardrails[policy] && policyGuardrails[policy].length > 0 && ( -
- Resolved Guardrails: -
- {policyGuardrails[policy].map((guardrail: string, gIndex: number) => ( - - {guardrail} - - ))} -
-
- )} -
- ))} -
- ) : ( - No policies configured - )} - - - - - - - {/* Members Panel */} - - - - - {/* Member Permissions Panel */} - {canEditTeam && ( - - - - )} - - {/* Settings Panel */} - - -
- Team Settings - {canEditTeam && !isEditing && ( - setIsEditing(true)}>Edit Settings - )} -
- - {isEditing ? ( -
rest)(info.metadata), - null, - 2, - ) - : "", - logging_settings: info.metadata?.logging || [], - secret_manager_settings: info.metadata?.secret_manager_settings - ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) - : "", - organization_id: info.organization_id, - vector_stores: info.object_permission?.vector_stores || [], - mcp_servers: info.object_permission?.mcp_servers || [], - mcp_access_groups: info.object_permission?.mcp_access_groups || [], - mcp_servers_and_groups: { - servers: info.object_permission?.mcp_servers || [], - accessGroups: info.object_permission?.mcp_access_groups || [], - }, - mcp_tool_permissions: info.object_permission?.mcp_tool_permissions || {}, - agents_and_groups: { - agents: info.object_permission?.agents || [], - accessGroups: info.object_permission?.agent_access_groups || [], - }, - }} - layout="vertical" - > - - - - - - form.setFieldValue("models", values)} - teamID={teamId} - organizationID={teamData?.team_info?.organization_id || undefined} - options={{ - includeSpecialOptions: true, - includeUserModels: !teamData?.team_info?.organization_id, - showAllProxyModelsOverride: isProxyAdminRole(userRole) && !teamData?.team_info?.organization_id, - }} - context="team" - dataTestId="models-select" - /> - - - - - - - - - - - - - - - - - - - - form.setFieldValue("team_member_budget_duration", value)} - value={form.getFieldValue("team_member_budget_duration")} - /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - Guardrails{" "} - - e.stopPropagation()} - > - - - - - } - name="guardrails" - help="Select existing guardrails or enter new ones" - > - ({ value: name, label: name }))} - /> - - - - form.setFieldValue("vector_stores", values)} - value={form.getFieldValue("vector_stores")} - accessToken={accessToken || ""} - placeholder="Select vector stores" - /> - - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes" - /> - - - - form.setFieldValue("mcp_servers_and_groups", val)} - value={form.getFieldValue("mcp_servers_and_groups")} - accessToken={accessToken || ""} - placeholder="Select MCP servers or access groups (optional)" - /> - - - {/* Hidden field to register mcp_tool_permissions with the form */} - - - - prevValues.mcp_servers_and_groups !== currentValues.mcp_servers_and_groups || - prevValues.mcp_tool_permissions !== currentValues.mcp_tool_permissions - } - > - {() => ( -
- form.setFieldsValue({ mcp_tool_permissions: toolPerms })} - /> -
- )} - - - - form.setFieldValue("agents_and_groups", val)} - value={form.getFieldValue("agents_and_groups")} - accessToken={accessToken || ""} - placeholder="Select agents or access groups (optional)" - /> - - - - - - - - form.setFieldValue("logging_settings", values)} - /> - - - { - if (!value) { - return Promise.resolve(); - } - try { - JSON.parse(value); - return Promise.resolve(); - } catch (error) { - return Promise.reject(new Error("Please enter valid JSON")); - } - }, - }, - ]} - > - - - - - - - -
-
- setIsEditing(false)} disabled={isTeamSaving}> - Cancel - - - Save Changes - -
-
- - ) : ( -
-
- Team Name -
{info.team_alias}
-
-
- Team ID -
{info.team_id}
-
-
- Created At -
{new Date(info.created_at).toLocaleString()}
-
-
- Models -
- {info.models.map((model, index) => ( - - {model} - - ))} -
-
-
- Rate Limits -
TPM: {info.tpm_limit || "Unlimited"}
-
RPM: {info.rpm_limit || "Unlimited"}
-
-
- Team Budget -
- Max Budget:{" "} - {info.max_budget !== null ? `$${formatNumberWithCommas(info.max_budget, 4)}` : "No Limit"} -
-
- Soft Budget:{" "} - {info.soft_budget !== null && info.soft_budget !== undefined - ? `$${formatNumberWithCommas(info.soft_budget, 4)}` - : "No Limit"} -
-
Budget Reset: {info.budget_duration || "Never"}
- {info.metadata?.soft_budget_alerting_emails && - Array.isArray(info.metadata.soft_budget_alerting_emails) && - info.metadata.soft_budget_alerting_emails.length > 0 && ( -
- Soft Budget Alerting Emails: {info.metadata.soft_budget_alerting_emails.join(", ")} -
- )} -
-
- - Team Member Settings{" "} - - - - -
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
-
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
-
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
-
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
-
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
-
-
- Organization ID -
{info.organization_id}
-
-
- Status - {info.blocked ? "Blocked" : "Active"} -
- -
- Disable Global Guardrails -
- {info.metadata?.disable_global_guardrails === true ? ( - Enabled - Global guardrails bypassed - ) : ( - Disabled - Global guardrails active - )} -
-
- - - - - - {info.metadata?.secret_manager_settings && ( -
- Secret Manager Settings - 
-                        {JSON.stringify(info.metadata.secret_manager_settings, null, 2)}
-
-
- )} -
- )} - - - - - - setIsEditMemberModalVisible(false)} - onSubmit={handleMemberUpdate} - initialData={selectedEditMember} - mode="edit" - config={{ - title: "Edit Member", - showEmail: true, - showUserId: true, - roleOptions: [ - { label: "Admin", value: "admin" }, - { label: "User", value: "user" }, - ], - additionalFields: [ - { - name: "max_budget_in_team", - label: ( - - Team Member Budget (USD){" "} - - - - - ), - type: "numerical" as const, - step: 0.01, - min: 0, - placeholder: "Budget limit for this member within this team", - }, - { - name: "tpm_limit", - label: ( - - Team Member TPM Limit{" "} - - - - - ), - type: "numerical" as const, - step: 1, - min: 0, - placeholder: "Tokens per minute limit for this member in this team", - }, - { - name: "rpm_limit", - label: ( - - Team Member RPM Limit{" "} - - - - - ), - type: "numerical" as const, - step: 1, - min: 0, - placeholder: "Requests per minute limit for this member in this team", - }, - ], - }} - /> - - setIsAddMemberModalVisible(false)} - onSubmit={handleMemberCreate} - accessToken={accessToken} - /> - - {/* Delete Member Confirmation Modal */} - -
- ); -}; - -export default TeamInfoView; diff --git a/ui/litellm-dashboard/src/components/team/team_member_view.test.tsx b/ui/litellm-dashboard/src/components/team/team_member_view.test.tsx deleted file mode 100644 index 30a06179c2f..00000000000 --- a/ui/litellm-dashboard/src/components/team/team_member_view.test.tsx +++ /dev/null @@ -1,193 +0,0 @@ -import { screen } from "@testing-library/react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "../../../tests/test-utils"; -import { TeamData } from "./team_info"; -import TeamMembersComponent from "./team_member_view"; - -// Mock the hooks -vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ - useUISettings: vi.fn(), -})); - -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: vi.fn(), -})); - -vi.mock("@/utils/roles", () => ({ - isUserTeamAdminForSingleTeam: vi.fn(() => false), - isProxyAdminRole: vi.fn(() => false), -})); - -import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; - -describe("TeamMembersComponent", () => { - const mockHandleMemberDelete = vi.fn(); - const mockSetSelectedEditMember = vi.fn(); - const mockSetIsEditMemberModalVisible = vi.fn(); - const mockSetIsAddMemberModalVisible = vi.fn(); - - const mockTeamData: TeamData = { - team_id: "team-123", - team_info: { - team_alias: "Test Team", - team_id: "team-123", - organization_id: null, - admins: ["admin@test.com"], - members: ["user1@test.com"], - members_with_roles: [ - { - user_id: "user1@test.com", - user_email: "user1@test.com", - role: "member", - }, - { - user_id: "user2@test.com", - user_email: "user2@test.com", - role: "admin", - }, - ], - metadata: {}, - tpm_limit: null, - rpm_limit: null, - max_budget: null, - budget_duration: null, - models: [], - blocked: false, - spend: 0, - max_parallel_requests: null, - budget_reset_at: null, - model_id: null, - litellm_model_table: null, - created_at: "2024-01-01T00:00:00Z", - team_member_budget_table: null, - }, - keys: [], - team_memberships: [ - { - user_id: "user1@test.com", - team_id: "team-123", - budget_id: "budget1", - spend: 100.5, - litellm_budget_table: { - budget_id: "budget1", - soft_budget: null, - max_budget: 1000, - max_parallel_requests: null, - tpm_limit: 10000, - rpm_limit: 100, - model_max_budget: null, - budget_duration: null, - }, - }, - ], - }; - - beforeEach(() => { - vi.clearAllMocks(); - vi.mocked(useUISettings).mockReturnValue({ - data: { values: { disable_team_admin_delete_team_user: false } }, - isLoading: false, - isError: false, - error: null, - isSuccess: true, - isFetching: false, - refetch: vi.fn(), - } as any); - - vi.mocked(useAuthorized).mockReturnValue({ - userId: "test-user-id", - userRole: "Admin", - accessToken: "test-token", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - }); - - it("should render team members table with headers", () => { - renderWithProviders( - , - ); - - expect(screen.getByText("User ID")).toBeInTheDocument(); - expect(screen.getByText("User Email")).toBeInTheDocument(); - expect(screen.getByText("Role")).toBeInTheDocument(); - expect(screen.getByText("Team Member Spend (USD)")).toBeInTheDocument(); - expect(screen.getByText("Team Member Budget (USD)")).toBeInTheDocument(); - expect(screen.getByText("Team Member Rate Limits")).toBeInTheDocument(); - expect(screen.getByText("Actions")).toBeInTheDocument(); - }); - - it("should render team members data", () => { - renderWithProviders( - , - ); - - // user1@test.com appears twice (User ID and User Email columns) - expect(screen.getAllByText("user1@test.com")).toHaveLength(2); - // user2@test.com appears twice (User ID and User Email columns) - expect(screen.getAllByText("user2@test.com")).toHaveLength(2); - expect(screen.getByText("member")).toBeInTheDocument(); - expect(screen.getByText("admin")).toBeInTheDocument(); - }); - - it("should render Add Member button", () => { - renderWithProviders( - , - ); - - expect(screen.getByText("Add Member")).toBeInTheDocument(); - }); - - it("should show delete button for proxy admin when canEditTeam is true", () => { - vi.mocked(isProxyAdminRole).mockReturnValue(true); - vi.mocked(isUserTeamAdminForSingleTeam).mockReturnValue(false); - - const { container } = renderWithProviders( - , - ); - - // Verify that action buttons are rendered when canEditTeam is true - // For proxy admin, both edit and delete buttons should be visible - // Check for clickable icon elements (Tremor Icon components with cursor-pointer class) - const clickableIcons = container.querySelectorAll('[class*="cursor-pointer"]'); - // Should have at least 4 icons: 2 edit buttons + 2 delete buttons for 2 members - expect(clickableIcons.length).toBeGreaterThanOrEqual(4); - - // Verify members are rendered - expect(screen.getAllByText("user1@test.com").length).toBeGreaterThan(0); - expect(screen.getAllByText("user2@test.com").length).toBeGreaterThan(0); - }); -}); diff --git a/ui/litellm-dashboard/src/components/team/team_member_view.tsx b/ui/litellm-dashboard/src/components/team/team_member_view.tsx deleted file mode 100644 index 10b3cbd83e6..00000000000 --- a/ui/litellm-dashboard/src/components/team/team_member_view.tsx +++ /dev/null @@ -1,193 +0,0 @@ -import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { Member } from "@/components/networking"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { - Card, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Text, - Button as TremorButton, -} from "@tremor/react"; -import { Tooltip } from "antd"; -import React from "react"; -import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; -import { TeamData } from "./team_info"; - -interface TeamMembersComponentProps { - teamData: TeamData; - canEditTeam: boolean; - handleMemberDelete: (member: Member) => void; - setSelectedEditMember: (member: Member) => void; - setIsEditMemberModalVisible: (visible: boolean) => void; - setIsAddMemberModalVisible: (visible: boolean) => void; -} - -const TeamMembersComponent: React.FC = ({ - teamData, - canEditTeam, - handleMemberDelete, - setSelectedEditMember, - setIsEditMemberModalVisible, - setIsAddMemberModalVisible, -}) => { - console.log("Team data", teamData); - // Helper function to convert scientific notation to normal decimal format - const formatNumber = (value: number | null): string => { - if (value === null || value === undefined) return "0"; - - if (typeof value === "number") { - // Convert scientific notation to normal decimal - const normalNumber = Number(value); - - // If it's a whole number, return it without decimals - if (normalNumber === Math.floor(normalNumber)) { - return normalNumber.toString(); - } - - // For decimal numbers, use toFixed and remove trailing zeros - return formatNumberWithCommas(normalNumber, 8).replace(/\.?0+$/, ""); - } - - return "0"; - }; - - // Helper function to get spend for a user - const getUserSpend = (userId: string | null): number | null => { - if (!userId) return 0; - const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); - return membership?.spend || 0; - }; - - const getUserBudget = (userId: string | null): string | null => { - if (!userId) return null; - const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); - console.log(`membership=${membership}`); - const maxBudget = membership?.litellm_budget_table?.max_budget; - if (maxBudget === null || maxBudget === undefined) { - return null; - } - return formatNumber(maxBudget); - }; - - // Helper function to get rate limits for a user - const getUserRateLimits = (userId: string | null): string => { - if (!userId) return "No Limits"; - const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); - const rpmLimit = membership?.litellm_budget_table?.rpm_limit; - const tpmLimit = membership?.litellm_budget_table?.tpm_limit; - - const rpmText = rpmLimit ? `${formatNumber(rpmLimit)} RPM` : null; - const tpmText = tpmLimit ? `${formatNumber(tpmLimit)} TPM` : null; - - const limits = [rpmText, tpmText].filter(Boolean); - return limits.length > 0 ? limits.join(" / ") : "No Limits"; - }; - - const { data: uiSettingsData } = useUISettings(); - const { userId, userRole } = useAuthorized(); - const disableTeamAdminDeleteTeamUser = Boolean(uiSettingsData?.values?.disable_team_admin_delete_team_user); - const isUserTeamAdmin = isUserTeamAdminForSingleTeam(teamData.team_info.members_with_roles, userId || ""); - const isProxyAdmin = isProxyAdminRole(userRole || ""); - - return ( -
- -
-
- - - User ID - User Email - Role - - Team Member Spend (USD){" "} - - - - - Team Member Budget (USD) - - Team Member Rate Limits{" "} - - - - - - Actions - - - - - - {teamData.team_info.members_with_roles.map((member: Member, index: number) => ( - - - {member.user_id} - - - {member.user_email ? member.user_email : "No Email"} - - - {member.role} - - - ${formatNumberWithCommas(getUserSpend(member.user_id), 4)} - - - - {getUserBudget(member.user_id) - ? `$${formatNumberWithCommas(Number(getUserBudget(member.user_id)), 4)}` - : "No Limit"} - - - - {getUserRateLimits(member.user_id)} - - - {canEditTeam && ( -
- { - // Get budget and rate limit data from team membership - const membership = teamData.team_memberships.find((tm) => tm.user_id === member.user_id); - const enhancedMember = { - ...member, - max_budget_in_team: membership?.litellm_budget_table?.max_budget || null, - tpm_limit: membership?.litellm_budget_table?.tpm_limit || null, - rpm_limit: membership?.litellm_budget_table?.rpm_limit || null, - }; - setSelectedEditMember(enhancedMember); - setIsEditMemberModalVisible(true); - }} - /> - {(isProxyAdmin || (isUserTeamAdmin && !disableTeamAdminDeleteTeamUser)) && ( - handleMemberDelete(member)} - /> - )} -
- )} - - - ))} - -
-
- - setIsAddMemberModalVisible(true)}>Add Member -
- ); -}; - -export default TeamMembersComponent; From 409d12b7a5e3abf88d726621562fc6f591f78c11 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Feb 2026 14:40:51 -0800 Subject: [PATCH 060/117] Add alert about email notifications --- .../src/components/CreateUserButton.test.tsx | 297 ++++++++++++++++++ ...e_user_button.tsx => CreateUserButton.tsx} | 141 +++++---- .../components/create_user_button.test.tsx | 35 --- .../organisms/create_key_button.tsx | 4 +- .../src/components/view_users.tsx | 4 +- 5 files changed, 373 insertions(+), 108 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/CreateUserButton.test.tsx rename ui/litellm-dashboard/src/components/{create_user_button.tsx => CreateUserButton.tsx} (81%) delete mode 100644 ui/litellm-dashboard/src/components/create_user_button.test.tsx diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx new file mode 100644 index 00000000000..bfaddacc319 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx @@ -0,0 +1,297 @@ +import { render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { CreateUserButton } from "./CreateUserButton"; +import * as networking from "./networking"; +import NotificationsManager from "./molecules/notifications_manager"; + +vi.mock("./networking", () => ({ + userCreateCall: vi.fn(), + modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), + invitationCreateCall: vi.fn(), + getProxyUISettings: vi.fn().mockResolvedValue({ + PROXY_BASE_URL: null, + PROXY_LOGOUT_URL: null, + DEFAULT_TEAM_DISABLED: false, + SSO_ENABLED: false, + }), + getProxyBaseUrl: vi.fn().mockReturnValue("http://localhost"), +})); + +vi.mock("./bulk_create_users_button", () => ({ + default: () =>
Bulk Create Users
, +})); + +const mockUserCreateCall = vi.mocked(networking.userCreateCall); +const mockInvitationCreateCall = vi.mocked(networking.invitationCreateCall); +const mockGetProxyUISettings = vi.mocked(networking.getProxyUISettings); +const mockNotificationsManager = vi.mocked(NotificationsManager); + +const createQueryClient = () => + new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 } }, + }); + +const defaultProps = { + userID: "123", + accessToken: "token", + teams: [], + possibleUIRoles: null as Record> | null, +}; + +function renderWithProviders(ui: React.ReactElement) { + const qc = createQueryClient(); + return render({ui}); +} + +describe("CreateUserButton", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetProxyUISettings.mockResolvedValue({ + PROXY_BASE_URL: null, + PROXY_LOGOUT_URL: null, + DEFAULT_TEAM_DISABLED: false, + SSO_ENABLED: false, + }); + }); + + it("should render the create user form when embedded", () => { + renderWithProviders( + , + ); + expect(screen.getByRole("button", { name: /create user/i })).toBeInTheDocument(); + }); + + it("should render the invite user button when not embedded", async () => { + renderWithProviders(); + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + }); + + it("should open the invite modal when invite user button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /\+ invite user/i })); + const dialog = screen.getByRole("dialog", { name: /invite user/i }); + expect(dialog).toBeInTheDocument(); + expect(within(dialog).getByRole("button", { name: /invite user/i })).toBeInTheDocument(); + }); + + it("should display email invitations info message in embedded mode", () => { + renderWithProviders(); + expect(screen.getByText("Email invitations")).toBeInTheDocument(); + }); + + it("should display user role options when possibleUIRoles is provided", async () => { + const possibleUIRoles = { + proxy_admin: { ui_label: "Admin", description: "Full access" }, + proxy_user: { ui_label: "User", description: "Limited access" }, + }; + renderWithProviders( + , + ); + await userEvent.click(screen.getByRole("combobox", { name: /user role/i })); + expect(screen.getByText("Admin")).toBeInTheDocument(); + expect(screen.getByText("User")).toBeInTheDocument(); + }); + + it("should call userCreateCall when form is submitted in embedded mode", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "new-user-123" } }); + mockInvitationCreateCall.mockResolvedValue({ + id: "inv-1", + user_id: "new-user-123", + has_user_setup_sso: false, + } as any); + + renderWithProviders( + , + ); + + await user.type(screen.getByLabelText(/user email/i), "test@example.com"); + await user.click(screen.getByRole("combobox", { name: /user role/i })); + await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /create user/i })); + + await waitFor(() => { + expect(mockUserCreateCall).toHaveBeenCalledWith("token", null, expect.objectContaining({ + user_email: "test@example.com", + user_role: "proxy_user", + })); + }); + }); + + it("should call onUserCreated callback when user is created in embedded mode", async () => { + const user = userEvent.setup(); + const onUserCreated = vi.fn(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "new-user-456" } }); + + renderWithProviders( + , + ); + + await user.type(screen.getByLabelText(/user email/i), "embedded@example.com"); + await user.click(screen.getByRole("combobox", { name: /user role/i })); + await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /create user/i })); + + await waitFor(() => { + expect(onUserCreated).toHaveBeenCalledWith("new-user-456"); + }); + }); + + it("should show success notification when user is created successfully in standalone mode", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "new-user-789" } }); + mockInvitationCreateCall.mockResolvedValue({ + id: "inv-2", + user_id: "new-user-789", + has_user_setup_sso: false, + } as any); + + renderWithProviders( + , + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /\+ invite user/i })); + + const dialog = screen.getByRole("dialog", { name: /invite user/i }); + await user.type(within(dialog).getByLabelText(/user email/i), "standalone@example.com"); + await user.click(within(dialog).getByRole("combobox", { name: /global proxy role/i })); + await user.click(screen.getByText("User")); + await user.click(within(dialog).getByRole("button", { name: /invite user/i })); + + await waitFor(() => { + expect(mockNotificationsManager.success).toHaveBeenCalledWith("API user Created"); + }); + }); + + it("should show error notification when user creation fails", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockRejectedValue({ response: { data: { detail: "Email already exists" } } }); + + renderWithProviders( + , + ); + + await user.type(screen.getByLabelText(/user email/i), "duplicate@example.com"); + await user.click(screen.getByRole("combobox", { name: /user role/i })); + await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /create user/i })); + + await waitFor(() => { + expect(mockNotificationsManager.fromBackend).toHaveBeenCalledWith("Email already exists"); + }); + }); + + it("should show info notification when making API call", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "new-user" } }); + mockInvitationCreateCall.mockResolvedValue({ + id: "inv-3", + user_id: "new-user", + has_user_setup_sso: false, + } as any); + + renderWithProviders( + , + ); + + await user.type(screen.getByLabelText(/user email/i), "info@example.com"); + await user.click(screen.getByRole("combobox", { name: /user role/i })); + await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /create user/i })); + + await waitFor(() => { + expect(mockNotificationsManager.info).toHaveBeenCalledWith("Making API Call"); + }); + }); + + it("should close modal when cancel is clicked in standalone mode", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /\+ invite user/i })); + expect(screen.getByRole("dialog", { name: /invite user/i })).toBeInTheDocument(); + + const dialog = screen.getByRole("dialog", { name: /invite user/i }); + await user.click(within(dialog).getByRole("button", { name: /close/i })); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("should show onboarding modal when user is created and SSO is disabled", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "sso-user" } }); + mockInvitationCreateCall.mockResolvedValue({ + id: "inv-sso", + user_id: "sso-user", + has_user_setup_sso: false, + } as any); + + renderWithProviders( + , + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /\+ invite user/i })); + + const dialog = screen.getByRole("dialog", { name: /invite user/i }); + await user.type(within(dialog).getByLabelText(/user email/i), "sso@example.com"); + await user.click(within(dialog).getByRole("combobox", { name: /global proxy role/i })); + await user.click(screen.getByText("User")); + await user.click(within(dialog).getByRole("button", { name: /invite user/i })); + + await waitFor(() => { + expect(mockInvitationCreateCall).toHaveBeenCalledWith("token", "sso-user"); + }); + await waitFor(() => { + expect(mockNotificationsManager.success).toHaveBeenCalledWith("API user Created"); + }); + }); + + it("should use SSO flow without invitationCreateCall when SSO is enabled", async () => { + const user = userEvent.setup(); + mockUserCreateCall.mockResolvedValue({ data: { user_id: "sso-enabled-user" } }); + mockGetProxyUISettings.mockResolvedValue({ + PROXY_BASE_URL: "http://localhost", + PROXY_LOGOUT_URL: null, + DEFAULT_TEAM_DISABLED: false, + SSO_ENABLED: true, + }); + + renderWithProviders( + , + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /\+ invite user/i })); + + const dialog = screen.getByRole("dialog", { name: /invite user/i }); + await user.type(within(dialog).getByLabelText(/user email/i), "sso-enabled@example.com"); + await user.click(within(dialog).getByRole("combobox", { name: /global proxy role/i })); + await user.click(screen.getByText("User")); + await user.click(within(dialog).getByRole("button", { name: /invite user/i })); + + await waitFor(() => { + expect(mockInvitationCreateCall).not.toHaveBeenCalled(); + }); + await waitFor(() => { + expect(mockNotificationsManager.success).toHaveBeenCalledWith("API user Created"); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx similarity index 81% rename from ui/litellm-dashboard/src/components/create_user_button.tsx rename to ui/litellm-dashboard/src/components/CreateUserButton.tsx index da155b042a3..3a15fbd5055 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -1,33 +1,29 @@ -import React, { useState, useEffect } from "react"; -import { Button, Modal, Form, Input, Select, Select as Select2 } from "antd"; -import { - Button as Button2, - Text, - TextInput, - SelectItem, - Accordion, - AccordionHeader, - AccordionBody, - Title, -} from "@tremor/react"; -import OnboardingModal from "./onboarding_link"; -import { InvitationLink } from "./onboarding_link"; -import { - userCreateCall, - modelAvailableCall, - invitationCreateCall, - getProxyUISettings, - getProxyBaseUrl, -} from "./networking"; -import BulkCreateUsers from "./bulk_create_users_button"; -const { Option } = Select; -import { Tooltip } from "antd"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; +import { InfoCircleOutlined, UserAddOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; -import NotificationsManager from "./molecules/notifications_manager"; +import { + Accordion, + AccordionBody, + AccordionHeader, + Button as Button2, + SelectItem, + TextInput, +} from "@tremor/react"; +import { Alert, Button, Form, Input, Modal, Select, Select as Select2, Space, Tooltip, Typography } from "antd"; +import React, { useEffect, useState } from "react"; +import BulkCreateUsers from "./bulk_create_users_button"; import TeamDropdown from "./common_components/team_dropdown"; - +import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; +import NotificationsManager from "./molecules/notifications_manager"; +import { + getProxyBaseUrl, + getProxyUISettings, + invitationCreateCall, + modelAvailableCall, + userCreateCall, +} from "./networking"; +import OnboardingModal, { InvitationLink } from "./onboarding_link"; +const { Option } = Select; +const { Text, Link, Title } = Typography; // Helper function to generate UUID compatible across all environments const generateUUID = (): string => { if (typeof crypto !== "undefined" && crypto.randomUUID) { @@ -58,14 +54,8 @@ interface UISettings { SSO_ENABLED: boolean; } -const Createuser: React.FC = ({ - userID, - accessToken, - teams, - possibleUIRoles, - onUserCreated, - isEmbedded = false, -}) => { +export const CreateUserButton: React.FC = ({ + userID, accessToken, teams, possibleUIRoles, onUserCreated, isEmbedded = false }) => { const queryClient = useQueryClient(); const [uiSettings, setUISettings] = useState(null); const [form] = Form.useForm(); @@ -75,28 +65,18 @@ const Createuser: React.FC = ({ const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] = useState(false); const [invitationLinkData, setInvitationLinkData] = useState(null); const [baseUrl, setBaseUrl] = useState(null); - // get all models useEffect(() => { const fetchData = async () => { try { - const userRole = "any"; // You may need to get the user role dynamically + const userRole = "any"; const modelDataResponse = await modelAvailableCall(accessToken, userID, userRole); - // Assuming modelDataResponse.data contains an array of model objects with a 'model_name' property const availableModels = []; for (let i = 0; i < modelDataResponse.data.length; i++) { const model = modelDataResponse.data[i]; availableModels.push(model.id); } - console.log("Model data response:", modelDataResponse.data); - console.log("Available models:", availableModels); - - // Assuming modelDataResponse.data contains an array of model names setUserModels(availableModels); - - // get ui settings const uiSettingsResponse = await getProxyUISettings(accessToken); - console.log("uiSettingsResponse:", uiSettingsResponse); - setUISettings(uiSettingsResponse); } catch (error) { console.error("Error fetching model data:", error); @@ -104,9 +84,8 @@ const Createuser: React.FC = ({ }; setBaseUrl(getProxyBaseUrl()); - - fetchData(); // Call the function to fetch model data when the component mounts - }, []); // Empty dependency array to run only once + fetchData(); + }, []); const handleOk = () => { setIsModalVisible(false); @@ -126,25 +105,19 @@ const Createuser: React.FC = ({ setIsModalVisible(true); } if ((!formValues.models || formValues.models.length === 0) && formValues.user_role !== "proxy_admin") { - console.log("formValues.user_role", formValues.user_role); - // If models is empty or undefined, set it to "no-default-models" formValues.models = ["no-default-models"]; } - console.log("formValues in create user:", formValues); const response = await userCreateCall(accessToken, null, formValues); await queryClient.invalidateQueries({ queryKey: ["userList"] }); - console.log("user create Response:", response); setApiuser(true); const user_id = response.data?.user_id || response.user_id; - // Call the callback if provided (for embedded mode) if (onUserCreated && isEmbedded) { onUserCreated(user_id); form.resetFields(); - return; // Skip the invitation flow when embedded + return; } - // only do invite link flow if sso is not enabled if (!uiSettings?.SSO_ENABLED) { invitationCreateCall(accessToken, user_id).then((data) => { data.has_user_setup_sso = false; @@ -184,6 +157,21 @@ const Createuser: React.FC = ({ if (isEmbedded) { return (
+ + New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured. + {" "} + + Learn how to set up email notifications + + + } + type="info" + showIcon + className="mb-4" + /> @@ -194,9 +182,9 @@ const Createuser: React.FC = ({
{ui_label}{" "} -

+ {description} -

+
))} @@ -234,16 +222,33 @@ const Createuser: React.FC = ({ onOk={handleOk} onCancel={handleCancel} > - Create a User who can own keys + + Create a User who can own keys + + New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured. + {" "} + + Learn how to set up email notifications + + + } + type="info" + showIcon + className="mb-4" + /> + - + Global Proxy Role{" "} - + @@ -256,9 +261,9 @@ const Createuser: React.FC = ({
{ui_label}{" "} -

+ {description} -

+
))} @@ -279,7 +284,7 @@ const Createuser: React.FC = ({ - Personal Key Creation + Personal Key Creation = ({
- +
@@ -326,6 +331,4 @@ const Createuser: React.FC = ({ )}
); -}; - -export default Createuser; +}; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/create_user_button.test.tsx b/ui/litellm-dashboard/src/components/create_user_button.test.tsx deleted file mode 100644 index e40a1e0ac3c..00000000000 --- a/ui/litellm-dashboard/src/components/create_user_button.test.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from "react"; -import { render } from "@testing-library/react"; -import { describe, it, expect, vi } from "vitest"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import Createuser from "./create_user_button"; - -vi.mock("./networking", () => ({ - userCreateCall: vi.fn(), - modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), - invitationCreateCall: vi.fn(), - getProxyUISettings: vi.fn().mockResolvedValue({ - PROXY_BASE_URL: null, - PROXY_LOGOUT_URL: null, - DEFAULT_TEAM_DISABLED: false, - SSO_ENABLED: false, - }), - getProxyBaseUrl: vi.fn().mockReturnValue("http://localhost"), -})); - -const createQueryClient = () => - new QueryClient({ - defaultOptions: { queries: { retry: false, gcTime: 0 } }, - }); - -describe("Create User Button", () => { - it("should render the create user button", () => { - const qc = createQueryClient(); - const { getByText } = render( - - - , - ); - expect(getByText("Create User")).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 80037280b63..abadbe10590 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -21,7 +21,7 @@ import PremiumLoggingSettings from "../common_components/PremiumLoggingSettings" import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem"; import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "../common_components/RouterSettingsAccordion"; import TeamDropdown from "../common_components/team_dropdown"; -import Createuser from "../create_user_button"; +import { CreateUserButton } from "../CreateUserButton"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { Team } from "../key_team_helpers/key_list"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; @@ -1347,7 +1347,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { footer={null} width={800} > - = ({ accessToken, toke ) : userID && accessToken ? ( <> - + + {oauthFlow.error &&

{oauthFlow.error}

} + {oauthFlow.status === "success" && oauthFlow.tokenResponse?.access_token && ( +

+ Token fetched. Expires in {oauthFlow.tokenResponse.expires_in ?? "?"} seconds. +

+ )} +
+ )} + + )} + + ); +}; + +export default OAuthFormFields; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index a56cbaf492a..ff476bd999d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -3,7 +3,8 @@ import { Modal, Tooltip, Form, Select, Input } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TextInput } from "@tremor/react"; import { createMCPServer } from "../networking"; -import { AUTH_TYPE, MCPServer, MCPServerCostInfo } from "./types"; +import { AUTH_TYPE, OAUTH_FLOW, MCPServer, MCPServerCostInfo } from "./types"; +import OAuthFormFields from "./OAuthFormFields"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPConnectionStatus from "./mcp_connection_status"; import MCPToolConfiguration from "./mcp_tool_configuration"; @@ -52,6 +53,7 @@ const CreateMCPServer: React.FC = ({ const authType = formValues.auth_type as string | undefined; const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false; const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2; + const isM2MFlow = isOAuthAuthType && formValues.oauth_flow_type === OAUTH_FLOW.M2M; const persistCreateUiState = () => { if (typeof window === "undefined") { @@ -477,7 +479,7 @@ const CreateMCPServer: React.FC = ({ rules={[ { required: false, - message: "Please enter a server description!!!!!!!!!", + message: "Please enter a server description", }, ]} > @@ -561,131 +563,16 @@ const CreateMCPServer: React.FC = ({ )} {transportType !== "stdio" && isOAuthAuthType && ( - <> - - OAuth Client ID (optional) - - - - - } - name={["credentials", "client_id"]} - > - - - - OAuth Client Secret (optional) - - - - - } - name={["credentials", "client_secret"]} - > - - - - OAuth Scopes (optional) - - - - - } - name={["credentials", "scopes"]} - > - - - - Authorization URL Override (optional) - - - - - } - name="authorization_url" - > - - - - Token URL Override (optional) - - - - - } - name="token_url" - > - - - - Registration URL Override (optional) - - - - - } - name="registration_url" - > - - -
-

Use OAuth to fetch a fresh access token and temporarily save it in the session as the authentication value.

- - {oauthError &&

{oauthError}

} - {oauthStatus === "success" && oauthTokenResponse?.access_token && ( -

- Token fetched. Expires in {oauthTokenResponse.expires_in ?? "?"} seconds. -

- )} -
- + )} {/* Permission Management / Access Control Section */} @@ -600,6 +492,7 @@ const MCPServerEdit: React.FC = ({ transport: mcpServer.transport, auth_type: mcpServer.auth_type, mcp_info: mcpServer.mcp_info, + oauth_flow_type: mcpServer.token_url ? OAUTH_FLOW.M2M : OAUTH_FLOW.INTERACTIVE, }} allowedTools={allowedTools} existingAllowedTools={mcpServer.allowed_tools || null} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index d575254fe35..5cb840ec7d4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -13,13 +13,17 @@ export const AUTH_TYPE = { OAUTH2: "oauth2", }; +export const OAUTH_FLOW = { + INTERACTIVE: "interactive", + M2M: "m2m", +}; + export const TRANSPORT = { SSE: "sse", HTTP: "http", }; export const handleTransport = (transport?: string | null): string => { - console.log(transport); if (transport === null || transport === undefined) { return TRANSPORT.SSE; } diff --git a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx index 1743470559d..0cb4288db0e 100644 --- a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx +++ b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx @@ -1,6 +1,6 @@ import { useState, useEffect } from "react"; import { testMCPToolsListRequest } from "../components/networking"; -import { AUTH_TYPE } from "@/components/mcp_tools/types"; +import { AUTH_TYPE, OAUTH_FLOW } from "@/components/mcp_tools/types"; interface MCPServerConfig { server_id?: string; @@ -52,7 +52,9 @@ export const useTestMCPConnection = ({ const [hasShownSuccessMessage, setHasShownSuccessMessage] = useState(false); // Check if we have the minimum required fields to fetch tools - const requiresOAuthToken = formValues.auth_type === AUTH_TYPE.OAUTH2; + const isM2MOAuth = formValues.auth_type === AUTH_TYPE.OAUTH2 + && formValues.oauth_flow_type === OAUTH_FLOW.M2M; + const requiresOAuthToken = formValues.auth_type === AUTH_TYPE.OAUTH2 && !isM2MOAuth; const canFetchTools = !!( formValues.url && formValues.transport && From 1ee43b11de1e4e5bd0e57749d290579b5aee175f Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Mon, 9 Feb 2026 19:38:44 -0800 Subject: [PATCH 068/117] [Fix] prevent shared backend model key from being polluted by per-deployment custom pricing (#20679) * bug: custom price override for models * added associated test --- litellm/router.py | 13 +- .../test_router_model_cost_isolation.py | 264 ++++++++++++++++++ 2 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/test_router_model_cost_isolation.py diff --git a/litellm/router.py b/litellm/router.py index 0f49cf90510..42058c79c17 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5925,9 +5925,20 @@ class Router: deployment.litellm_params.custom_llm_provider + "/" + _model_name ) + # For the shared backend key, strip custom pricing fields so that + # one deployment's pricing overrides don't pollute another + # deployment sharing the same backend model name. + # Each deployment's full pricing is already stored under its + # unique model_id above. + _custom_pricing_fields = CustomPricingLiteLLMParams.model_fields.keys() + _shared_model_info = { + k: v + for k, v in _model_info.items() + if k not in _custom_pricing_fields + } litellm.register_model( model_cost={ - _model_name: _model_info, + _model_name: _shared_model_info, } ) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py new file mode 100644 index 00000000000..2112295e040 --- /dev/null +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -0,0 +1,264 @@ +""" +Test that per-deployment custom pricing does not pollute the shared backend +model key in litellm.model_cost. + +When two deployments share the same backend model (e.g. vertex_ai/gemini-2.5-flash) +and one has explicit zero-cost pricing in model_info, the other deployment +should still use the built-in pricing. +""" + +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import Router + + +def test_should_not_pollute_shared_key_with_zero_cost_pricing(): + """ + When deployment A has input_cost_per_token=0 and deployment B has no + custom pricing, deployment B should still report the built-in pricing + (not zero). + """ + backend_model = "vertex_ai/gemini-2.5-flash" + + # Grab built-in pricing before creating any router + builtin_info = litellm.get_model_info(model=backend_model) + builtin_input_cost = builtin_info["input_cost_per_token"] + builtin_output_cost = builtin_info["output_cost_per_token"] + + # Sanity: built-in pricing should be non-zero for this model + assert builtin_input_cost > 0, "Test requires a model with non-zero built-in pricing" + assert builtin_output_cost > 0, "Test requires a model with non-zero built-in pricing" + + router = Router( + model_list=[ + # Deployment A: explicit zero-cost pricing + { + "model_name": "custom-zero-cost-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-1", + }, + "model_info": { + "id": "deployment-a-zero-cost", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + # Deployment B: no custom pricing, relies on built-in + { + "model_name": "standard-cost-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-2", + }, + "model_info": { + "id": "deployment-b-builtin-cost", + }, + }, + ], + ) + + # Deployment A: should report zero pricing via its unique model_id + info_a = router.get_deployment_model_info( + model_id="deployment-a-zero-cost", + model_name=backend_model, + ) + assert info_a is not None + assert info_a["input_cost_per_token"] == 0.0 + assert info_a["output_cost_per_token"] == 0.0 + + # Deployment B: should report built-in pricing, NOT zero + info_b = router.get_deployment_model_info( + model_id="deployment-b-builtin-cost", + model_name=backend_model, + ) + assert info_b is not None + assert info_b["input_cost_per_token"] == builtin_input_cost, ( + f"Deployment B should use built-in input cost {builtin_input_cost}, " + f"got {info_b['input_cost_per_token']}" + ) + assert info_b["output_cost_per_token"] == builtin_output_cost, ( + f"Deployment B should use built-in output cost {builtin_output_cost}, " + f"got {info_b['output_cost_per_token']}" + ) + + +def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): + """ + A deployment with custom (non-zero) pricing should not overwrite + the shared backend key's built-in pricing. + """ + backend_model = "vertex_ai/gemini-2.5-flash" + + builtin_info = litellm.get_model_info(model=backend_model) + builtin_input_cost = builtin_info["input_cost_per_token"] + + router = Router( + model_list=[ + # Deployment with custom high pricing + { + "model_name": "expensive-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-3", + }, + "model_info": { + "id": "deployment-expensive", + "input_cost_per_token": 0.99, + "output_cost_per_token": 0.99, + }, + }, + # Deployment relying on built-in pricing + { + "model_name": "standard-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-4", + }, + "model_info": { + "id": "deployment-standard", + }, + }, + ], + ) + + # Custom pricing deployment should see its custom values + info_expensive = router.get_deployment_model_info( + model_id="deployment-expensive", + model_name=backend_model, + ) + assert info_expensive is not None + assert info_expensive["input_cost_per_token"] == 0.99 + assert info_expensive["output_cost_per_token"] == 0.99 + + # Standard deployment should still see built-in pricing + info_standard = router.get_deployment_model_info( + model_id="deployment-standard", + model_name=backend_model, + ) + assert info_standard is not None + assert info_standard["input_cost_per_token"] == builtin_input_cost, ( + f"Standard deployment should use built-in pricing {builtin_input_cost}, " + f"got {info_standard['input_cost_per_token']}" + ) + + +def test_should_store_full_pricing_under_deployment_model_id(): + """ + Per-deployment pricing (including zero) should be stored and + retrievable via the unique model_id key in litellm.model_cost. + """ + backend_model = "vertex_ai/gemini-2.5-flash" + + router = Router( + model_list=[ + { + "model_name": "zero-cost-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-5", + }, + "model_info": { + "id": "deployment-zero-check", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ], + ) + + # The model_id entry should exist and have the zero pricing + entry = litellm.model_cost.get("deployment-zero-check") + assert entry is not None, "Deployment should be registered by model_id" + assert entry["input_cost_per_token"] == 0.0 + assert entry["output_cost_per_token"] == 0.0 + + +def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): + """ + The built-in pricing should be preserved no matter which deployment + is processed first (zero-cost first, or standard first). + """ + backend_model = "vertex_ai/gemini-2.5-flash" + + builtin_info = litellm.get_model_info(model=backend_model) + builtin_input_cost = builtin_info["input_cost_per_token"] + builtin_output_cost = builtin_info["output_cost_per_token"] + + # Order 1: standard first, then zero-cost + router1 = Router( + model_list=[ + { + "model_name": "standard-first", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-6", + }, + "model_info": {"id": "order1-standard"}, + }, + { + "model_name": "zero-cost-second", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-7", + }, + "model_info": { + "id": "order1-zero", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ], + ) + + info_std_1 = router1.get_deployment_model_info( + model_id="order1-standard", model_name=backend_model + ) + assert info_std_1["input_cost_per_token"] == builtin_input_cost + assert info_std_1["output_cost_per_token"] == builtin_output_cost + + # Order 2: zero-cost first, then standard + router2 = Router( + model_list=[ + { + "model_name": "zero-cost-first", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-8", + }, + "model_info": { + "id": "order2-zero", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + { + "model_name": "standard-second", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-9", + }, + "model_info": {"id": "order2-standard"}, + }, + ], + ) + + info_std_2 = router2.get_deployment_model_info( + model_id="order2-standard", model_name=backend_model + ) + assert info_std_2["input_cost_per_token"] == builtin_input_cost, ( + f"Order should not matter. Expected {builtin_input_cost}, " + f"got {info_std_2['input_cost_per_token']}" + ) + assert info_std_2["output_cost_per_token"] == builtin_output_cost, ( + f"Order should not matter. Expected {builtin_output_cost}, " + f"got {info_std_2['output_cost_per_token']}" + ) From 969710477fd2b31c61cd0fc7073d62e7bd3b8724 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Tue, 10 Feb 2026 04:58:37 +0100 Subject: [PATCH 069/117] fix(mcp): resolve OAuth2 root endpoints returning "MCP server not found" (#20784) When MCP SDK hits root-level /register, /authorize, /token without server name prefix, auto-resolve to the single configured OAuth2 server. Also fix WWW-Authenticate header to use correct public URL behind reverse proxy. --- .../mcp_server/discoverable_endpoints.py | 54 ++++ .../proxy/_experimental/mcp_server/server.py | 5 +- .../mcp_server/test_discoverable_endpoints.py | 274 ++++++++++++++++++ 3 files changed, 332 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 8b052dd0da1..bdf4cc312d9 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -16,6 +16,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.utils import get_server_root_path +from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer router = APIRouter( @@ -125,6 +126,29 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data +def _resolve_oauth2_server_for_root_endpoints( + client_ip: Optional[str] = None, +) -> Optional[MCPServer]: + """ + Resolve the MCP server for root-level OAuth endpoints (no server name in path). + + When the MCP SDK hits root-level endpoints like /register, /authorize, /token + without a server name prefix, we try to find the right server automatically. + Returns the server if exactly one OAuth2 server is configured, else None. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + registry = global_mcp_server_manager.get_filtered_registry(client_ip=client_ip) + oauth2_servers = [ + s for s in registry.values() if s.auth_type == MCPAuth.oauth2 + ] + if len(oauth2_servers) == 1: + return oauth2_servers[0] + return None + + async def authorize_with_server( request: Request, mcp_server: MCPServer, @@ -305,6 +329,8 @@ async def authorize( mcp_server = global_mcp_server_manager.get_mcp_server_by_name( lookup_name, client_ip=client_ip ) + if mcp_server is None and mcp_server_name is None: + mcp_server = _resolve_oauth2_server_for_root_endpoints() if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") return await authorize_with_server( @@ -350,6 +376,8 @@ async def token_endpoint( mcp_server = global_mcp_server_manager.get_mcp_server_by_name( lookup_name, client_ip=client_ip ) + if mcp_server is None and mcp_server_name is None: + mcp_server = _resolve_oauth2_server_for_root_endpoints() if mcp_server is None: raise HTTPException(status_code=404, detail="MCP server not found") return await exchange_token_with_server( @@ -430,6 +458,13 @@ def _build_oauth_protected_resource_response( ) request_base_url = get_request_base_url(request) + + # When no server name provided, try to resolve the single OAuth2 server + if mcp_server_name is None: + resolved = _resolve_oauth2_server_for_root_endpoints() + if resolved: + mcp_server_name = resolved.server_name or resolved.name + mcp_server: Optional[MCPServer] = None if mcp_server_name: client_ip = IPAddressUtils.get_mcp_client_ip(request) @@ -535,6 +570,12 @@ def _build_oauth_authorization_server_response( request_base_url = get_request_base_url(request) + # When no server name provided, try to resolve the single OAuth2 server + if mcp_server_name is None: + resolved = _resolve_oauth2_server_for_root_endpoints() + if resolved: + mcp_server_name = resolved.server_name or resolved.name + authorization_endpoint = ( f"{request_base_url}/{mcp_server_name}/authorize" if mcp_server_name @@ -640,6 +681,19 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non "redirect_uris": [f"{request_base_url}/callback"], } if not mcp_server_name: + resolved = _resolve_oauth2_server_for_root_endpoints() + if resolved: + return await register_client_with_server( + request=request, + mcp_server=resolved, + client_name=data.get("client_name", ""), + grant_types=data.get("grant_types", []), + response_types=data.get("response_types", []), + token_endpoint_auth_method=data.get( + "token_endpoint_auth_method", "" + ), + fallback_client_id=resolved.server_name or resolved.name, + ) return dummy_return client_ip = IPAddressUtils.get_mcp_client_ip(request) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 890c4ae8fb2..58cd8c99e7b 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -31,6 +31,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) +from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + get_request_base_url, +) from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, @@ -1972,7 +1975,7 @@ if MCP_AVAILABLE: ) if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers: request = StarletteRequest(scope) - base_url = str(request.base_url).rstrip("/") + base_url = get_request_base_url(request) authorization_uri = ( f"Bearer authorization_uri=" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index e4b4d3ba189..faabe40f2dc 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2,6 +2,8 @@ import pytest from unittest.mock import AsyncMock, MagicMock, patch +from fastapi import HTTPException + # Fixture to mock IP address check for all MCP tests # This prevents tests from failing due to IP-based access control @@ -260,10 +262,16 @@ async def test_register_client_without_mcp_server_name_returns_dummy(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( register_client, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry to ensure no OAuth2 servers exist (otherwise resolver would find one) + global_mcp_server_manager.registry.clear() + mock_request = MagicMock(spec=Request) mock_request.base_url = "https://proxy.litellm.example/" mock_request.headers = {} @@ -680,10 +688,16 @@ async def test_register_client_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( register_client, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry to ensure no OAuth2 servers exist (otherwise resolver would find one) + global_mcp_server_manager.registry.clear() + # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) mock_request.base_url = "http://proxy.litellm.example/" # HTTP @@ -1017,3 +1031,263 @@ def test_get_request_base_url_comprehensive( f"X-Forwarded-Host={x_forwarded_host}, " f"X-Forwarded-Port={x_forwarded_port}" ) + + +# ------------------------------------------------------------------- +# Tests for root-level OAuth endpoint resolution (no server name) +# ------------------------------------------------------------------- + + +def _create_oauth2_server( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + client_id="test_client_id", + client_secret="test_client_secret", +): + """Helper to create a mock OAuth2 MCPServer.""" + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport + + return MCPServer( + server_id=server_id, + name=name, + server_name=server_name, + alias=alias, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id=client_id, + client_secret=client_secret, + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + + +@pytest.mark.asyncio +async def test_authorize_root_resolves_single_oauth2_server(): + """When /authorize is hit without server name and exactly 1 OAuth2 server exists, resolve it.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from fastapi import Request + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + oauth2_server = _create_oauth2_server() + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://llm.example.com/" + mock_request.headers = {} + + try: + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" + ) as mock_encrypt: + mock_encrypt.return_value = "mocked_encrypted_state" + + # Call /authorize WITHOUT mcp_server_name, with dummy_client as client_id + response = await authorize( + request=mock_request, + client_id="dummy_client", + mcp_server_name=None, + redirect_uri="http://localhost:62646/callback", + state="test_state", + ) + + # Should resolve to the single OAuth2 server and redirect + assert response.status_code == 307 + location = response.headers["location"] + assert "https://provider.com/oauth/authorize" in location + assert "client_id=test_client_id" in location + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_authorize_root_fails_with_multiple_oauth2_servers(): + """When /authorize is hit without server name and multiple OAuth2 servers exist, return 404.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from fastapi import Request + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + server1 = _create_oauth2_server( + server_id="server1", name="server1", server_name="server1", alias="server1" + ) + server2 = _create_oauth2_server( + server_id="server2", name="server2", server_name="server2", alias="server2" + ) + global_mcp_server_manager.registry[server1.server_id] = server1 + global_mcp_server_manager.registry[server2.server_id] = server2 + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://llm.example.com/" + mock_request.headers = {} + + try: + with pytest.raises(HTTPException) as exc_info: + await authorize( + request=mock_request, + client_id="dummy_client", + mcp_server_name=None, + redirect_uri="http://localhost:62646/callback", + state="test_state", + ) + assert exc_info.value.status_code == 404 + assert "MCP server not found" in str(exc_info.value.detail) + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_token_root_resolves_single_oauth2_server(): + """When /token is hit without server name and exactly 1 OAuth2 server exists, resolve it.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + token_endpoint, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from fastapi import Request + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + oauth2_server = _create_oauth2_server() + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://llm.example.com/" + mock_request.headers = {} + + mock_response = MagicMock() + mock_response.json.return_value = { + "access_token": "ya29.test_token", + "token_type": "Bearer", + "expires_in": 3599, + } + mock_response.raise_for_status = MagicMock() + + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=mock_response) + + try: + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + ) as mock_get_client: + mock_get_client.return_value = mock_async_client + + # Call /token WITHOUT mcp_server_name + response = await token_endpoint( + request=mock_request, + grant_type="authorization_code", + code="test_auth_code", + redirect_uri="http://localhost:62646/callback", + client_id="dummy_client", + mcp_server_name=None, + client_secret=None, + code_verifier="test_verifier", + ) + + # Should resolve and exchange token with the upstream server + import json + + token_data = json.loads(response.body) + assert token_data["access_token"] == "ya29.test_token" + + # Verify it called the correct upstream token URL + call_args = mock_async_client.post.call_args + assert call_args.args[0] == "https://provider.com/oauth/token" + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_register_root_resolves_single_oauth2_server(): + """When /register is hit without server name and exactly 1 OAuth2 server exists, resolve it.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from fastapi import Request + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + oauth2_server = _create_oauth2_server() + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://llm.example.com/" + mock_request.headers = {} + + try: + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body", + new=AsyncMock(return_value={}), + ): + result = await register_client(request=mock_request, mcp_server_name=None) + + # Should resolve to the single server and return its name as client_id + assert result["client_id"] == "test_oauth" + assert "redirect_uris" in result + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_discovery_root_includes_server_name_prefix(): + """When root discovery is hit and exactly 1 OAuth2 server exists, include server name in URLs.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_authorization_server_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from fastapi import Request + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + oauth2_server = _create_oauth2_server() + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://llm.example.com/" + mock_request.headers = {} + + try: + # Call with mcp_server_name=None (root discovery) + response = _build_oauth_authorization_server_response( + request=mock_request, + mcp_server_name=None, + ) + + # Should resolve to the single server and include its name in endpoint URLs + assert "/test_oauth/authorize" in response["authorization_endpoint"] + assert "/test_oauth/token" in response["token_endpoint"] + assert "/test_oauth/register" in response["registration_endpoint"] + assert response["scopes_supported"] == ["read", "write"] + finally: + global_mcp_server_manager.registry.clear() From 9bdb163269be0f073f6777497564438b5ba19efc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 12:06:40 +0530 Subject: [PATCH 070/117] Add error file ids as managed files --- .../proxy/hooks/managed_files.py | 66 +++++++------- .../proxy/hooks/test_managed_files.py | 85 +++++++++++++++++++ 2 files changed, 119 insertions(+), 32 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 569ea17f6d8..7cbee64b08e 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -899,40 +899,42 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): batch_id=response.id, model_id=model_id ) - if ( - response.output_file_id and model_id - ): # return a file id with the model_id and output_file_id - original_output_file_id = response.output_file_id - response.output_file_id = self.get_unified_output_file_id( - output_file_id=response.output_file_id, - model_id=model_id, - model_name=model_name, - ) - - # Fetch the actual file object for the output file - file_object = None - try: - # Use litellm to retrieve the file object from the provider - from litellm import afile_retrieve - file_object = await afile_retrieve( - custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", - file_id=original_output_file_id + # Handle both output_file_id and error_file_id + for file_attr in ["output_file_id", "error_file_id"]: + file_id_value = getattr(response, file_attr, None) + if file_id_value and model_id: + original_file_id = file_id_value + unified_file_id = self.get_unified_output_file_id( + output_file_id=original_file_id, + model_id=model_id, + model_name=model_name, ) - verbose_logger.debug( - f"Successfully retrieved file object for output_file_id={original_output_file_id}" + setattr(response, file_attr, unified_file_id) + + # Fetch the actual file object from the provider + file_object = None + try: + # Use litellm to retrieve the file object from the provider + from litellm import afile_retrieve + file_object = await afile_retrieve( + custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", + file_id=original_file_id + ) + verbose_logger.debug( + f"Successfully retrieved file object for {file_attr}={original_file_id}" + ) + except Exception as e: + verbose_logger.warning( + f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand." + ) + + await self.store_unified_file_id( + file_id=unified_file_id, + file_object=file_object, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_mappings={model_id: original_file_id}, + user_api_key_dict=user_api_key_dict, ) - except Exception as e: - verbose_logger.warning( - f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand." - ) - - await self.store_unified_file_id( - file_id=response.output_file_id, - file_object=file_object, - litellm_parent_otel_span=user_api_key_dict.parent_otel_span, - model_mappings={model_id: original_output_file_id}, - user_api_key_dict=user_api_key_dict, - ) asyncio.create_task( self.store_unified_object_id( unified_object_id=response.id, diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 946c5ad1729..8a22a43c379 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -373,6 +373,91 @@ async def test_output_file_id_for_batch_retrieve(): assert not cast(LiteLLMBatch, response).output_file_id.startswith("file-") +@pytest.mark.asyncio +async def test_error_file_id_for_failed_batch(): + """ + Test that the error_file_id is properly managed when a batch fails + """ + from typing import cast + + from openai.types.batch import BatchRequestCounts + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.llms.openai import OpenAIFileObject + from litellm.types.utils import LiteLLMBatch + + batch = LiteLLMBatch( + id="bGl0ZWxsbV9wcm94eTttb2RlbF9pZDoxMjM0NTY3OTtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz", + completion_window="24h", + created_at=1714508499, + endpoint="/v1/chat/completions", + input_file_id="file-abc123", + object="batch", + status="failed", + cancelled_at=None, + cancelling_at=None, + completed_at=None, + error_file_id="error-abc123", + errors=None, + expired_at=None, + expires_at=1714536634, + failed_at=None, + finalizing_at=None, + in_progress_at=None, + metadata=None, + output_file_id=None, + request_counts=BatchRequestCounts(completed=0, failed=0, total=0), + usage=None, + ) + + batch._hidden_params = { + "litellm_call_id": "test-call-id", + "api_base": "https://api.openai.com", + "model_id": "test-model-id", + "model_name": "gpt-4o", + "response_cost": 0.0, + "additional_headers": {}, + "litellm_model_name": "gpt-4o", + "unified_batch_id": "litellm_proxy;model_id:test-model-id;llm_batch_id:batch_abc123", + } + + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=AsyncMock() + ) + + # Create a proper OpenAIFileObject for the error file + error_file_object = OpenAIFileObject( + id="error-abc123", + object="file", + bytes=1234, + created_at=1714508500, + filename="error.jsonl", + purpose="batch_output", + status="processed", + ) + + # Mock the afile_retrieve to simulate retrieving error file metadata + with patch("litellm.afile_retrieve", new_callable=AsyncMock) as mock_retrieve: + mock_retrieve.return_value = error_file_object + + user_api_key_dict = UserAPIKeyAuth( + user_id="test-user-123", + parent_otel_span=MagicMock() + ) + + response = await proxy_managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=user_api_key_dict, + response=batch, + ) + + # Verify that error_file_id was transformed to a managed file ID + assert cast(LiteLLMBatch, response).error_file_id is not None + assert not cast(LiteLLMBatch, response).error_file_id.startswith("error-") + # Verify it's a base64 encoded managed file ID + assert _is_base64_encoded_unified_file_id(cast(LiteLLMBatch, response).error_file_id) + + @pytest.mark.asyncio async def test_async_post_call_success_hook_twice_assert_no_unique_violation(): import asyncio From 45c8edbe25b8a1e021c546b01e570f0162e347eb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 12:25:26 +0530 Subject: [PATCH 071/117] Fix: Bug: Batch Rate Limiter Cannot Access User Files --- litellm/proxy/hooks/batch_rate_limiter.py | 6 +- tests/batches_tests/test_batch_rate_limits.py | 230 ++++++++++++++++++ .../proxy/hooks/test_managed_files.py | 2 +- 3 files changed, 236 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index ecad8bc1b11..78a371ad66e 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -17,7 +17,7 @@ Quick summary: - async_log_success_event() fires on GET /v1/batches/{id} (batch completion) """ -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from fastapi import HTTPException from pydantic import BaseModel @@ -241,6 +241,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): self, file_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", + user_api_key_dict: Optional[UserAPIKeyAuth] = None, ) -> BatchFileUsage: """ Count number of requests and tokens in a batch input file. @@ -248,6 +249,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): Args: file_id: The file ID to read custom_llm_provider: The custom LLM provider to use for token encoding + user_api_key_dict: User authentication information for file access (required for managed files) Returns: BatchFileUsage with total_tokens and request_count @@ -257,6 +259,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): file_content = await litellm.afile_content( file_id=file_id, custom_llm_provider=custom_llm_provider, + user_api_key_dict=user_api_key_dict, ) file_content_as_dict = _get_file_content_as_dictionary( @@ -336,6 +339,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): batch_usage = await self.count_input_file_usage( file_id=input_file_id, custom_llm_provider=custom_llm_provider, + user_api_key_dict=user_api_key_dict, ) verbose_proxy_logger.debug( diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py index 776aba438c2..13241e94d51 100644 --- a/tests/batches_tests/test_batch_rate_limits.py +++ b/tests/batches_tests/test_batch_rate_limits.py @@ -389,3 +389,233 @@ async def test_batch_rate_limit_multiple_requests(): print(f" Error: {exc_info.value.detail}") finally: os.unlink(file_path_2) + + +@pytest.mark.asyncio() +@pytest.mark.skipif( + os.environ.get("OPENAI_API_KEY") is None, + reason="OPENAI_API_KEY not set - skipping integration test" +) +async def test_batch_rate_limiter_with_managed_files(): + """ + Test for GEN-2166: Verify batch rate limiter can read user files when managed files are enabled. + + This test ensures that: + 1. The batch rate limiter passes user_api_key_dict to afile_content() + 2. The managed files hook can verify file ownership correctly + 3. Rate limiting is enforced (not silently bypassed) + 4. No 403 Permission Denied errors occur for files owned by the user + """ + import tempfile + from unittest.mock import AsyncMock, MagicMock, patch + + CUSTOM_LLM_PROVIDER = "openai" + + # Setup: Create internal usage cache and rate limiter + dual_cache = DualCache() + internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) + rate_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=internal_usage_cache + ) + + # Setup: Get batch rate limiter + batch_limiter = rate_limiter._get_batch_rate_limiter() + assert batch_limiter is not None, "Batch rate limiter should be available" + + # Setup: Create user API key with TPM = 500, RPM = 10 + test_user_id = "test-user-abc123" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key-managed-files", + user_id=test_user_id, + tpm_limit=500, + rpm_limit=10, + ) + + print(f"\n=== Testing Batch Rate Limiter with Managed Files ===") + print(f"User ID: {test_user_id}") + + # Create a batch file with ~200 tokens + import json as json_lib + message = "This is a test message for batch rate limiting with managed files. " * 5 + requests = [] + for i in range(1, 4): + request_obj = { + "custom_id": f"request-{i}", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": message}] + } + } + requests.append(json_lib.dumps(request_obj)) + + batch_content = "\n".join(requests) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + f.write(batch_content) + file_path = f.name + + try: + # Step 1: Upload file to OpenAI (simulating user upload) + print("\n1. Uploading batch input file...") + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="batch", + custom_llm_provider=CUSTOM_LLM_PROVIDER, + ) + print(f" ✓ File uploaded: {file_obj.id}") + await asyncio.sleep(1) # Give API time to process + + # Step 2: Mock managed files hook to simulate file ownership check + # In a real scenario, the managed files hook would check if the user owns the file + # For this test, we'll verify that user_api_key_dict is passed correctly + print("\n2. Testing rate limiter file access with user context...") + + # Track if user_api_key_dict was passed to afile_content + original_afile_content = litellm.afile_content + user_context_passed = {"value": False} + + async def mock_afile_content(*args, **kwargs): + # Check if user_api_key_dict was passed + if "user_api_key_dict" in kwargs and kwargs["user_api_key_dict"] is not None: + user_context_passed["value"] = True + print(f" ✓ user_api_key_dict passed to afile_content") + print(f" User ID: {kwargs['user_api_key_dict'].user_id}") + else: + print(f" ✗ user_api_key_dict NOT passed to afile_content (BUG!)") + + # Call original function + return await original_afile_content(*args, **kwargs) + + # Patch afile_content to track the call + with patch('litellm.afile_content', side_effect=mock_afile_content): + data = { + "model": "gpt-3.5-turbo", + "input_file_id": file_obj.id, + "custom_llm_provider": CUSTOM_LLM_PROVIDER, + } + + # Step 3: Submit batch and verify rate limiting works + print("\n3. Submitting batch with rate limiting...") + result = await batch_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=dual_cache, + data=data, + call_type="acreate_batch", + ) + + tokens_used = result.get('_batch_token_count', 0) + requests_count = result.get('_batch_request_count', 0) + print(f" ✓ Batch submitted successfully") + print(f" Tokens counted: {tokens_used}") + print(f" Requests counted: {requests_count}") + print(f" Rate limit usage: {tokens_used}/500 TPM, {requests_count}/10 RPM") + + # Step 4: Verify user context was passed + print("\n4. Verifying fix for GEN-2166...") + assert user_context_passed["value"], ( + "FAILED: user_api_key_dict was not passed to afile_content(). " + "This means the bug GEN-2166 is not fixed!" + ) + print(" ✓ Fix verified: user_api_key_dict is correctly passed") + + # Step 5: Verify rate limiting is actually enforced (not bypassed) + print("\n5. Verifying rate limiting is enforced...") + assert tokens_used > 0, "Token count should be greater than 0" + assert requests_count > 0, "Request count should be greater than 0" + print(" ✓ Rate limiting is active (not silently bypassed)") + + print("\n=== Test Passed: GEN-2166 Fix Verified ===") + print("✓ Batch rate limiter can access user files") + print("✓ User context is correctly passed") + print("✓ Rate limiting is enforced") + print("✓ No silent failures") + + except HTTPException as e: + if e.status_code == 403: + pytest.fail( + f"FAILED: Got 403 Permission Denied error. " + f"This indicates the bug GEN-2166 is not fixed. " + f"Error: {e.detail}" + ) + else: + raise + except Exception as e: + pytest.fail(f"Unexpected error: {str(e)}") + finally: + os.unlink(file_path) + + +@pytest.mark.asyncio() +async def test_batch_rate_limiter_without_user_context(): + """ + Test that verifies the bug scenario from GEN-2166. + + When user_api_key_dict is NOT passed to count_input_file_usage(), + the function should still work for non-managed files, but would fail + for managed files (which is the bug we fixed). + + This test documents the expected behavior with and without user context. + """ + import tempfile + + CUSTOM_LLM_PROVIDER = "openai" + + # Setup + BATCH_LIMITER = _PROXY_BatchRateLimiter( + internal_usage_cache=None, + parallel_request_limiter=None, + ) + + # Create a simple batch file + batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}}""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + f.write(batch_content) + file_path = f.name + + try: + # Upload file + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="batch", + custom_llm_provider=CUSTOM_LLM_PROVIDER, + ) + await asyncio.sleep(1) + + # Test 1: Without user context (old behavior - would fail with managed files) + print("\n=== Test 1: count_input_file_usage WITHOUT user context ===") + try: + usage_without_context = await BATCH_LIMITER.count_input_file_usage( + file_id=file_obj.id, + custom_llm_provider=CUSTOM_LLM_PROVIDER, + user_api_key_dict=None, # Explicitly passing None + ) + print(f"✓ Works for non-managed files (tokens: {usage_without_context.total_tokens})") + print(" Note: Would fail with 403 for managed files (GEN-2166 bug)") + except Exception as e: + print(f"✗ Failed: {str(e)}") + + # Test 2: With user context (new behavior - works with managed files) + print("\n=== Test 2: count_input_file_usage WITH user context ===") + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + user_id="test-user-123", + ) + + usage_with_context = await BATCH_LIMITER.count_input_file_usage( + file_id=file_obj.id, + custom_llm_provider=CUSTOM_LLM_PROVIDER, + user_api_key_dict=user_api_key_dict, # Passing user context + ) + print(f"✓ Works with user context (tokens: {usage_with_context.total_tokens})") + print(" Note: This fixes GEN-2166 for managed files") + + # Verify both return the same results + assert usage_with_context.total_tokens == usage_without_context.total_tokens + assert usage_with_context.request_count == usage_without_context.request_count + print("\n✓ Both methods return identical results for non-managed files") + + finally: + os.unlink(file_path) diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 8a22a43c379..ff957ed395b 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -934,7 +934,7 @@ async def test_check_file_ids_access_with_unified_file_ids(): Test that check_file_ids_access validates user access to managed file IDs. """ from litellm.proxy._types import UserAPIKeyAuth - + # Create a unified file ID unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1fb3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRfZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFkMDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" regular_file_id = "file-abc123" From ce4bebbedfe5cf7f6ddc276cd9c7b9b24a08c825 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 12:42:39 +0530 Subject: [PATCH 072/117] Changed asyncio.create_task() to await for storing batch objects --- .../proxy/hooks/managed_files.py | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 7cbee64b08e..a41b3f3bf6f 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -935,15 +935,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_mappings={model_id: original_file_id}, user_api_key_dict=user_api_key_dict, ) - asyncio.create_task( - self.store_unified_object_id( - unified_object_id=response.id, - file_object=response, - litellm_parent_otel_span=user_api_key_dict.parent_otel_span, - model_object_id=original_response_id, - file_purpose="batch", - user_api_key_dict=user_api_key_dict, - ) + await self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="batch", + user_api_key_dict=user_api_key_dict, ) elif isinstance(response, LiteLLMFineTuningJob): ## Check if unified_file_id is in the response @@ -960,15 +958,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): response.id = self.get_unified_generic_response_id( model_id=model_id, generic_response_id=response.id ) - asyncio.create_task( - self.store_unified_object_id( - unified_object_id=response.id, - file_object=response, - litellm_parent_otel_span=user_api_key_dict.parent_otel_span, - model_object_id=original_response_id, - file_purpose="fine-tune", - user_api_key_dict=user_api_key_dict, - ) + await self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="fine-tune", + user_api_key_dict=user_api_key_dict, ) elif isinstance(response, AsyncCursorPage): """ From cb8ce09b0d1ce2a744cb20f20e5d0000545bd23f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 13:37:17 +0530 Subject: [PATCH 073/117] Add support for langchain_aws via litellm passthrough --- docs/my-website/docs/pass_through/bedrock.md | 144 ++++++++++++++++++ litellm/proxy/auth/user_api_key_auth.py | 27 ++++ .../llm_passthrough_endpoints.py | 50 +++++- .../proxy/auth/test_user_api_key_auth.py | 14 +- 4 files changed, 224 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index b8d20d77da0..65c5d8caadc 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -556,3 +556,147 @@ for event in response.get("completion"): print(completion) ``` + +## Using LangChain AWS SDK with LiteLLM + +You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integrations/chat/bedrock/) with LiteLLM Proxy to get cost tracking, load balancing, and other LiteLLM features. + +### Quick Start + +**1. Install LangChain AWS**: + +```bash showLineNumbers +pip install langchain-aws +``` + +**2. Setup LiteLLM Proxy**: + +Create a `config.yaml`: + +```yaml showLineNumbers +model_list: + - model_name: claude-sonnet + litellm_params: + model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 + aws_region_name: us-east-1 + custom_llm_provider: bedrock +``` + +Start the proxy: + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" + +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**3. Use LangChain with LiteLLM**: + +```python showLineNumbers +from langchain_aws import ChatBedrockConverse +from langchain_core.messages import HumanMessage + +# Your LiteLLM API key +API_KEY = "Bearer sk-1234" + +# Initialize ChatBedrockConverse pointing to LiteLLM proxy +llm = ChatBedrockConverse( + model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + endpoint_url="http://localhost:4000/bedrock", + region_name="us-east-1", + aws_access_key_id=API_KEY, + aws_secret_access_key="bedrock" # Any non-empty value works +) + +# Invoke the model +messages = [HumanMessage(content="Hello, how are you?")] +response = llm.invoke(messages) + +print(response.content) +``` + +### Advanced Example: PDF Document Processing with Citations + +LangChain AWS SDK supports Bedrock's document processing features. Here's how to use it with LiteLLM: + +```python showLineNumbers +import os +import json +from langchain_aws import ChatBedrockConverse +from langchain_core.messages import HumanMessage + +# Your LiteLLM API key +API_KEY = "Bearer sk-1234" + +def get_llm() -> ChatBedrockConverse: + """Initialize LLM pointing to LiteLLM proxy""" + llm = ChatBedrockConverse( + model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + base_model_id="anthropic.claude-3-7-sonnet-20250219-v1:0", + endpoint_url="http://localhost:4000/bedrock", + region_name="us-east-1", + aws_access_key_id=API_KEY, + aws_secret_access_key="bedrock" + ) + return llm + +if __name__ == "__main__": + # Initialize the LLM + llm = get_llm() + + # Read PDF file as bytes (Converse API requires raw bytes) + with open("your-document.pdf", "rb") as file: + file_bytes = file.read() + + # Prepare messages with document attachment + messages = [ + HumanMessage(content=[ + {"text": "What is the policy number in this document?"}, + { + "document": { + "format": "pdf", + "name": "PolicyDocument", + "source": {"bytes": file_bytes}, + "citations": {"enabled": True} + } + } + ]) + ] + + # Invoke the LLM + response = llm.invoke(messages) + + # Print response with citations + print(json.dumps(response.content, indent=4)) +``` + +### Supported LangChain Features + +All LangChain AWS features work with LiteLLM: + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Text Generation | ✅ | Full support | +| Streaming | ✅ | Use `stream()` method | +| Document Processing | ✅ | PDF, images, etc. | +| Citations | ✅ | Enable in document config | +| Tool Use | ✅ | Function calling support | +| Multi-modal | ✅ | Text + images + documents | + +### Troubleshooting + +**Issue**: `UnknownOperationException` error + +**Solution**: Make sure you're using the correct endpoint URL format: +- ✅ Correct: `http://localhost:4000/bedrock` +- ❌ Wrong: `http://localhost:4000/bedrock/v2` + +**Issue**: Authentication errors + +**Solution**: Ensure your API key is in the correct format: +```python +aws_access_key_id="Bearer sk-1234" # Include "Bearer " prefix +``` diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 05eeab3f611..42f10ff8598 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -8,6 +8,7 @@ Returns a UserAPIKeyAuth object if the API key is valid """ import asyncio +import re import secrets from datetime import datetime, timezone from typing import List, Optional, Tuple, cast @@ -115,6 +116,18 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str: api_key = api_key.replace("Basic ", "") # handle langfuse input elif api_key.startswith("bearer "): api_key = api_key.replace("bearer ", "") + elif api_key.startswith("AWS4-HMAC-SHA256"): + # Handle AWS Signature V4 format from LangChain + # Format: AWS4-HMAC-SHA256 Credential=Bearer sk-12345/date/region/service/aws4_request, SignedHeaders=..., Signature=... + # Extract the Bearer token from the Credential field + match = re.search(r'Credential=Bearer\s+([^/\s,]+)', api_key) + if match: + api_key = match.group(1) + else: + # If no Bearer token found in Credential, try to extract just the credential value + match = re.search(r'Credential=([^/\s,]+)', api_key) + if match: + api_key = match.group(1) return api_key @@ -128,6 +141,20 @@ def _get_bearer_token( api_key = api_key.replace("Basic ", "") # handle langfuse input elif api_key.startswith("bearer "): api_key = api_key.replace("bearer ", "") + elif api_key.startswith("AWS4-HMAC-SHA256"): + # Handle AWS Signature V4 format from LangChain + # Format: AWS4-HMAC-SHA256 Credential=Bearer sk-12345/date/region/service/aws4_request, SignedHeaders=..., Signature=... + # Extract the Bearer token from the Credential field + match = re.search(r'Credential=Bearer\s+([^/\s,]+)', api_key) + if match: + api_key = match.group(1) + else: + # If no Bearer token found in Credential, try to extract just the credential value + match = re.search(r'Credential=([^/\s,]+)', api_key) + if match: + api_key = match.group(1) + else: + api_key = "" else: api_key = "" return api_key diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 3dab6ea14f8..81144ad9f31 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -642,10 +642,10 @@ def _extract_model_from_bedrock_endpoint(endpoint: str) -> str: by finding the action in the endpoint and extracting everything between "model" and the action. Args: - endpoint: The endpoint path (e.g., "/model/aws/anthropic/model-name/invoke") + endpoint: The endpoint path (e.g., "/model/aws/anthropic/model-name/invoke" or "v2/model/model-name/invoke") Returns: - The extracted model name (e.g., "aws/anthropic/model-name") + The extracted model name (e.g., "aws/anthropic/model-name" or "model-name") Raises: ValueError: If model cannot be extracted from endpoint @@ -657,7 +657,34 @@ def _extract_model_from_bedrock_endpoint(endpoint: str) -> str: # Format: model/application-inference-profile/{profile-id}/{action} return "/".join(endpoint_parts[1:3]) - # Format: model/{modelId}/{action} + # Format: model/{modelId}/{action} or v2/model/{modelId}/{action} + # Find the index of "model" in the endpoint parts + model_index = None + for idx, part in enumerate(endpoint_parts): + if part == "model": + model_index = idx + break + + # If "model" keyword not found, try to extract model from the endpoint + # by finding the action and taking everything before it + if model_index is None: + # Find the index of the action in the endpoint parts + action_index = None + for idx, part in enumerate(endpoint_parts): + if part in BEDROCK_ENDPOINT_ACTIONS: + action_index = idx + break + + if action_index is not None and action_index > 1: + # Join all parts before the action (excluding empty strings) + model_parts = [p for p in endpoint_parts[1:action_index] if p] + if model_parts: + return "/".join(model_parts) + + raise ValueError( + f"'model' keyword not found and unable to extract model from endpoint. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}" + ) + # Find the index of the action in the endpoint parts action_index = None for idx, part in enumerate(endpoint_parts): @@ -665,13 +692,22 @@ def _extract_model_from_bedrock_endpoint(endpoint: str) -> str: action_index = idx break - if action_index is not None and action_index > 1: - # Join all parts between "model" and the action - return "/".join(endpoint_parts[1:action_index]) + if action_index is not None and action_index > model_index + 1: + # Join all parts between "model" and the action (excluding "model" itself) + return "/".join(endpoint_parts[model_index + 1:action_index]) # Fallback to taking everything after "model" if no action found - return "/".join(endpoint_parts[1:]) + model_parts = [p for p in endpoint_parts[model_index + 1:] if p] + if model_parts: + return "/".join(model_parts) + raise ValueError( + f"No model ID found after 'model' keyword. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}" + ) + + except ValueError: + # Re-raise ValueError as-is + raise except Exception as e: raise ValueError( f"Model missing from endpoint. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}" diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 1f379f4371e..9b7b7f46155 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -41,6 +41,12 @@ def test_get_api_key(): ("Basic sk-12345678", "sk-12345678", "Basic sk-12345678"), ("bearer sk-12345678", "sk-12345678", "bearer sk-12345678"), ("sk-12345678", "sk-12345678", "sk-12345678"), + # AWS Signature V4 format (LangChain AWS SDK) + ( + "AWS4-HMAC-SHA256 Credential=Bearer sk-12345678/20260210/us-east-1/bedrock/aws4_request, SignedHeaders=host, Signature=abc123", + "sk-12345678", + "AWS4-HMAC-SHA256 Credential=Bearer sk-12345678/20260210/us-east-1/bedrock/aws4_request, SignedHeaders=host, Signature=abc123", + ), ], ) def test_get_api_key_with_custom_litellm_key_header( @@ -243,10 +249,10 @@ async def test_proxy_admin_expired_key_from_cache(): Regression test for issue where PROXY_ADMIN keys from cache skipped expiration check. """ from datetime import datetime, timedelta, timezone - + from fastapi import Request from starlette.datastructures import URL - + from litellm.proxy._types import ( LitellmUserRoles, ProxyErrorTypes, @@ -255,7 +261,7 @@ async def test_proxy_admin_expired_key_from_cache(): ) from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder from litellm.proxy.proxy_server import hash_token - + # Create an expired PROXY_ADMIN key api_key = "sk-test-proxy-admin-key" hashed_key = hash_token(api_key) @@ -368,7 +374,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): from user_obj attributes. """ from datetime import datetime - + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import _return_user_api_key_auth_obj From e3c00027b91e05d896ee2c18e0070d96cb0f922f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 14:06:18 +0530 Subject: [PATCH 074/117] Handle image in assitant message for gemini --- .../llms/vertex_ai/gemini/transformation.py | 21 ++ .../test_vertex_ai_gemini_transformation.py | 266 +++++++++++++++++- 2 files changed, 273 insertions(+), 14 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 3004f39b973..1d55629f079 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -437,6 +437,27 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 else: assistant_content.append(PartType(text=assistant_text)) # type: ignore + ## HANDLE ASSISTANT IMAGES FIELD + # Process images field if present (for generated images from assistant) + assistant_images = assistant_msg.get("images") + if assistant_images is not None and isinstance(assistant_images, list): + for image_item in assistant_images: + if isinstance(image_item, dict): + image_url_obj = image_item.get("image_url") + if isinstance(image_url_obj, dict): + image_url = image_url_obj.get("url") + format = image_url_obj.get("format") + detail = image_url_obj.get("detail") + media_resolution_enum = _convert_detail_to_media_resolution_enum(detail) + if image_url: + _part = _process_gemini_media( + image_url=image_url, + format=format, + media_resolution_enum=media_resolution_enum, + model=model, + ) + assistant_content.append(_part) + ## HANDLE ASSISTANT FUNCTION CALL if ( assistant_msg.get("tool_calls", []) is not None diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index a9c27e30930..c474461e0a2 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -7,6 +7,7 @@ from litellm.llms.vertex_ai.gemini.transformation import ( check_if_part_exists_in_parts, ) from litellm.types.llms.vertex_ai import BlobType +from litellm.types.utils import Message def test_check_if_part_exists_in_parts(): @@ -735,8 +736,9 @@ def test_file_data_field_order(): Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order. """ import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - + # Test with HTTPS URL and explicit format (audio file) file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123" format = "audio/mpeg" @@ -770,8 +772,9 @@ def test_file_data_field_order(): def test_file_data_field_order_gcs_urls(): """Test that GCS URLs also maintain correct field order.""" import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - + # Test with GCS URL gcs_url = "gs://bucket/audio.mp3" @@ -802,11 +805,14 @@ def test_extract_file_data_with_path_object(): Related issue: Files uploaded with wrong MIME type cause Gemini API to reject requests where the specified format doesn't match the uploaded file's MIME type. """ - from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data - import tempfile import os - + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + # Create a temporary MP3 file with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp: tmp.write(b"fake mp3 content") @@ -835,10 +841,13 @@ def test_extract_file_data_with_path_object(): def test_extract_file_data_with_string_path(): """Test that filename is correctly extracted from string paths.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data - import tempfile import os - + import tempfile + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + # Create a temporary WAV file with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: tmp.write(b"fake wav content") @@ -866,8 +875,10 @@ def test_extract_file_data_with_string_path(): def test_extract_file_data_with_tuple_format(): """Test that tuple format (with explicit content_type) still works correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data - + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + # Test with tuple format: (filename, content, content_type) filename = "test_audio.mp3" content = b"test audio content" @@ -883,10 +894,13 @@ def test_extract_file_data_with_tuple_format(): def test_extract_file_data_fallback_to_octet_stream(): """Test that unknown file types fall back to application/octet-stream.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data - import tempfile import os - + import tempfile + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + # Create a temporary file with unknown extension with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp: tmp.write(b"unknown content") @@ -1086,3 +1100,227 @@ def test_convert_tool_response_with_nested_file_object(): assert "mime_type" in inline_data assert inline_data["mime_type"] == "application/pdf" assert inline_data["data"] == test_pdf_base64 + +def test_assistant_message_with_images_field(): + """ + Test that assistant messages with images field are properly converted to Gemini format. + + This handles the case where an assistant message contains generated images in the + `images` field (e.g., from image generation models like gemini-2.5-flash-image). + The images should be converted to inline_data parts in the Gemini format. + """ + # Create a small test image (1x1 red pixel PNG) + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + # Create messages with assistant message containing images field + messages = [ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM" + }, + { + "role": "assistant", + "content": "Here's your banana in a LiteLLM costume!", + "images": [ + { + "image_url": { + "url": image_data_uri, + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } + ] + } + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify structure + assert len(contents) == 2, f"Expected 2 content blocks, got {len(contents)}" + + # Verify user message + assert contents[0]["role"] == "user" + assert len(contents[0]["parts"]) == 1 + assert contents[0]["parts"][0]["text"] == "Generate an image of a banana wearing a costume that says LiteLLM" + + # Verify assistant message + assert contents[1]["role"] == "model" + assert len(contents[1]["parts"]) == 2, f"Expected 2 parts (text + image), got {len(contents[1]['parts'])}" + + # Find text part and inline_data part + text_part = None + inline_data_part = None + for part in contents[1]["parts"]: + if "text" in part: + text_part = part + elif "inline_data" in part: + inline_data_part = part + + # Verify text part + assert text_part is not None, "Missing text part in assistant message" + assert text_part["text"] == "Here's your banana in a LiteLLM costume!" + + # Verify inline_data part (image) + assert inline_data_part is not None, "Missing inline_data part in assistant message" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "image/png" + assert inline_data["data"] == test_image_base64 + + +def test_assistant_message_with_multiple_images(): + """Test that assistant messages with multiple images are properly converted.""" + # Create two test images + test_image1_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + test_image2_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image1_data_uri = f"data:image/png;base64,{test_image1_base64}" + image2_data_uri = f"data:image/jpeg;base64,{test_image2_base64}" + + messages = [ + { + "role": "user", + "content": "Generate two images" + }, + { + "role": "assistant", + "content": "Here are your images:", + "images": [ + { + "image_url": { + "url": image1_data_uri, + "detail": "auto" + }, + "index": 0, + "type": "image_url" + }, + { + "image_url": { + "url": image2_data_uri, + "detail": "high" + }, + "index": 1, + "type": "image_url" + } + ] + } + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify assistant message has 3 parts (1 text + 2 images) + assert contents[1]["role"] == "model" + assert len(contents[1]["parts"]) == 3, f"Expected 3 parts (text + 2 images), got {len(contents[1]['parts'])}" + + # Count inline_data parts + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert len(inline_data_parts) == 2, f"Expected 2 inline_data parts, got {len(inline_data_parts)}" + + # Verify first image + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_data_parts[0]["inline_data"]["data"] == test_image1_base64 + + # Verify second image + assert inline_data_parts[1]["inline_data"]["mime_type"] == "image/jpeg" + assert inline_data_parts[1]["inline_data"]["data"] == test_image2_base64 + + +def test_assistant_message_with_images_using_message_object(): + """Test that Message objects with images field are properly converted.""" + # Create a small test image + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + # Create messages using Message object (as returned by LiteLLM) + user_message = { + "role": "user", + "content": "Generate an image" + } + + assistant_message = Message( + content="Here's your image!", + role="assistant", + tool_calls=None, + function_call=None, + images=[ + { + "image_url": { + "url": image_data_uri, + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } + ] + ) + + messages = [user_message, assistant_message] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify assistant message has both text and image + assert contents[1]["role"] == "model" + assert len(contents[1]["parts"]) == 2 + + # Verify image was converted + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert len(inline_data_parts) == 1 + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_data_parts[0]["inline_data"]["data"] == test_image_base64 + + +def test_assistant_message_with_images_in_conversation_history(): + """ + Test multi-turn conversation where assistant message with images is in history. + + This simulates the real use case where: + 1. User asks for image generation + 2. Assistant generates image (with images field) + 3. User asks follow-up question about the image + """ + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + messages = [ + { + "role": "user", + "content": "Generate an image of a cat" + }, + { + "role": "assistant", + "content": "Here's a cat image:", + "images": [ + { + "image_url": { + "url": image_data_uri, + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } + ] + }, + { + "role": "user", + "content": "Can you make it more colorful?" + } + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify structure: user -> model (with image) -> user + assert len(contents) == 3 + assert contents[0]["role"] == "user" + assert contents[1]["role"] == "model" + assert contents[2]["role"] == "user" + + # Verify assistant message has image in history + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert len(inline_data_parts) == 1 + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" \ No newline at end of file From ab670a74f4fe9f77acfae5e600f483af26a050a9 Mon Sep 17 00:00:00 2001 From: Praveena Mundolimoole <103165192+Praveena-617@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:27:19 +0530 Subject: [PATCH 075/117] Add support for extra fields in Generic SSO via GENERIC_USER_EXTRA_ATTRIBUTES (#20761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add chat completion support for websearch * Add chat completion tool calls support and response transformation * Add new methods in chat completion * Add chat completion tool format * Add callback for websearch in completion method * Add test for web search * Potential fix for code scanning alert no. 4046: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * Update litellm/integrations/websearch_interception/tools.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: empty guardrails/policies arrays should not trigger enterprise license check (#20567) * fix: empty guardrails/policies arrays should not trigger enterprise license check (#20304) The UI sends empty arrays for enterprise-only fields (guardrails, policies, logging) even when the user has not configured these features. The backend `is not None` check treated `[]` as a truthy intent to use the feature, falsely requiring an enterprise license for basic team operations. Backend: Add `and updated_kv[field] != [] and updated_kv[field] != {}` guards in `_update_metadata_fields` so empty collections are skipped. UI: Conditionally omit guardrails, logging, and policies from the payload when empty instead of defaulting to `[]`. Fixes #20304 * fix: allow clearing fields with empty collections while skipping enterprise check Address PR review feedback: 1. Move the empty-collection guard into _update_metadata_field (singular) so that empty lists/dicts skip only the premium license check but still get written into metadata. This lets users intentionally clear a previously-set field (e.g. guardrails: []) without being blocked, while the UI's default empty arrays still don't trigger a false enterprise error. 2. Remove sys.path hack from test file; use standard imports that work with pytest discovery. 3. Add tests verifying that empty collections are moved into metadata (field clearing works) even though they bypass the premium check. Fixes #20304 * fix critical CVE vulnerabliltes (#20683) * fix: add hook to handle db case (#20635) * Add team policy mapping for zguard (#20608) * support policy mapping on team key level * update document * update document * address comments * update document * add unit test for new feature * add more test case * feat: add support for anthropic_messages call type in prompt caching (#19233) * feat: add support for anthropic_messages call type in prompt caching * test: move anthropic_messages prompt caching test to main router test file * add tutorial on using claude code with prompt cache routing * docs: add SDK proxy authentication (OAuth2/JWT auto-refresh) documentation (#20680) Adds documentation for the litellm.proxy_auth feature that automatically obtains and refreshes OAuth2/JWT tokens when connecting to a LiteLLM Proxy. * Fixes #20582 (#20663) * fix: show error details instead of Data Not Available for failed requests (#20656) * fix(ui): add null guard for models in API keys table (#20655) The VirtualKeysTable crashed when rendering keys with null or undefined models field. The className expression tried to access .length on null, throwing a TypeError that broke the entire keys table. Added Array.isArray() guard before accessing .length on the models value. Fixes #20611 * Fix: Spend logs pickle error with Pydantic models and redaction (#20685) * docs: add callback registration optimization to v1.81.9 release notes (#20681) * docs: add callback registration optimization to v1.81.9 release notes * Update v1.81.9.md --------- Co-authored-by: Alexsander Hamir * Fix spend logs pickle error with Pydantic models Replace copy.deepcopy() with Pydantic-safe serialization to avoid "cannot pickle '_thread.RLock' object" errors when request/response redaction is enabled. Changes: - Add _convert_to_json_serializable_dict() helper that uses model_dump() for Pydantic models instead of pickle - Replace copy.deepcopy() calls in request and response redaction paths with the new helper function - Recursively handles nested dicts, lists, and Pydantic models Root cause: Pydantic v2 BaseModel instances contain internal _thread.RLock objects for thread-safety. When copy.deepcopy() attempts to pickle these objects, it fails because threading primitives cannot be pickled. Fixes #20647 * chore: remove unused copy import Remove unused copy import that was causing lint failure. The copy.deepcopy() calls were replaced with _convert_to_json_serializable_dict() helper function in the previous commit, making the copy module no longer needed. --------- Co-authored-by: ryan-crabbe <128659760+ryan-crabbe@users.noreply.github.com> Co-authored-by: Alexsander Hamir * fix(vertex_ai): propagate extra_headers anthropic-beta to request body (#20666) Vertex AI requires Anthropic beta flags in the request body (anthropic_beta array), not as HTTP headers. The Bedrock handler already extracts user-specified beta headers from the headers dict, but the Vertex handler was missing this, causing extra_headers like interleaved-thinking-2025-05-14 to be silently dropped. This extracts anthropic-beta values from optional_params extra_headers and merges them into the anthropic_beta request body field, and also removes extra_headers from the request body since the parent's transform_request spreads optional_params into data. * fix(streaming): preserve interleaved thinking/redacted blocks * test(streaming): build thinking chunks with typed Delta/StreamingChoices * Fix video list pagination cursors not encoded with provider metadata first_id and last_id in the video list response were returned as raw provider IDs while data[].id was properly wrapped with encode_video_id_with_provider(). This caused pagination to break when clients passed unencoded cursors back as the `after` parameter. - Encode first_id/last_id in transform_video_list_response - Decode the `after` param in transform_video_list_request via extract_original_video_id() - Add 6 unit tests covering encoding, decoding, passthrough, and full round-trip pagination Fixes #20708 Co-Authored-By: Claude Opus 4.6 * fix(responses): preserve streamed tool deltas when id is omitted * fix(responses): guard ambiguous tool-call index reuse * Add compaction for vertex ai * Add all new feat for v1/messages * Add inference_geo as supported messages param * Add inference based costing * Add inference_geo as supported messages param * Add support for fast param * Add fast mode for other providers * Add documentation for Fast Mode * add missing indexes on VerificationToken table * Fix structured response of tool call * Add tests for WebSearch interception with chat completions API * Add doc for chat completion web search * Fix: is_web_search_tool_chat_completion * Fix double json import * Add new vercel ai anthropic models * Fix: base_model name for body and deplyment name in URL * Add output_config as supported param * Add response schema for vercel ai sonnet 4.5 * handle when litellm_parrams might be none * Fix : litellm/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py * fix: Missing return statement for async streaming * Fix: get_supported_anthropic_messages_params * Fix mypy issues * Fix mypy issues * Add support for extra fields in Generic SSO via GENERIC_USER_EXTRA_ATTRIBUTES Enables extraction of additional fields from the Generic SSO userinfo endpoint response beyond the standard 8 fields (id, email, name, etc.). Custom handlers can now access these fields via CustomOpenID.extra_fields dict. Changes: - Add extra_fields: Optional[Dict[str, Any]] to CustomOpenID type - Add GENERIC_USER_EXTRA_ATTRIBUTES env var (comma-separated field names) - Extract specified fields using get_nested_value() with dot notation support - Add 4 test cases covering basic, nested, and missing field scenarios - Update custom_sso.py example showing how to access extra_fields Backward compatible: extra_fields is None when env var not set * docs: Add documentation for GENERIC_USER_EXTRA_ATTRIBUTES Document the new GENERIC_USER_EXTRA_ATTRIBUTES environment variable for Generic SSO - Add to admin_ui_sso.md: explanation and usage examples - Add to config_settings.md: environment variable reference - Add to custom_sso.md: code example showing how to access extra_fields - Includes examples for nested field paths with dot notation --------- Co-authored-by: Sameer Kankute Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> Co-authored-by: Varun Chawla <34209028+veeceey@users.noreply.github.com> Co-authored-by: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Co-authored-by: jwang-gif Co-authored-by: nuernber Co-authored-by: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Co-authored-by: John Lathouwers Co-authored-by: ryan-crabbe <128659760+ryan-crabbe@users.noreply.github.com> Co-authored-by: Alexsander Hamir Co-authored-by: Elias Högbom Aronsson Co-authored-by: Emerson Gomes Co-authored-by: tshushan Co-authored-by: Claude Opus 4.6 Co-authored-by: Carlo Alberto Ferraris --- docs/my-website/docs/proxy/admin_ui_sso.md | 35 + docs/my-website/docs/proxy/config_settings.md | 1 + docs/my-website/docs/proxy/custom_sso.md | 12 + litellm/proxy/custom_sso.py | 9 +- litellm/proxy/management_endpoints/types.py | 3 +- litellm/proxy/management_endpoints/ui_sso.py | 11 + .../proxy/management_endpoints/test_ui_sso.py | 120 ++ .../src/components/team/team_info.tsx | 1296 +++++++++++++++++ 8 files changed, 1483 insertions(+), 4 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/team/team_info.tsx diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md index 37e45b50284..f88d3480446 100644 --- a/docs/my-website/docs/proxy/admin_ui_sso.md +++ b/docs/my-website/docs/proxy/admin_ui_sso.md @@ -223,6 +223,7 @@ GENERIC_USER_FIRST_NAME_ATTRIBUTE = "first_name" GENERIC_USER_LAST_NAME_ATTRIBUTE = "last_name" GENERIC_USER_ROLE_ATTRIBUTE = "given_role" GENERIC_USER_PROVIDER_ATTRIBUTE = "provider" +GENERIC_USER_EXTRA_ATTRIBUTES = "department,employee_id,manager" # comma-separated list of additional fields to extract from SSO response GENERIC_CLIENT_STATE = "some-state" # if the provider needs a state parameter GENERIC_INCLUDE_CLIENT_ID = "false" # some providers enforce that the client_id is not in the body GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not enough to retrieve basic user info like first_name and last_name located in profile scope @@ -239,6 +240,40 @@ Use `GENERIC_USER_ROLE_ATTRIBUTE` to specify which attribute in the SSO token co Nested attribute paths are supported (e.g., `claims.role` or `attributes.litellm_role`). +**Capturing Additional SSO Fields** + +Use `GENERIC_USER_EXTRA_ATTRIBUTES` to extract additional fields from the SSO provider response beyond the standard user attributes (id, email, name, etc.). This is useful when you need to access custom organization-specific data (e.g., department, employee ID, groups) in your [custom SSO handler](./custom_sso.md). + +```shell +# Comma-separated list of field names to extract +GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,manager,groups" +``` + +**Accessing Extra Fields in Custom SSO Handler:** + +```python +from litellm.proxy.management_endpoints.types import CustomOpenID + +async def custom_sso_handler(userIDPInfo: CustomOpenID): + # Access the extra fields + extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {} + + user_department = extra_fields.get("department") + employee_id = extra_fields.get("employee_id") + user_groups = extra_fields.get("groups", []) + + # Use these fields for custom logic (e.g., team assignment, access control) + # ... +``` + +**Nested Field Paths:** + +Dot notation is supported for nested fields: + +```shell +GENERIC_USER_EXTRA_ATTRIBUTES="org_info.department,org_info.cost_center,metadata.employee_type" +``` + - Set Redirect URI, if your provider requires it - Set a redirect url = `/sso/callback` ```shell diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 5cdae51f448..c78a5c9243e 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -640,6 +640,7 @@ router_settings: | GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers | GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth | GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth +| GENERIC_USER_EXTRA_ATTRIBUTES | Comma-separated list of additional fields to extract from generic SSO provider response (e.g., "department,employee_id,groups"). Accessible via `CustomOpenID.extra_fields` in custom SSO handlers. Supports dot notation for nested fields | GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth | GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth | GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth diff --git a/docs/my-website/docs/proxy/custom_sso.md b/docs/my-website/docs/proxy/custom_sso.md index bbd7f41bee1..8b7adeb0c5a 100644 --- a/docs/my-website/docs/proxy/custom_sso.md +++ b/docs/my-website/docs/proxy/custom_sso.md @@ -142,6 +142,18 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues: f"No ID found for user. userIDPInfo.id is None {userIDPInfo}" ) + ################################################# + # Access extra fields from SSO provider (requires GENERIC_USER_EXTRA_ATTRIBUTES env var) + # Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,groups" + extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {} + user_department = extra_fields.get("department") + employee_id = extra_fields.get("employee_id") + user_groups = extra_fields.get("groups", []) + + print(f"User department: {user_department}") # noqa + print(f"Employee ID: {employee_id}") # noqa + print(f"User groups: {user_groups}") # noqa + ################################################# ################################################# # Run your custom code / logic here diff --git a/litellm/proxy/custom_sso.py b/litellm/proxy/custom_sso.py index 210e9eea3dc..b2b028dfbe3 100644 --- a/litellm/proxy/custom_sso.py +++ b/litellm/proxy/custom_sso.py @@ -24,9 +24,12 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues: print(f"userIDPInfo: {userIDPInfo}") # noqa if userIDPInfo.id is None: - raise ValueError( - f"No ID found for user. userIDPInfo.id is None {userIDPInfo}" - ) + raise ValueError(f"No ID found for user. userIDPInfo.id is None {userIDPInfo}") + + # Access extra fields from the IDP response (requires GENERIC_USER_EXTRA_ATTRIBUTES env var) + # Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="group,NTID,domain" to capture these fields + # extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {} + # user_groups = extra_fields.get("group", []) # check if user exists in litellm proxy DB _user_info = await user_info(user_id=userIDPInfo.id) diff --git a/litellm/proxy/management_endpoints/types.py b/litellm/proxy/management_endpoints/types.py index a35fc4a5f3f..295c2ad50b3 100644 --- a/litellm/proxy/management_endpoints/types.py +++ b/litellm/proxy/management_endpoints/types.py @@ -4,7 +4,7 @@ Types for the management endpoints Might include fastapi/proxy requirements.txt related imports """ -from typing import List, Optional, cast +from typing import Any, Dict, List, Optional, cast from fastapi_sso.sso.base import OpenID @@ -56,3 +56,4 @@ def get_litellm_user_role(role_str) -> Optional[LitellmUserRoles]: class CustomOpenID(OpenID): team_ids: List[str] user_role: Optional[LitellmUserRoles] = None + extra_fields: Optional[Dict[str, Any]] = None diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 278f3bdaafd..823f6400226 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -401,6 +401,8 @@ def generic_response_convertor( generic_user_role_attribute_name = os.getenv("GENERIC_USER_ROLE_ATTRIBUTE", "role") + generic_user_extra_attributes = os.getenv("GENERIC_USER_EXTRA_ATTRIBUTES", None) + verbose_proxy_logger.debug( f" generic_user_id_attribute_name: {generic_user_id_attribute_name}\n generic_user_email_attribute_name: {generic_user_email_attribute_name}" ) @@ -473,6 +475,14 @@ def generic_response_convertor( f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" ) + # Build extra_fields dict from GENERIC_USER_EXTRA_ATTRIBUTES if specified + extra_fields = None + if generic_user_extra_attributes: + extra_fields = {} + for attr_name in generic_user_extra_attributes.split(","): + attr_name = attr_name.strip() + extra_fields[attr_name] = get_nested_value(response, attr_name) + return CustomOpenID( id=get_nested_value(response, generic_user_id_attribute_name), display_name=get_nested_value( @@ -484,6 +494,7 @@ def generic_response_convertor( provider=get_nested_value(response, generic_provider_attribute_name), team_ids=all_teams, user_role=user_role, + extra_fields=extra_fields, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 16f80826798..5791f9d36cd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -4071,3 +4071,123 @@ def test_process_sso_jwt_access_token_with_role_mappings(): # Should get highest privilege role assert result.user_role == LitellmUserRoles.PROXY_ADMIN + +def test_generic_response_convertor_with_extra_attributes(monkeypatch): + """Test that extra attributes are extracted when GENERIC_USER_EXTRA_ATTRIBUTES is set""" + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + monkeypatch.setenv("GENERIC_CLIENT_ID", "test_client") + monkeypatch.setenv("GENERIC_USER_EXTRA_ATTRIBUTES", "custom_field1,custom_field2,custom_field3") + + mock_response = { + "sub": "user-id-123", + "email": "user@example.com", + "given_name": "John", + "family_name": "Doe", + "name": "John Doe", + "provider": "generic", + "custom_field1": "value1", + "custom_field2": ["item1", "item2"], + "custom_field3": {"nested": "data"}, + } + + mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler.get_team_ids_from_jwt.return_value = [] + + result = generic_response_convertor( + response=mock_response, + jwt_handler=mock_jwt_handler, + sso_jwt_handler=None, + role_mappings=None, + ) + + assert result.extra_fields is not None + assert result.extra_fields["custom_field1"] == "value1" + assert result.extra_fields["custom_field2"] == ["item1", "item2"] + assert result.extra_fields["custom_field3"] == {"nested": "data"} + +def test_generic_response_convertor_without_extra_attributes(monkeypatch): + """Test backward compatibility - extra_fields is None when env var not set""" + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + monkeypatch.setenv("GENERIC_CLIENT_ID", "test_client") + # Don't set GENERIC_USER_EXTRA_ATTRIBUTES + + mock_response = { + "sub": "user-id-123", + "email": "user@example.com", + "given_name": "John", + "family_name": "Doe", + "name": "John Doe", + "provider": "generic", + "custom_field1": "value1", + "custom_field2": "value2", + } + + mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler.get_team_ids_from_jwt.return_value = [] + + result = generic_response_convertor( + response=mock_response, + jwt_handler=mock_jwt_handler, + sso_jwt_handler=None, + role_mappings=None, + ) + + assert result.extra_fields is None + +def test_generic_response_convertor_extra_attributes_with_nested_paths(monkeypatch): + """Test that nested paths work with dot notation""" + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + monkeypatch.setenv("GENERIC_CLIENT_ID", "test_client") + monkeypatch.setenv("GENERIC_USER_EXTRA_ATTRIBUTES", "org_info.department,org_info.manager") + + mock_response = { + "sub": "user-id-123", + "email": "user@example.com", + "org_info": { + "department": "Engineering", + "manager": "Jane Smith" + } + } + + mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler.get_team_ids_from_jwt.return_value = [] + + result = generic_response_convertor( + response=mock_response, + jwt_handler=mock_jwt_handler, + sso_jwt_handler=None, + role_mappings=None, + ) + + assert result.extra_fields is not None + assert result.extra_fields["org_info.department"] == "Engineering" + assert result.extra_fields["org_info.manager"] == "Jane Smith" + +def test_generic_response_convertor_extra_attributes_missing_field(monkeypatch): + """Test that missing fields return None""" + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + monkeypatch.setenv("GENERIC_CLIENT_ID", "test_client") + monkeypatch.setenv("GENERIC_USER_EXTRA_ATTRIBUTES", "missing_field,another_missing") + + mock_response = { + "sub": "user-id-123", + "email": "user@example.com", + } + + mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler.get_team_ids_from_jwt.return_value = [] + + result = generic_response_convertor( + response=mock_response, + jwt_handler=mock_jwt_handler, + sso_jwt_handler=None, + role_mappings=None, + ) + + assert result.extra_fields is not None + assert result.extra_fields["missing_field"] is None + assert result.extra_fields["another_missing"] is None \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx new file mode 100644 index 00000000000..014f8fb9010 --- /dev/null +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -0,0 +1,1296 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import UserSearchModal from "@/components/common_components/user_search_modal"; +import { + getGuardrailsList, + getPoliciesList, + getPolicyInfoWithGuardrails, + Member, + Organization, + organizationInfoCall, + teamInfoCall, + teamMemberAddCall, + teamMemberDeleteCall, + teamMemberUpdateCall, + teamUpdateCall, +} from "@/components/networking"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; +import { isProxyAdminRole } from "@/utils/roles"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { ArrowLeftIcon } from "@heroicons/react/outline"; +import { + Badge, + Card, + Grid, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, + Text, + TextInput, + Title, + Button as TremorButton, +} from "@tremor/react"; +import { Button, Form, Input, message, Select, Switch, Tooltip } from "antd"; +import { CheckIcon, CopyIcon } from "lucide-react"; +import React, { useEffect, useMemo, useState } from "react"; +import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; +import AgentSelector from "../agent_management/AgentSelector"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import DurationSelect from "../common_components/DurationSelect"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; +import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; +import LoggingSettingsView from "../logging_settings_view"; +import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; +import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; +import { ModelSelect } from "../ModelSelect/ModelSelect"; +import NotificationsManager from "../molecules/notifications_manager"; +import { fetchMCPAccessGroups } from "../networking"; +import ObjectPermissionsView from "../object_permissions_view"; +import NumericalInput from "../shared/numerical_input"; +import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import EditLoggingSettings from "./EditLoggingSettings"; +import MemberModal from "./EditMembership"; +import MemberPermissions from "./member_permissions"; +import TeamMembersComponent from "./team_member_view"; + +export interface TeamMembership { + user_id: string; + team_id: string; + budget_id: string; + spend: number; + litellm_budget_table: { + budget_id: string; + soft_budget: number | null; + max_budget: number | null; + max_parallel_requests: number | null; + tpm_limit: number | null; + rpm_limit: number | null; + model_max_budget: Record | null; + budget_duration: string | null; + }; +} + +export interface TeamData { + team_id: string; + team_info: { + team_alias: string; + team_id: string; + organization_id: string | null; + admins: string[]; + members: string[]; + members_with_roles: Member[]; + metadata: Record; + tpm_limit: number | null; + rpm_limit: number | null; + max_budget: number | null; + soft_budget?: number | null; + budget_duration: string | null; + models: string[]; + blocked: boolean; + spend: number; + max_parallel_requests: number | null; + budget_reset_at: string | null; + model_id: string | null; + litellm_model_table: { + model_aliases: Record; + } | null; + created_at: string; + guardrails?: string[]; + policies?: string[]; + object_permission?: { + object_permission_id: string; + mcp_servers: string[]; + mcp_access_groups?: string[]; + mcp_tool_permissions?: Record; + vector_stores: string[]; + agents?: string[]; + agent_access_groups?: string[]; + }; + team_member_budget_table: { + max_budget: number; + budget_duration: string; + tpm_limit: number | null; + rpm_limit: number | null; + } | null; + }; + keys: any[]; + team_memberships: TeamMembership[]; +} + +export interface TeamInfoProps { + teamId: string; + onUpdate: (data: any) => void; + onClose: () => void; + accessToken: string | null; + is_team_admin: boolean; + is_proxy_admin: boolean; + userModels: string[]; + editTeam: boolean; + premiumUser?: boolean; +} + +const getOrganizationModels = (organization: Organization | null, userModels: string[]) => { + let tempModelsToPick = []; + + if (organization) { + // Check if organization has "all-proxy-models" in its models array + if (organization.models.includes("all-proxy-models")) { + // Treat as all-proxy-models (use userModels) + tempModelsToPick = userModels; + } else if (organization.models.length > 0) { + // Organization has specific models + tempModelsToPick = organization.models; + } else { + // Empty array [] is treated as all-proxy-models + tempModelsToPick = userModels; + } + } else { + // No organization, show all available models + tempModelsToPick = userModels; + } + + return unfurlWildcardModelsInList(tempModelsToPick, userModels); +}; + +const TeamInfoView: React.FC = ({ + teamId, + onClose, + accessToken, + is_team_admin, + is_proxy_admin, + userModels, + editTeam, + premiumUser = false, + onUpdate, +}) => { + const [teamData, setTeamData] = useState(null); + const [loading, setLoading] = useState(true); + const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false); + const [form] = Form.useForm(); + const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); + const [selectedEditMember, setSelectedEditMember] = useState(null); + const [isEditing, setIsEditing] = useState(false); + const [mcpAccessGroups, setMcpAccessGroups] = useState([]); + const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); + const [copiedStates, setCopiedStates] = useState>({}); + const [guardrailsList, setGuardrailsList] = useState([]); + const [policiesList, setPoliciesList] = useState([]); + const [policyGuardrails, setPolicyGuardrails] = useState>({}); + const [loadingPolicies, setLoadingPolicies] = useState(false); + const [memberToDelete, setMemberToDelete] = useState(null); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); + const [isTeamSaving, setIsTeamSaving] = useState(false); + const [organization, setOrganization] = useState(null); + const { userRole } = useAuthorized(); + + const canEditTeam = is_team_admin || is_proxy_admin; + + const fetchTeamInfo = async () => { + try { + setLoading(true); + if (!accessToken) return; + const response = await teamInfoCall(accessToken, teamId); + setTeamData(response); + } catch (error) { + NotificationsManager.fromBackend("Failed to load team information"); + console.error("Error fetching team info:", error); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchTeamInfo(); + }, [teamId, accessToken]); + + // Fetch organization data when team has organization_id + useEffect(() => { + const fetchOrganization = async () => { + if (!accessToken || !teamData?.team_info?.organization_id) { + setOrganization(null); + return; + } + + try { + const orgData = await organizationInfoCall(accessToken, teamData.team_info.organization_id); + setOrganization(orgData); + } catch (error) { + console.error("Error fetching organization info:", error); + setOrganization(null); + } + }; + + fetchOrganization(); + }, [accessToken, teamData?.team_info?.organization_id]); + + // Compute modelsToPick based on organization and userModels + const modelsToPick = useMemo(() => { + return getOrganizationModels(organization, userModels); + }, [organization, userModels]); + + const fetchMcpAccessGroups = async () => { + if (!accessToken) return; + if (mcpAccessGroupsLoaded) return; + try { + const groups = await fetchMCPAccessGroups(accessToken); + setMcpAccessGroups(groups); + setMcpAccessGroupsLoaded(true); + } catch (error) { + console.error("Failed to fetch MCP access groups:", error); + } + }; + + useEffect(() => { + const fetchGuardrails = async () => { + try { + if (!accessToken) return; + const response = await getGuardrailsList(accessToken); + const guardrailNames = response.guardrails.map((g: { guardrail_name: string }) => g.guardrail_name); + setGuardrailsList(guardrailNames); + } catch (error) { + console.error("Failed to fetch guardrails:", error); + } + }; + + const fetchPolicies = async () => { + try { + if (!accessToken) return; + const response = await getPoliciesList(accessToken); + const policyNames = response.policies.map((p: { policy_name: string }) => p.policy_name); + setPoliciesList(policyNames); + } catch (error) { + console.error("Failed to fetch policies:", error); + } + }; + + fetchGuardrails(); + fetchPolicies(); + }, [accessToken]); + + // Fetch resolved guardrails for all policies + useEffect(() => { + const fetchPolicyGuardrails = async () => { + if (!accessToken || !teamData?.team_info?.policies || teamData.team_info.policies.length === 0) { + return; + } + + setLoadingPolicies(true); + const guardrailsMap: Record = {}; + + try { + await Promise.all( + teamData.team_info.policies.map(async (policyName: string) => { + try { + const policyInfo = await getPolicyInfoWithGuardrails(accessToken, policyName); + guardrailsMap[policyName] = policyInfo.resolved_guardrails || []; + } catch (error) { + console.error(`Failed to fetch guardrails for policy ${policyName}:`, error); + guardrailsMap[policyName] = []; + } + }) + ); + setPolicyGuardrails(guardrailsMap); + } catch (error) { + console.error("Failed to fetch policy guardrails:", error); + } finally { + setLoadingPolicies(false); + } + }; + + fetchPolicyGuardrails(); + }, [accessToken, teamData?.team_info?.policies]); + + const handleMemberCreate = async (values: any) => { + try { + if (accessToken == null) return; + + const member: Member = { + user_email: values.user_email, + user_id: values.user_id, + role: values.role, + }; + + await teamMemberAddCall(accessToken, teamId, member); + + NotificationsManager.success("Team member added successfully"); + setIsAddMemberModalVisible(false); + form.resetFields(); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error: any) { + let errMsg = "Failed to add team member"; + + if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + + NotificationsManager.fromBackend(errMsg); + console.error("Error adding team member:", error); + } + }; + + const handleMemberUpdate = async (values: any) => { + try { + if (accessToken == null) { + return; + } + + const member: Member = { + user_email: values.user_email, + user_id: values.user_id, + role: values.role, + max_budget_in_team: values.max_budget_in_team, + tpm_limit: values.tpm_limit, + rpm_limit: values.rpm_limit, + }; + console.log("Updating member with values:", member); + message.destroy(); // Remove all existing toasts + + await teamMemberUpdateCall(accessToken, teamId, member); + + NotificationsManager.success("Team member updated successfully"); + setIsEditMemberModalVisible(false); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error: any) { + let errMsg = "Failed to update team member"; + if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + setIsEditMemberModalVisible(false); + + message.destroy(); // Remove all existing toasts + + NotificationsManager.fromBackend(errMsg); + console.error("Error updating team member:", error); + } + }; + + const handleMemberDelete = (member: Member) => { + setMemberToDelete(member); + setIsDeleteModalOpen(true); + }; + + const handleDeleteConfirm = async () => { + if (!memberToDelete || !accessToken) return; + + setIsDeleting(true); + try { + await teamMemberDeleteCall(accessToken, teamId, memberToDelete); + + NotificationsManager.success("Team member removed successfully"); + + // Fetch updated team info + const updatedTeamData = await teamInfoCall(accessToken, teamId); + setTeamData(updatedTeamData); + + // Notify parent component of the update + onUpdate(updatedTeamData); + } catch (error) { + NotificationsManager.fromBackend("Failed to remove team member"); + console.error("Error removing team member:", error); + } finally { + setIsDeleting(false); + setIsDeleteModalOpen(false); + setMemberToDelete(null); + } + }; + + const handleDeleteCancel = () => { + setIsDeleteModalOpen(false); + setMemberToDelete(null); + }; + + const handleTeamUpdate = async (values: any) => { + try { + if (!accessToken) return; + setIsTeamSaving(true); + + let parsedMetadata = {}; + try { + const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; + // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately + const { soft_budget_alerting_emails, ...rest } = rawMetadata; + parsedMetadata = rest; + } catch (e) { + NotificationsManager.fromBackend("Invalid JSON in metadata field"); + return; + } + + let secretManagerSettings: Record | undefined; + if (typeof values.secret_manager_settings === "string") { + const trimmedSecretConfig = values.secret_manager_settings.trim(); + if (trimmedSecretConfig.length > 0) { + try { + secretManagerSettings = JSON.parse(values.secret_manager_settings); + } catch (e) { + NotificationsManager.fromBackend("Invalid JSON in secret manager settings"); + return; + } + } + } + + const sanitizeNumeric = (v: any) => { + if (v === null || v === undefined) return null; + if (typeof v === "string" && v.trim() === "") return null; + if (typeof v === "number" && Number.isNaN(v)) return null; + return v; + }; + + const updateData: any = { + team_id: teamId, + team_alias: values.team_alias, + models: values.models, + tpm_limit: sanitizeNumeric(values.tpm_limit), + rpm_limit: sanitizeNumeric(values.rpm_limit), + max_budget: values.max_budget, + soft_budget: sanitizeNumeric(values.soft_budget), + budget_duration: values.budget_duration, + metadata: { + ...parsedMetadata, + ...(values.guardrails?.length > 0 ? { guardrails: values.guardrails } : {}), + ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), + disable_global_guardrails: values.disable_global_guardrails || false, + soft_budget_alerting_emails: + typeof values.soft_budget_alerting_emails === "string" + ? values.soft_budget_alerting_emails + .split(",") + .map((email: string) => email.trim()) + .filter((email: string) => email.length > 0) + : values.soft_budget_alerting_emails || [], + ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), + }, + ...(values.policies?.length > 0 ? { policies: values.policies } : {}), + organization_id: values.organization_id, + }; + + updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); + updateData.team_member_budget_duration = values.team_member_budget_duration; + + if (values.team_member_budget !== undefined) { + updateData.team_member_budget = Number(values.team_member_budget); + } + + if (values.team_member_key_duration !== undefined) { + updateData.team_member_key_duration = values.team_member_key_duration; + } + + if (values.team_member_tpm_limit !== undefined || values.team_member_rpm_limit !== undefined) { + updateData.team_member_tpm_limit = sanitizeNumeric(values.team_member_tpm_limit); + updateData.team_member_rpm_limit = sanitizeNumeric(values.team_member_rpm_limit); + } + + // Handle object_permission updates + const { servers, accessGroups } = values.mcp_servers_and_groups || { + servers: [], + accessGroups: [], + }; + const serverIds = new Set(servers || []); + const mcpToolPermissions = Object.fromEntries( + Object.entries(values.mcp_tool_permissions || {}).filter(([serverId]) => serverIds.has(serverId)), + ); + + updateData.object_permission = {}; + if (servers) { + updateData.object_permission.mcp_servers = servers; + } + if (accessGroups) { + updateData.object_permission.mcp_access_groups = accessGroups; + } + if (mcpToolPermissions) { + updateData.object_permission.mcp_tool_permissions = mcpToolPermissions; + } + delete values.mcp_servers_and_groups; + delete values.mcp_tool_permissions; + + // Handle agent permissions + const { agents, accessGroups: agentAccessGroups } = values.agents_and_groups || { + agents: [], + accessGroups: [], + }; + if (agents && agents.length > 0) { + updateData.object_permission.agents = agents; + } + if (agentAccessGroups && agentAccessGroups.length > 0) { + updateData.object_permission.agent_access_groups = agentAccessGroups; + } + delete values.agents_and_groups; + + // Handle vector stores permissions + if (values.vector_stores && values.vector_stores.length > 0) { + updateData.object_permission.vector_stores = values.vector_stores; + } + + const response = await teamUpdateCall(accessToken, updateData); + + NotificationsManager.success("Team settings updated successfully"); + setIsEditing(false); + fetchTeamInfo(); + } catch (error) { + console.error("Error updating team:", error); + } finally { + setIsTeamSaving(false); + } + }; + + if (loading) { + return
Loading...
; + } + + if (!teamData?.team_info) { + return
Team not found
; + } + + const { team_info: info } = teamData; + + const copyToClipboard = async (text: string, key: string) => { + const success = await utilCopyToClipboard(text); + if (success) { + setCopiedStates((prev) => ({ ...prev, [key]: true })); + setTimeout(() => { + setCopiedStates((prev) => ({ ...prev, [key]: false })); + }, 2000); + } + }; + + return ( +
+
+
+ + Back to Teams + + {info.team_alias} +
+ {info.team_id} +
+
+
+ + + + {[ + Overview, + ...(canEditTeam + ? [ + Members, + Member Permissions, + Settings, + ] + : []), + ]} + + + + {/* Overview Panel */} + + + + Budget Status +
+ ${formatNumberWithCommas(info.spend, 4)} + + of {info.max_budget === null ? "Unlimited" : `$${formatNumberWithCommas(info.max_budget, 4)}`} + + {info.budget_duration && Reset: {info.budget_duration}} +
+ {info.team_member_budget_table && ( + + Team Member Budget: ${formatNumberWithCommas(info.team_member_budget_table.max_budget, 4)} + + )} +
+ + + + Rate Limits +
+ TPM: {info.tpm_limit || "Unlimited"} + RPM: {info.rpm_limit || "Unlimited"} + {info.max_parallel_requests && Max Parallel Requests: {info.max_parallel_requests}} +
+ + + + Models +
+ {info.models.length === 0 ? ( + All proxy models + ) : ( + info.models.map((model, index) => ( + + {model} + + )) + )} +
+ + + + Virtual Keys +
+ User Keys: {teamData.keys.filter((key) => key.user_id).length} + Service Account Keys: {teamData.keys.filter((key) => !key.user_id).length} + Total: {teamData.keys.length} +
+ + + + + + Guardrails + {info.guardrails && info.guardrails.length > 0 ? ( +
+ {info.guardrails.map((guardrail: string, index: number) => ( + + {guardrail} + + ))} +
+ ) : ( + No guardrails configured + )} + {info.metadata?.disable_global_guardrails && ( +
+ Global Guardrails Disabled +
+ )} + + + + Policies + {info.policies && info.policies.length > 0 ? ( +
+ {info.policies.map((policy: string, index: number) => ( +
+
+ {policy} + {loadingPolicies && Loading guardrails...} +
+ {!loadingPolicies && policyGuardrails[policy] && policyGuardrails[policy].length > 0 && ( +
+ Resolved Guardrails: +
+ {policyGuardrails[policy].map((guardrail: string, gIndex: number) => ( + + {guardrail} + + ))} +
+
+ )} +
+ ))} +
+ ) : ( + No policies configured + )} + + + + + + + {/* Members Panel */} + + + + + {/* Member Permissions Panel */} + {canEditTeam && ( + + + + )} + + {/* Settings Panel */} + + +
+ Team Settings + {canEditTeam && !isEditing && ( + setIsEditing(true)}>Edit Settings + )} +
+ + {isEditing ? ( +
rest)(info.metadata), + null, + 2, + ) + : "", + logging_settings: info.metadata?.logging || [], + secret_manager_settings: info.metadata?.secret_manager_settings + ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) + : "", + organization_id: info.organization_id, + vector_stores: info.object_permission?.vector_stores || [], + mcp_servers: info.object_permission?.mcp_servers || [], + mcp_access_groups: info.object_permission?.mcp_access_groups || [], + mcp_servers_and_groups: { + servers: info.object_permission?.mcp_servers || [], + accessGroups: info.object_permission?.mcp_access_groups || [], + }, + mcp_tool_permissions: info.object_permission?.mcp_tool_permissions || {}, + agents_and_groups: { + agents: info.object_permission?.agents || [], + accessGroups: info.object_permission?.agent_access_groups || [], + }, + }} + layout="vertical" + > + + + + + + form.setFieldValue("models", values)} + teamID={teamId} + organizationID={teamData?.team_info?.organization_id || undefined} + options={{ + includeSpecialOptions: true, + includeUserModels: !teamData?.team_info?.organization_id, + showAllProxyModelsOverride: isProxyAdminRole(userRole) && !teamData?.team_info?.organization_id, + }} + context="team" + dataTestId="models-select" + /> + + + + + + + + + + + + + + + + + + + + form.setFieldValue("team_member_budget_duration", value)} + value={form.getFieldValue("team_member_budget_duration")} + /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + Guardrails{" "} + + e.stopPropagation()} + > + + + + + } + name="guardrails" + help="Select existing guardrails or enter new ones" + > + ({ value: name, label: name }))} + /> + + + + form.setFieldValue("vector_stores", values)} + value={form.getFieldValue("vector_stores")} + accessToken={accessToken || ""} + placeholder="Select vector stores" + /> + + + + form.setFieldValue("allowed_passthrough_routes", values)} + value={form.getFieldValue("allowed_passthrough_routes")} + accessToken={accessToken || ""} + placeholder="Select pass through routes" + /> + + + + form.setFieldValue("mcp_servers_and_groups", val)} + value={form.getFieldValue("mcp_servers_and_groups")} + accessToken={accessToken || ""} + placeholder="Select MCP servers or access groups (optional)" + /> + + + {/* Hidden field to register mcp_tool_permissions with the form */} + + + + prevValues.mcp_servers_and_groups !== currentValues.mcp_servers_and_groups || + prevValues.mcp_tool_permissions !== currentValues.mcp_tool_permissions + } + > + {() => ( +
+ form.setFieldsValue({ mcp_tool_permissions: toolPerms })} + /> +
+ )} + + + + form.setFieldValue("agents_and_groups", val)} + value={form.getFieldValue("agents_and_groups")} + accessToken={accessToken || ""} + placeholder="Select agents or access groups (optional)" + /> + + + + + + + + form.setFieldValue("logging_settings", values)} + /> + + + { + if (!value) { + return Promise.resolve(); + } + try { + JSON.parse(value); + return Promise.resolve(); + } catch (error) { + return Promise.reject(new Error("Please enter valid JSON")); + } + }, + }, + ]} + > + + + + + + + +
+
+ setIsEditing(false)} disabled={isTeamSaving}> + Cancel + + + Save Changes + +
+
+
+ ) : ( +
+
+ Team Name +
{info.team_alias}
+
+
+ Team ID +
{info.team_id}
+
+
+ Created At +
{new Date(info.created_at).toLocaleString()}
+
+
+ Models +
+ {info.models.map((model, index) => ( + + {model} + + ))} +
+
+
+ Rate Limits +
TPM: {info.tpm_limit || "Unlimited"}
+
RPM: {info.rpm_limit || "Unlimited"}
+
+
+ Team Budget +
+ Max Budget:{" "} + {info.max_budget !== null ? `$${formatNumberWithCommas(info.max_budget, 4)}` : "No Limit"} +
+
+ Soft Budget:{" "} + {info.soft_budget !== null && info.soft_budget !== undefined + ? `$${formatNumberWithCommas(info.soft_budget, 4)}` + : "No Limit"} +
+
Budget Reset: {info.budget_duration || "Never"}
+ {info.metadata?.soft_budget_alerting_emails && + Array.isArray(info.metadata.soft_budget_alerting_emails) && + info.metadata.soft_budget_alerting_emails.length > 0 && ( +
+ Soft Budget Alerting Emails: {info.metadata.soft_budget_alerting_emails.join(", ")} +
+ )} +
+
+ + Team Member Settings{" "} + + + + +
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
+
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
+
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
+
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
+
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
+
+
+ Organization ID +
{info.organization_id}
+
+
+ Status + {info.blocked ? "Blocked" : "Active"} +
+ +
+ Disable Global Guardrails +
+ {info.metadata?.disable_global_guardrails === true ? ( + Enabled - Global guardrails bypassed + ) : ( + Disabled - Global guardrails active + )} +
+
+ + + + + + {info.metadata?.secret_manager_settings && ( +
+ Secret Manager Settings + 
+                        {JSON.stringify(info.metadata.secret_manager_settings, null, 2)}
+
+
+ )} +
+ )} + + + + + + setIsEditMemberModalVisible(false)} + onSubmit={handleMemberUpdate} + initialData={selectedEditMember} + mode="edit" + config={{ + title: "Edit Member", + showEmail: true, + showUserId: true, + roleOptions: [ + { label: "Admin", value: "admin" }, + { label: "User", value: "user" }, + ], + additionalFields: [ + { + name: "max_budget_in_team", + label: ( + + Team Member Budget (USD){" "} + + + + + ), + type: "numerical" as const, + step: 0.01, + min: 0, + placeholder: "Budget limit for this member within this team", + }, + { + name: "tpm_limit", + label: ( + + Team Member TPM Limit{" "} + + + + + ), + type: "numerical" as const, + step: 1, + min: 0, + placeholder: "Tokens per minute limit for this member in this team", + }, + { + name: "rpm_limit", + label: ( + + Team Member RPM Limit{" "} + + + + + ), + type: "numerical" as const, + step: 1, + min: 0, + placeholder: "Requests per minute limit for this member in this team", + }, + ], + }} + /> + + setIsAddMemberModalVisible(false)} + onSubmit={handleMemberCreate} + accessToken={accessToken} + /> + + {/* Delete Member Confirmation Modal */} + +
+ ); +}; + +export default TeamInfoView; From 2e680ca62b26e277d04b4bf3d7ffbcee78abc814 Mon Sep 17 00:00:00 2001 From: DenisStefanAndrei <93513013+DenisStefanAndrei@users.noreply.github.com> Date: Tue, 10 Feb 2026 04:53:39 +0200 Subject: [PATCH 076/117] [Critical] - Fix ollama_chat reasoning content (#20750) * Fix ollama_chat reasoning_context. For ollama_chat models, reasoning context is ignored after 2 consecutive thinking chunks. * add test --- litellm/llms/ollama/chat/transformation.py | 11 +- .../ollama/test_ollama_chat_transformation.py | 130 +++++++++++++++++- 2 files changed, 134 insertions(+), 7 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 8c98cc54050..bc5aa654aad 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -502,13 +502,12 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): reasoning_content: Optional[str] = None content: Optional[str] = None if chunk["message"].get("thinking") is not None: - if self.started_reasoning_content is False: - reasoning_content = chunk["message"].get("thinking") - self.started_reasoning_content = True - elif self.finished_reasoning_content is False: - reasoning_content = chunk["message"].get("thinking") - self.finished_reasoning_content = True + reasoning_content = chunk["message"].get("thinking") + self.started_reasoning_content = True elif chunk["message"].get("content") is not None: + if self.started_reasoning_content and not self.finished_reasoning_content: + self.finished_reasoning_content = True + message_content = chunk["message"].get("content") if "" in message_content: message_content = message_content.replace("", "") diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index af6481a6cb0..02495106a84 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -10,7 +10,8 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) -from litellm.llms.ollama.chat.transformation import OllamaChatConfig +from litellm.llms.ollama.chat.transformation import OllamaChatConfig, OllamaChatCompletionResponseIterator + from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_optional_params @@ -473,3 +474,130 @@ class TestOllamaToolCalling: # finish_reason should be "stop" (default behavior) assert result.choices[0].finish_reason == "stop" assert result.choices[0].message.tool_calls is None + + +class TestOllamaReasoningContentStreaming: + """Test that reasoning_content is properly extracted from all thinking chunks.""" + + def test_multiple_thinking_chunks_all_returned_as_reasoning_content(self): + """ + Test that more than 2 consecutive thinking chunks are all returned as reasoning_content. + + Previously, the code had a bug where finished_reasoning_content was set to True + after just 2 chunks with 'thinking', causing subsequent thinking content to be lost. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), # Not used in chunk_parser + sync_stream=True, + ) + + # Simulate 5 consecutive chunks with 'thinking' content + thinking_chunks = [ + { + "model": "deepseek-r1", + "message": {"role": "assistant", "thinking": f"Thinking chunk {i}"}, + "done": False, + } + for i in range(1, 6) + ] + + # Process all thinking chunks + reasoning_contents = [] + for chunk in thinking_chunks: + result = iterator.chunk_parser(chunk) + rc = result.choices[0].delta.reasoning_content + reasoning_contents.append(rc) + + # ALL chunks should have reasoning_content, not just the first 2 + assert len(reasoning_contents) == 5 + assert reasoning_contents[0] == "Thinking chunk 1" + assert reasoning_contents[1] == "Thinking chunk 2" + assert reasoning_contents[2] == "Thinking chunk 3" # This was previously None + assert reasoning_contents[3] == "Thinking chunk 4" # This was previously None + assert reasoning_contents[4] == "Thinking chunk 5" # This was previously None + + # Verify none of them are None + for i, rc in enumerate(reasoning_contents): + assert rc is not None, f"Chunk {i+1} reasoning_content should not be None" + + def test_thinking_to_content_transition(self): + """ + Test that transition from thinking to regular content works correctly. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + # First: thinking chunks + thinking_chunk = { + "model": "deepseek-r1", + "message": {"role": "assistant", "thinking": "Let me think about this..."}, + "done": False, + } + result1 = iterator.chunk_parser(thinking_chunk) + assert result1.choices[0].delta.reasoning_content == "Let me think about this..." + assert result1.choices[0].delta.content is None + + # Then: regular content chunk + content_chunk = { + "model": "deepseek-r1", + "message": {"role": "assistant", "content": "Here is my answer."}, + "done": False, + } + result2 = iterator.chunk_parser(content_chunk) + assert result2.choices[0].delta.content == "Here is my answer." + # reasoning_content is not set when there's no thinking in the chunk + assert getattr(result2.choices[0].delta, 'reasoning_content', None) is None + + def test_think_tags_in_content(self): + """ + Test that tags embedded in content are properly parsed. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + # Content with tag + chunk1 = { + "model": "deepseek-r1", + "message": {"role": "assistant", "content": "I need to analyze this"}, + "done": False, + } + result1 = iterator.chunk_parser(chunk1) + assert result1.choices[0].delta.reasoning_content == "I need to analyze this" + assert result1.choices[0].delta.content is None + + # Content with tag (end of thinking) + chunk2 = { + "model": "deepseek-r1", + "message": {"role": "assistant", "content": "The answer is 42."}, + "done": False, + } + result2 = iterator.chunk_parser(chunk2) + assert result2.choices[0].delta.content == "The answer is 42." + # reasoning_content is not set when it's regular content + assert getattr(result2.choices[0].delta, 'reasoning_content', None) is None + + def test_done_chunk_with_thinking(self): + """ + Test that the final chunk with done=True and thinking content works. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + # Final chunk with thinking + done_chunk = { + "model": "deepseek-r1", + "message": {"role": "assistant", "thinking": "Final thought"}, + "done": True, + "done_reason": "stop", + } + result = iterator.chunk_parser(done_chunk) + assert result.choices[0].delta.reasoning_content == "Final thought" + assert result.choices[0].finish_reason == "stop" + + From 7f895088066387f655b971cecd5f74022fb51341 Mon Sep 17 00:00:00 2001 From: Zero Clover <13190004+ZeroClover@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:57:15 +0800 Subject: [PATCH 077/117] fix(sso): handle opaque access tokens in process_sso_jwt_access_token (#20726) OIDC providers like Logto may return opaque (non-JWT) access tokens, which caused jwt.decode() to raise DecodeError and crash the SSO callback with a 500 error. Catch DecodeError and skip JWT-based extraction gracefully, since user info is already available from the UserInfo endpoint. Fixes #20724 --- litellm/proxy/management_endpoints/ui_sso.py | 12 +- .../proxy/management_endpoints/test_ui_sso.py | 193 +++++++++++------- 2 files changed, 133 insertions(+), 72 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 823f6400226..fbb6f380a54 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -190,9 +190,15 @@ def process_sso_jwt_access_token( if access_token_str and result: import jwt - access_token_payload = jwt.decode( - access_token_str, options={"verify_signature": False} - ) + try: + access_token_payload = jwt.decode( + access_token_str, options={"verify_signature": False} + ) + except jwt.exceptions.DecodeError: + verbose_proxy_logger.debug( + "Access token is not a valid JWT (possibly an opaque token), skipping JWT-based extraction" + ) + return # Extract team IDs from access token if sso_jwt_handler is available if sso_jwt_handler: diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 5791f9d36cd..74d36c0acac 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -2374,47 +2374,6 @@ class TestProcessSSOJWTAccessToken: "groups": ["team1", "team2", "team3"], } - def test_process_sso_jwt_access_token_with_valid_token( - self, mock_jwt_handler, sample_jwt_token, sample_jwt_payload - ): - """Test processing a valid JWT access token with team extraction""" - from litellm.proxy.management_endpoints.ui_sso import ( - process_sso_jwt_access_token, - ) - - # Create a result object without team_ids - result = CustomOpenID( - id="test_user", - email="test@example.com", - first_name="Test", - last_name="User", - display_name="Test User", - provider="generic", - team_ids=[], - ) - - with patch("jwt.decode", return_value=sample_jwt_payload) as mock_jwt_decode: - # Act - process_sso_jwt_access_token( - access_token_str=sample_jwt_token, - sso_jwt_handler=mock_jwt_handler, - result=result, - ) - - # Assert - # Verify JWT was decoded correctly - mock_jwt_decode.assert_called_once_with( - sample_jwt_token, options={"verify_signature": False} - ) - - # Verify team IDs were extracted from JWT - mock_jwt_handler.get_team_ids_from_jwt.assert_called_once_with( - sample_jwt_payload - ) - - # Verify team IDs were set on the result object - assert result.team_ids == ["team1", "team2", "team3"] - def test_process_sso_jwt_access_token_with_existing_team_ids( self, mock_jwt_handler, sample_jwt_token ): @@ -2549,27 +2508,6 @@ class TestProcessSSOJWTAccessToken: mock_jwt_handler.get_team_ids_from_jwt.assert_not_called() assert result.team_ids == [] - def test_process_sso_jwt_access_token_no_sso_jwt_handler(self, sample_jwt_token): - """Test that JWT is decoded for role extraction even when sso_jwt_handler is None, - but team_ids are not extracted (team extraction requires sso_jwt_handler).""" - from litellm.proxy.management_endpoints.ui_sso import ( - process_sso_jwt_access_token, - ) - - result = CustomOpenID(id="test_user", email="test@example.com", team_ids=[]) - - mock_payload = {"sub": "test_user", "email": "test@example.com"} - with patch("jwt.decode", return_value=mock_payload) as mock_jwt_decode: - # Act - process_sso_jwt_access_token( - access_token_str=sample_jwt_token, sso_jwt_handler=None, result=result - ) - - # JWT is decoded (for role extraction) but team_ids are not extracted - mock_jwt_decode.assert_called_once() - assert result.team_ids == [] - assert result.user_role is None - def test_process_sso_jwt_access_token_no_result( self, mock_jwt_handler, sample_jwt_token ): @@ -2590,10 +2528,12 @@ class TestProcessSSOJWTAccessToken: mock_jwt_decode.assert_not_called() mock_jwt_handler.get_team_ids_from_jwt.assert_not_called() - def test_process_sso_jwt_access_token_jwt_decode_exception( + def test_process_sso_jwt_access_token_non_decode_exception_propagates( self, mock_jwt_handler, sample_jwt_token ): - """Test that JWT decode exceptions are not caught (should propagate up)""" + """Test that non-DecodeError JWT exceptions still propagate up.""" + import jwt as pyjwt + from litellm.proxy.management_endpoints.ui_sso import ( process_sso_jwt_access_token, ) @@ -2601,19 +2541,16 @@ class TestProcessSSOJWTAccessToken: result = CustomOpenID(id="test_user", email="test@example.com", team_ids=[]) with patch( - "jwt.decode", side_effect=Exception("JWT decode error") + "jwt.decode", side_effect=pyjwt.exceptions.InvalidKeyError("Invalid key") ) as mock_jwt_decode: - # Act & Assert - with pytest.raises(Exception, match="JWT decode error"): + with pytest.raises(pyjwt.exceptions.InvalidKeyError, match="Invalid key"): process_sso_jwt_access_token( access_token_str=sample_jwt_token, sso_jwt_handler=mock_jwt_handler, result=result, ) - # Verify JWT decode was attempted mock_jwt_decode.assert_called_once() - # But team extraction should not have been called mock_jwt_handler.get_team_ids_from_jwt.assert_not_called() def test_process_sso_jwt_access_token_empty_team_ids_from_jwt( @@ -2646,6 +2583,124 @@ class TestProcessSSOJWTAccessToken: # Even empty team IDs should be set assert result.team_ids == [] + def test_process_sso_jwt_access_token_with_opaque_token(self, mock_jwt_handler): + """Test that opaque (non-JWT) access tokens are handled gracefully without raising.""" + from litellm.proxy.management_endpoints.ui_sso import ( + process_sso_jwt_access_token, + ) + + result = CustomOpenID( + id="test_user", + email="test@example.com", + first_name="Test", + last_name="User", + display_name="Test User", + provider="generic", + team_ids=["existing_team"], + user_role=None, + ) + + # Opaque tokens like those from Logto are short random strings, not JWTs + opaque_token = "uTxyjXbS_random_opaque_token_string" + + # Should NOT raise - opaque tokens should be silently skipped + process_sso_jwt_access_token( + access_token_str=opaque_token, + sso_jwt_handler=mock_jwt_handler, + result=result, + ) + + # Result should be untouched + mock_jwt_handler.get_team_ids_from_jwt.assert_not_called() + assert result.team_ids == ["existing_team"] + assert result.user_role is None + + def test_process_sso_jwt_access_token_real_jwt_with_role_and_teams( + self, mock_jwt_handler + ): + """Test that a real JWT containing role and team fields is correctly processed.""" + import jwt as pyjwt + + from litellm.proxy.management_endpoints.ui_sso import ( + process_sso_jwt_access_token, + ) + + payload = { + "sub": "user123", + "email": "admin@example.com", + "role": "proxy_admin", + "groups": ["team_alpha", "team_beta"], + } + real_jwt_token = pyjwt.encode(payload, "test-secret", algorithm="HS256") + + mock_jwt_handler.get_team_ids_from_jwt.return_value = [ + "team_alpha", + "team_beta", + ] + + result = CustomOpenID( + id="user123", + email="admin@example.com", + first_name="Admin", + last_name="User", + display_name="Admin User", + provider="generic", + team_ids=[], + user_role=None, + ) + + process_sso_jwt_access_token( + access_token_str=real_jwt_token, + sso_jwt_handler=mock_jwt_handler, + result=result, + ) + + # Team IDs should be extracted via sso_jwt_handler + mock_jwt_handler.get_team_ids_from_jwt.assert_called_once_with(payload) + assert result.team_ids == ["team_alpha", "team_beta"] + + # Role should be extracted from the "role" field in the JWT + from litellm.proxy._types import LitellmUserRoles + + assert result.user_role == LitellmUserRoles.PROXY_ADMIN + + def test_process_sso_jwt_access_token_real_jwt_without_role_and_teams(self): + """Test that a real JWT without role/team fields leaves result unchanged.""" + import jwt as pyjwt + + from litellm.proxy.management_endpoints.ui_sso import ( + process_sso_jwt_access_token, + ) + + payload = { + "sub": "user456", + "email": "plain@example.com", + "iat": 1700000000, + } + real_jwt_token = pyjwt.encode(payload, "test-secret", algorithm="HS256") + + result = CustomOpenID( + id="user456", + email="plain@example.com", + first_name="Plain", + last_name="User", + display_name="Plain User", + provider="generic", + team_ids=[], + user_role=None, + ) + + # No sso_jwt_handler, no role/team fields in JWT + process_sso_jwt_access_token( + access_token_str=real_jwt_token, + sso_jwt_handler=None, + result=result, + ) + + # Nothing should be modified + assert result.team_ids == [] + assert result.user_role is None + @pytest.mark.asyncio async def test_get_ui_settings_includes_api_doc_base_url(): From 59c81a30a0e1678acd74cdcddcd55ecad68ff241 Mon Sep 17 00:00:00 2001 From: Rohith sai <81635089+whitehatjr1001@users.noreply.github.com> Date: Tue, 10 Feb 2026 08:30:05 +0530 Subject: [PATCH 078/117] fix(anthropic): route thinking requests through OpenAI responses (#20755) * fix(anthropic): route thinking requests through OpenAI responses * Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: Krish Dholakia Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../adapters/handler.py | 55 +++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 9 ++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index a17eba75b3b..a3e06d7ab38 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -30,6 +30,56 @@ ANTHROPIC_ADAPTER = AnthropicAdapter() class LiteLLMMessagesToCompletionTransformationHandler: + @staticmethod + def _route_openai_thinking_to_responses_api_if_needed( + completion_kwargs: Dict[str, Any], + *, + thinking: Optional[Dict[str, Any]], + ) -> None: + """ + When users call `litellm.anthropic.messages.*` with a non-Anthropic model and + `thinking={"type": "enabled", ...}`, LiteLLM converts this into OpenAI + `reasoning_effort`. + + For OpenAI models, Chat Completions typically does not return reasoning text + (only token accounting). To return a thinking-like content block in the + Anthropic response format, we route the request through OpenAI's Responses API + and request a reasoning summary. + """ + custom_llm_provider = completion_kwargs.get("custom_llm_provider") + if custom_llm_provider is None: + try: + _, inferred_provider, _, _ = litellm.utils.get_llm_provider( + model=cast(str, completion_kwargs.get("model")) + ) + custom_llm_provider = inferred_provider + except Exception: + custom_llm_provider = None + + if custom_llm_provider != "openai": + return + + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return + + model = completion_kwargs.get("model") + if isinstance(model, str) and model and not model.startswith("responses/"): + + reasoning_effort = completion_kwargs.get("reasoning_effort") + if isinstance(reasoning_effort, str) and reasoning_effort: + completion_kwargs["reasoning_effort"] = { + "effort": reasoning_effort, + "summary": "detailed", + } + elif isinstance(reasoning_effort, dict): + if ( + "summary" not in reasoning_effort + and "generate_summary" not in reasoning_effort + ): + updated_reasoning_effort = dict(reasoning_effort) + updated_reasoning_effort["summary"] = "detailed" + completion_kwargs["reasoning_effort"] = updated_reasoning_effort + @staticmethod def _prepare_completion_kwargs( *, @@ -123,6 +173,11 @@ class LiteLLMMessagesToCompletionTransformationHandler: ): completion_kwargs[key] = value + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, + thinking=thinking, + ) + return completion_kwargs, tool_name_mapping @staticmethod diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 66d62aae1ec..80fd3ab698a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -185,7 +185,14 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): # Verify reasoning_effort is set (converted from thinking) assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" - assert call_kwargs["reasoning_effort"] == "minimal", f"reasoning_effort should be 'minimal' for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + assert call_kwargs["reasoning_effort"] == { + "effort": "minimal", + "summary": "detailed", + }, f"reasoning_effort should request a reasoning summary for OpenAI responses API, got {call_kwargs.get('reasoning_effort')}" + + # Verify OpenAI thinking requests are routed to the Responses API + assert call_kwargs.get("model") == "responses/gpt-5.2" + # Verify thinking is NOT passed (non-Claude model) assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" From 96206ec14164193beb22e481779f412faeb3241a Mon Sep 17 00:00:00 2001 From: moophlo Date: Tue, 10 Feb 2026 07:01:37 +0400 Subject: [PATCH 079/117] mcp: support http(s) URLs for spec_path in OpenAPI MCP loader (#20753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(responses): preserve streamed tool deltas when id is omitted * fix(responses): guard ambiguous tool-call index reuse * add missing indexes on VerificationToken table * mcp: support http(s) URLs for spec_path in OpenAPI MCP loader * test(mcp): add unit test for OpenAPI spec_path URL support * Fix OpenAPI spec URL loading to use shared MCP httpx client Ensure URL-based OpenAPI loading honors LiteLLM’s custom httpx configuration, add missing imports, and harden tests to prevent regressions or accidental direct httpx usage. * removed unused import urlparse * removed unsupported timeout argument --------- Co-authored-by: Emerson Gomes Co-authored-by: Sameer Kankute Co-authored-by: Carlo Alberto Ferraris Co-authored-by: Andrea Odorisio --- .../mcp_server/openapi_to_mcp_generator.py | 34 ++++++- tests/mcp_tests/test_openapi_spec_path_url.py | 92 +++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/mcp_tests/test_openapi_spec_path_url.py diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index b635f15ed09..deb0b4f9549 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -3,6 +3,8 @@ This module is used to generate MCP tools from OpenAPI specs. """ import json +import asyncio +import os from pathlib import PurePosixPath from typing import Any, Dict, Optional from urllib.parse import quote @@ -45,8 +47,36 @@ def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: def load_openapi_spec(filepath: str) -> Dict[str, Any]: - """Load OpenAPI specification from JSON file.""" - with open(filepath, "r") as f: + """ + Sync wrapper. For URL specs, use the shared/custom MCP httpx client. + """ + try: + # If we're already inside an event loop, prefer the async function. + asyncio.get_running_loop() + raise RuntimeError( + "load_openapi_spec() was called from within a running event loop. " + "Use 'await load_openapi_spec_async(...)' instead." + ) + except RuntimeError as e: + # "no running event loop" is fine; other RuntimeErrors we re-raise + if "no running event loop" not in str(e).lower(): + raise + return asyncio.run(load_openapi_spec_async(filepath)) + +async def load_openapi_spec_async(filepath: str) -> Dict[str, Any]: + if filepath.startswith("http://") or filepath.startswith("https://"): + client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) + # NOTE: do not close shared client if get_async_httpx_client returns a shared singleton. + # If it returns a new client each time, consider wrapping it in an async context manager. + r = await client.get(filepath) + r.raise_for_status() + return r.json() + + # fallback: local file + # Local filesystem path + if not os.path.exists(filepath): + raise FileNotFoundError(f"OpenAPI spec not found at {filepath}") + with open(filepath, "r", encoding="utf-8") as f: return json.load(f) diff --git a/tests/mcp_tests/test_openapi_spec_path_url.py b/tests/mcp_tests/test_openapi_spec_path_url.py new file mode 100644 index 00000000000..03e9db94967 --- /dev/null +++ b/tests/mcp_tests/test_openapi_spec_path_url.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from typing import Any, Dict + +import httpx +import pytest + +from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator as gen + + +class _FakeAsyncHTTPHandler: + """ + Minimal stand-in for the object returned by get_async_httpx_client(). + openapi_to_mcp_generator.load_openapi_spec_async() calls: + + client = get_async_httpx_client(...) + r = await client.get(url, timeout=30.0) + + So we must implement async get(). + """ + + def __init__(self, response: httpx.Response, expected_url: str): + self._response = response + self._expected_url = expected_url + self.calls = 0 + + async def get(self, request_url: str, timeout: float = 30.0): + self.calls += 1 + assert request_url == self._expected_url + assert timeout == 30.0 + return self._response + + +def test_load_openapi_spec_supports_http_url(monkeypatch: pytest.MonkeyPatch) -> None: + url = "http://example.local/openapi.json" + expected: Dict[str, Any] = { + "openapi": "3.0.0", + "info": {"title": "Test API", "version": "1.0.0"}, + "paths": {}, + } + + # httpx.Response must include a Request for raise_for_status() to work. + req = httpx.Request("GET", url) + resp = httpx.Response(status_code=200, json=expected, request=req) + + calls = {"get_async_httpx_client": 0} + handler_holder: Dict[str, Any] = {} + + def fake_get_async_httpx_client(*args, **kwargs): + calls["get_async_httpx_client"] += 1 + h = _FakeAsyncHTTPHandler(resp, expected_url=url) + handler_holder["handler"] = h + return h + + # Ensure shared/custom client path is used + monkeypatch.setattr(gen, "get_async_httpx_client", fake_get_async_httpx_client) + + # Fail loudly if someone reintroduces direct httpx.get() + def boom(*args, **kwargs): + raise AssertionError("Direct httpx.get() must not be used for URL spec loading") + + monkeypatch.setattr(httpx, "get", boom) + + spec = gen.load_openapi_spec(url) + + assert spec == expected + assert calls["get_async_httpx_client"] == 1 + assert handler_holder["handler"].calls == 1 + + +def test_load_openapi_spec_supports_local_file_path(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + expected: Dict[str, Any] = { + "openapi": "3.0.0", + "info": {"title": "Local API", "version": "1.0.0"}, + "paths": {}, + } + + p = tmp_path / "openapi.json" + p.write_text( + '{"openapi":"3.0.0","info":{"title":"Local API","version":"1.0.0"},"paths":{}}', + encoding="utf-8", + ) + + # For local files, shared client must NOT be used. + def boom_client(*args, **kwargs): + raise AssertionError("get_async_httpx_client() must not be called for local file paths") + + monkeypatch.setattr(gen, "get_async_httpx_client", boom_client) + + spec = gen.load_openapi_spec(str(p)) + assert spec == expected + From 4256c547c2e3a2ad34282ca886912692a5e0121b Mon Sep 17 00:00:00 2001 From: Neel Harsola <81899840+skylarkoo7@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:34:40 +0530 Subject: [PATCH 080/117] fix(bedrock): handle concatenated JSON in tool call arguments (#20742) * fix(responses): preserve streamed tool deltas when id is omitted * fix(responses): guard ambiguous tool-call index reuse * add missing indexes on VerificationToken table * fix(bedrock): handle concatenated JSON in tool call arguments When using Bedrock Claude Sonnet 4.5 with tools enabled, the model sometimes returns multiple tool call arguments as concatenated JSON objects in a single arguments string, e.g. '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' json.loads() fails on this with "Extra data", crashing the entire request in _convert_to_bedrock_tool_call_invoke. This commit: - Adds split_concatenated_json_objects() helper in common_utils.py that uses json.JSONDecoder.raw_decode() to walk a string and extract each JSON object individually. - Updates _convert_to_bedrock_tool_call_invoke() to catch JSONDecodeError and attempt splitting concatenated objects into separate Bedrock toolUse blocks (first block keeps original ID, subsequent blocks get suffixed IDs). - Fixes duplicate json.loads calls and a shadowed 'id' builtin. - Adds 12 unit tests covering normal, empty, concatenated, and edge cases. Fixes #20543 --------- Co-authored-by: Emerson Gomes Co-authored-by: Sameer Kankute Co-authored-by: Carlo Alberto Ferraris --- .../prompt_templates/common_utils.py | 56 +++++++ .../prompt_templates/factory.py | 59 ++++++- ...ore_utils_prompt_templates_common_utils.py | 56 +++++++ ...llm_core_utils_prompt_templates_factory.py | 151 ++++++++++++++++++ 4 files changed, 314 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b1c2d0a52f5..cdddee4e54e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1272,3 +1272,59 @@ def parse_tool_call_arguments( ) raise ValueError(error_message) from e + + +def split_concatenated_json_objects(raw: str) -> List[Dict[str, Any]]: + """ + Split a string that contains one or more concatenated JSON objects into + a list of parsed dicts. + + LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return + multiple tool-call argument objects concatenated in a single + ``arguments`` string, e.g.:: + + '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' + + ``json.loads()`` fails on this with ``JSONDecodeError: Extra data``. + This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string + and extract each JSON object individually. + + Returns + ------- + list[dict] + A list of parsed dicts – one per JSON object found. If *raw* is + empty or whitespace-only, an empty list is returned. + + Raises + ------ + json.JSONDecodeError + If the string contains text that cannot be parsed as JSON at all. + """ + import json + + raw = raw.strip() + if not raw: + return [] + + decoder = json.JSONDecoder() + results: List[Dict[str, Any]] = [] + idx = 0 + length = len(raw) + + while idx < length: + # Skip whitespace between objects + while idx < length and raw[idx] in " \t\n\r": + idx += 1 + if idx >= length: + break + + obj, end_idx = decoder.raw_decode(raw, idx) + if isinstance(obj, dict): + results.append(obj) + else: + # Non-dict JSON value – wrap in empty dict (Bedrock requires + # toolUse.input to be an object). + results.append({}) + idx = end_idx + + return results diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index f9ecd78ff1c..c907ed32b95 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3287,25 +3287,68 @@ def _convert_to_bedrock_tool_call_invoke( - extract name - extract id """ + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + split_concatenated_json_objects, + ) try: _parts_list: List[BedrockContentBlock] = [] for tool in tool_calls: if "function" in tool: - id = tool["id"] + tool_id = tool["id"] name = tool["function"].get("name", "") arguments = tool["function"].get("arguments", "") - arguments_dict = json.loads(arguments) if arguments else {} - # Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object) - # When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns "" - if not isinstance(arguments_dict, dict): - arguments_dict = {} + if not arguments or not arguments.strip(): arguments_dict = {} else: - arguments_dict = json.loads(arguments) + try: + arguments_dict = json.loads(arguments) + # Ensure arguments_dict is always a dict + # (Bedrock requires toolUse.input to be an object). + # Some providers return arguments: '""' which + # json.loads decodes to a bare string. + if not isinstance(arguments_dict, dict): + arguments_dict = {} + except json.JSONDecodeError: + # The model may return multiple JSON objects + # concatenated in a single arguments string, e.g. + # '{"cmd":"a"}{"cmd":"b"}{"cmd":"c"}' + # Split them and emit one toolUse block per object. + # Fixes: https://github.com/BerriAI/litellm/issues/20543 + parsed_objects = split_concatenated_json_objects( + arguments + ) + if parsed_objects: + # First object keeps the original tool id. + for obj_idx, obj in enumerate(parsed_objects): + block_id = ( + tool_id + if obj_idx == 0 + else f"{tool_id}_{obj_idx}" + ) + bedrock_tool = BedrockToolUseBlock( + input=obj, name=name, toolUseId=block_id + ) + _parts_list.append( + BedrockContentBlock(toolUse=bedrock_tool) + ) + # cache_control applies to the whole original + # tool call; attach after the last split block. + if tool.get("cache_control", None) is not None: + _parts_list.append( + BedrockContentBlock( + cachePoint=CachePointBlock( + type="default" + ) + ) + ) + continue + # Fallback: no objects extracted — use empty dict. + arguments_dict = {} + bedrock_tool = BedrockToolUseBlock( - input=arguments_dict, name=name, toolUseId=id + input=arguments_dict, name=name, toolUseId=tool_id ) bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool) _parts_list.append(bedrock_content_block) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 980693aa73a..f566f91841d 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -12,6 +12,7 @@ sys.path.insert( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, + split_concatenated_json_objects, update_messages_with_model_file_ids, ) @@ -143,3 +144,58 @@ def test_convert_prefix_message_to_non_prefix_messages(): }, {"role": "assistant", "content": "value"}, ] + + +# ── split_concatenated_json_objects tests ── + + +def test_split_concatenated_json_single_object(): + """A single valid JSON object is returned as a one-element list.""" + result = split_concatenated_json_objects('{"location": "Boston"}') + assert result == [{"location": "Boston"}] + + +def test_split_concatenated_json_multiple_objects(): + """ + Multiple JSON objects concatenated without separators are split correctly. + This is the exact pattern from issue #20543 where Bedrock Claude Sonnet 4.5 + returns concatenated JSON in a single tool call arguments string. + """ + raw = ( + '{"command": ["curl", "-i", "http://localhost:9009"]}' + '{"command": ["curl", "-i", "http://localhost:9009/robots.txt"]}' + '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]}' + ) + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"command": ["curl", "-i", "http://localhost:9009"]} + assert result[1] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt"]} + assert result[2] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]} + + +def test_split_concatenated_json_with_whitespace(): + """Objects separated by whitespace are handled correctly.""" + raw = '{"a": 1} {"b": 2}\n{"c": 3}' + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"a": 1} + assert result[1] == {"b": 2} + assert result[2] == {"c": 3} + + +def test_split_concatenated_json_empty_string(): + """Empty or whitespace-only strings return an empty list.""" + assert split_concatenated_json_objects("") == [] + assert split_concatenated_json_objects(" ") == [] + + +def test_split_concatenated_json_non_dict_value(): + """Non-dict JSON values (e.g. arrays, strings) are replaced with {}.""" + result = split_concatenated_json_objects('[1, 2, 3]') + assert result == [{}] + + +def test_split_concatenated_json_invalid_raises(): + """Completely invalid JSON raises JSONDecodeError.""" + with pytest.raises(json.JSONDecodeError): + split_concatenated_json_objects("not json at all") diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index e87233a52a3..707b5bdc777 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( BAD_MESSAGE_ERROR_STR, BedrockConverseMessagesProcessor, BedrockImageProcessor, + _convert_to_bedrock_tool_call_invoke, ollama_pt, ) @@ -1590,3 +1591,153 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs(): # Verify $defs have been removed (Bedrock doesn't support them) tool_schema = result[0]["toolSpec"].get("inputSchema", {}).get("json", {}) assert "$defs" not in tool_schema, "$defs should be removed after expansion" + + +# ── _convert_to_bedrock_tool_call_invoke tests ── + + +def test_bedrock_tool_call_invoke_normal_single_tool(): + """Normal single tool call with valid JSON arguments.""" + tool_calls = [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Boston, MA"}', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["toolUseId"] == "call_abc123" + assert result[0]["toolUse"]["name"] == "get_weather" + assert result[0]["toolUse"]["input"] == {"location": "Boston, MA"} + + +def test_bedrock_tool_call_invoke_empty_arguments(): + """Tool call with empty arguments produces an empty dict input.""" + tool_calls = [ + { + "id": "call_empty", + "type": "function", + "function": {"name": "do_something", "arguments": ""}, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_concatenated_json(): + """ + Tool call whose arguments contain multiple concatenated JSON objects + (the bug from issue #20543) is split into separate Bedrock toolUse blocks. + + Bedrock Claude Sonnet 4.5 sometimes returns multiple tool call arguments + concatenated in a single string like: + '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' + """ + tool_calls = [ + { + "id": "tooluse_L7I3TewYAUhoheJZQEuwVN", + "type": "function", + "function": { + "name": "shell", + "arguments": ( + '{"command": ["curl", "-i", "http://localhost:9009", "-m", "10"]}' + '{"command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"]}' + '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"]}' + ), + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + + # Should produce 3 separate toolUse blocks + assert len(result) == 3 + + # First block keeps original tool id + assert result[0]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN" + assert result[0]["toolUse"]["name"] == "shell" + assert result[0]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009", "-m", "10"] + } + + # Subsequent blocks get suffixed ids + assert result[1]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_1" + assert result[1]["toolUse"]["name"] == "shell" + assert result[1]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"] + } + + assert result[2]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_2" + assert result[2]["toolUse"]["name"] == "shell" + assert result[2]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"] + } + + +def test_bedrock_tool_call_invoke_concatenated_json_with_cache_control(): + """ + When a tool call has cache_control AND concatenated JSON arguments, + the cachePoint block is appended after the last split block. + """ + tool_calls = [ + { + "id": "call_cached", + "type": "function", + "cache_control": {"type": "default"}, + "function": { + "name": "shell", + "arguments": '{"a": 1}{"b": 2}', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + + # 2 toolUse blocks + 1 cachePoint block + assert len(result) == 3 + assert "toolUse" in result[0] + assert "toolUse" in result[1] + assert "cachePoint" in result[2] + + +def test_bedrock_tool_call_invoke_non_dict_arguments(): + """Arguments that parse to a non-dict (e.g. '""') produce empty dict input.""" + tool_calls = [ + { + "id": "call_non_dict", + "type": "function", + "function": {"name": "tool", "arguments": '""'}, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_multiple_normal_tools(): + """Multiple separate tool calls (normal parallel calling) work correctly.""" + tool_calls = [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "LA"}', + }, + }, + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 2 + assert result[0]["toolUse"]["toolUseId"] == "call_1" + assert result[1]["toolUse"]["toolUseId"] == "call_2" From 28e15f4fd679065bd5bf1275ea7dd80b078fa67d Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 9 Feb 2026 22:08:20 -0600 Subject: [PATCH 081/117] fix(responses): preserve cached tool call objects in tool result recovery (#20700) * fix(responses): preserve cached tool call objects in tool result recovery * fix(responses): support attr-based cached tool call recovery * Update litellm/responses/litellm_completion_transformation/transformation.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../transformation.py | 116 ++++++++++++++++-- .../test_litellm_completion_responses.py | 97 ++++++++++++++- 2 files changed, 203 insertions(+), 10 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index df298f7c448..f9835ff24f2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -602,18 +602,53 @@ class LiteLLMCompletionResponsesConfig: } return None + @staticmethod + def _get_mapping_or_attr_value(obj: Any, key: str, default: Any = None) -> Any: + """ + Safely read a field from dict-like or attribute-based objects. + """ + if obj is None: + return default + + if isinstance(obj, dict): + return obj.get(key, default) + + getter = getattr(obj, "get", None) + if callable(getter): + try: + return getter(key, default) + except (TypeError, AttributeError): + pass + + return getattr(obj, key, default) + @staticmethod def _create_tool_call_chunk( tool_use_definition: Dict[str, Any], tool_call_id: str, index: int ) -> ChatCompletionToolCallChunk: """Create a ChatCompletionToolCallChunk from tool_use_definition.""" - function_raw = tool_use_definition.get("function") - function: Dict[str, Any] = function_raw if isinstance(function_raw, dict) else {} - tool_use_id_raw = tool_use_definition.get("id") + function_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "function" + ) + function_name_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + function_raw, "name" + ) + function_arguments_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + function_raw, "arguments" + ) + function: Dict[str, Any] = { + "name": function_name_raw or "", + "arguments": function_arguments_raw or "{}", + } + tool_use_id_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "id" + ) tool_use_id: str = ( str(tool_use_id_raw) if tool_use_id_raw is not None else str(tool_call_id) ) - tool_use_type_raw = tool_use_definition.get("type") + tool_use_type_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "type" + ) tool_use_type: str = ( str(tool_use_type_raw) if tool_use_type_raw is not None else "function" ) @@ -627,6 +662,63 @@ class LiteLLMCompletionResponsesConfig: index=index, ) + @staticmethod + def _normalize_tool_use_definition( + tool_use_definition: Any, tool_call_id: str + ) -> Optional[Dict[str, Any]]: + """ + Normalize cached tool_call definitions to a dict-like shape consumed by _create_tool_call_chunk. + """ + if not tool_use_definition: + return None + + if isinstance(tool_use_definition, dict): + normalized_definition: Dict[str, Any] = dict(tool_use_definition) + else: + tool_use_id_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "id" + ) + tool_use_type_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "type" + ) + function_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + tool_use_definition, "function" + ) + + # Object does not expose the expected tool_call fields. + if ( + tool_use_id_raw is None + and tool_use_type_raw is None + and function_raw is None + ): + return None + + normalized_definition = { + "id": tool_use_id_raw, + "type": tool_use_type_raw, + "function": function_raw, + } + + function_raw = normalized_definition.get("function") + if function_raw is not None and not isinstance(function_raw, dict): + function_name_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + function_raw, "name" + ) + function_arguments_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value( + function_raw, "arguments" + ) + if function_name_raw is not None or function_arguments_raw is not None: + normalized_definition["function"] = { + "name": function_name_raw, + "arguments": function_arguments_raw, + } + + normalized_definition["id"] = normalized_definition.get("id") or tool_call_id + normalized_definition["type"] = ( + normalized_definition.get("type") or "function" + ) + return normalized_definition + @staticmethod def _add_tool_call_to_assistant( assistant_message: Any, tool_call_chunk: ChatCompletionToolCallChunk @@ -740,13 +832,19 @@ class LiteLLMCompletionResponsesConfig: tool_call_id, tools ) ) - - if _tool_use_definition: - if not isinstance(_tool_use_definition, dict): - _tool_use_definition = {} + + normalized_tool_use_definition = ( + LiteLLMCompletionResponsesConfig._normalize_tool_use_definition( + _tool_use_definition, tool_call_id + ) + ) + + if normalized_tool_use_definition: tool_call_chunk = ( LiteLLMCompletionResponsesConfig._create_tool_call_chunk( - _tool_use_definition, tool_call_id, len(tool_calls) + normalized_tool_use_definition, + tool_call_id, + len(tool_calls), ) ) LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 5074bbf4397..a96ebe2b792 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -7,6 +7,7 @@ sys.path.insert( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, + TOOL_CALLS_CACHE, ) from litellm.types.llms.openai import ( ChatCompletionResponseMessage, @@ -17,6 +18,8 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, Message, ModelResponse, + Function, + ChatCompletionMessageToolCall, PromptTokensDetailsWrapper, Usage, ) @@ -755,6 +758,98 @@ class TestFunctionCallTransformation: tool_call = tool_calls[0] assert tool_call.get("id") == "fallback_id" + def test_ensure_tool_results_preserves_cached_openai_object_tool_call(self): + """ + Test cached ChatCompletionMessageToolCall objects are normalized correctly. + """ + tool_call_id = "call_cached_openai_object" + TOOL_CALLS_CACHE.set_cache( + key=tool_call_id, + value=ChatCompletionMessageToolCall( + id=tool_call_id, + type="function", + function=Function( + name="search_web", + arguments='{"query": "python bugs"}', + ), + ), + ) + + messages_missing_tool_calls = [ + {"role": "user", "content": "Search for python bugs"}, + {"role": "assistant", "content": None, "tool_calls": []}, + {"role": "tool", "content": "Found 5 results", "tool_call_id": tool_call_id}, + ] + + try: + fixed_messages = LiteLLMCompletionResponsesConfig._ensure_tool_results_have_corresponding_tool_calls( + messages=messages_missing_tool_calls, + tools=None, + ) + finally: + TOOL_CALLS_CACHE.delete_cache(key=tool_call_id) + + assistant_msg = fixed_messages[1] + tool_calls = assistant_msg.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + function = tool_call.get("function", {}) + assert function.get("name") == "search_web" + assert function.get("arguments") == '{"query": "python bugs"}' + + def test_ensure_tool_results_preserves_cached_attr_object_tool_call(self): + """ + Test cached attribute-only tool call objects are normalized correctly. + """ + + class AttrOnlyFunction: + def __init__(self, name: str, arguments: str): + self.name = name + self.arguments = arguments + + class AttrOnlyToolCall: + def __init__(self, id: str, type: str, function: AttrOnlyFunction): + self.id = id + self.type = type + self.function = function + + tool_call_id = "call_cached_attr_object" + TOOL_CALLS_CACHE.set_cache( + key=tool_call_id, + value=AttrOnlyToolCall( + id=tool_call_id, + type="function", + function=AttrOnlyFunction( + name="search_web", + arguments='{"query": "attribute objects"}', + ), + ), + ) + + messages_missing_tool_calls = [ + {"role": "user", "content": "Search using attr object"}, + {"role": "assistant", "content": None, "tool_calls": []}, + {"role": "tool", "content": "Found 3 results", "tool_call_id": tool_call_id}, + ] + + try: + fixed_messages = LiteLLMCompletionResponsesConfig._ensure_tool_results_have_corresponding_tool_calls( + messages=messages_missing_tool_calls, + tools=None, + ) + finally: + TOOL_CALLS_CACHE.delete_cache(key=tool_call_id) + + assistant_msg = fixed_messages[1] + tool_calls = assistant_msg.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + function = tool_call.get("function", {}) + assert function.get("name") == "search_web" + assert function.get("arguments") == '{"query": "attribute objects"}' + class TestToolChoiceTransformation: """Test the tool_choice transformation fix for Cursor IDE bug""" @@ -1637,4 +1732,4 @@ class TestStreamingIDConsistency: # Verify it matches the cached ID assert iterator._cached_item_id is not None - assert iterator._cached_item_id == text_done_id \ No newline at end of file + assert iterator._cached_item_id == text_done_id From fc9cf1c870b644a7a9f59ba0ea29aaf83770d7fb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 16:03:18 +0530 Subject: [PATCH 082/117] Fix indentation error --- .../anthropic/experimental_pass_through/adapters/handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index a3e06d7ab38..296ae97aead 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -64,8 +64,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: model = completion_kwargs.get("model") if isinstance(model, str) and model and not model.startswith("responses/"): - - reasoning_effort = completion_kwargs.get("reasoning_effort") + reasoning_effort = completion_kwargs.get("reasoning_effort") if isinstance(reasoning_effort, str) and reasoning_effort: completion_kwargs["reasoning_effort"] = { "effort": reasoning_effort, From 08824f9912a3c199e4f7426e8567e6d10378ba69 Mon Sep 17 00:00:00 2001 From: Lei Nie Date: Wed, 4 Feb 2026 23:38:50 -0800 Subject: [PATCH 083/117] [Bug] Fix missing image_tokens in Responses API output_tokens_details (#20404) When transforming chat completion responses to Responses API format, image_tokens from completion_tokens_details was not being included in output_tokens_details. This affected Vertex AI/Gemini models that return image token counts in candidatesTokensDetails with modality="IMAGE". The fix adds image_tokens handling alongside existing reasoning_tokens and text_tokens transformation. --- .../transformation.py | 6 +++ .../test_litellm_completion_responses.py | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index df298f7c448..c3add946014 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1794,6 +1794,12 @@ class LiteLLMCompletionResponsesConfig: ): output_details_dict["text_tokens"] = completion_details.text_tokens + if ( + hasattr(completion_details, "image_tokens") + and completion_details.image_tokens is not None + ): + output_details_dict["image_tokens"] = completion_details.image_tokens + if output_details_dict: response_usage.output_tokens_details = OutputTokensDetails( **output_details_dict diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 5074bbf4397..b183875d050 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1424,6 +1424,47 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is None assert response_usage.output_tokens_details is None + def test_transform_usage_with_image_tokens(self): + """Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details""" + # Setup: Simulate Vertex AI/Gemini usage with image_tokens in completion_tokens_details + usage = Usage( + prompt_tokens=10, + completion_tokens=150, + total_tokens=160, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=0, + text_tokens=50, + image_tokens=100, # From Vertex AI candidatesTokensDetails with modality="IMAGE" + ), + ) + + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="gemini-2.0-flash", + object="chat.completion", + usage=usage, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="Here is the generated image.", role="assistant"), + ) + ], + ) + + # Execute + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=chat_completion_response + ) + + # Assert + assert response_usage.output_tokens == 150 + assert response_usage.output_tokens_details is not None + assert response_usage.output_tokens_details.reasoning_tokens == 0 + assert response_usage.output_tokens_details.text_tokens == 50 + assert response_usage.output_tokens_details.image_tokens == 100 + class TestStreamingIDConsistency: """Test cases for consistent IDs across streaming events (issue #14962)""" From eb39582547f6c7ba519048e943746e49a5bae15c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 5 Feb 2026 16:37:03 +0900 Subject: [PATCH 084/117] fix(proxy): allow safe MCP server discovery for virtual keys (#20421) --- litellm/proxy/_types.py | 86 ++-- .../mcp_management_endpoints.py | 123 ++++- .../proxy/auth/test_route_checks.py | 108 +++-- .../test_mcp_management_endpoints.py | 442 ++++++++++++------ 4 files changed, 506 insertions(+), 253 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ecefe772247..fedfa011fdd 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -420,6 +420,8 @@ class LiteLLMRoutes(enum.Enum): "/mcp/tools", "/mcp/tools/list", "/mcp/tools/call", + # Read-only MCP discovery endpoint (virtual keys may be allowed here) + "/v1/mcp/server", ] agent_routes = [ @@ -845,9 +847,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase): allowed_cache_controls: Optional[list] = [] config: Optional[dict] = {} permissions: Optional[dict] = {} - model_max_budget: Optional[ - dict - ] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} + model_max_budget: Optional[dict] = ( + {} + ) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} model_config = ConfigDict(protected_namespaces=()) model_rpm_limit: Optional[dict] = None @@ -1396,12 +1398,12 @@ class NewCustomerRequest(BudgetNewRequest): blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget spend: Optional[float] = None - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) @model_validator(mode="before") @classmethod @@ -1423,12 +1425,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase): blocked: bool = False # allow/disallow requests for this end-user max_budget: Optional[float] = None budget_id: Optional[str] = None # give either a budget_id or max_budget - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) class DeleteCustomerRequest(LiteLLMPydanticObjectBase): @@ -1516,15 +1518,15 @@ class NewTeamRequest(TeamBase): ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm model_tpm_limit: Optional[Dict[str, int]] = None - team_member_budget: Optional[ - float - ] = None # allow user to set a budget for all team members - team_member_rpm_limit: Optional[ - int - ] = None # allow user to set RPM limit for all team members - team_member_tpm_limit: Optional[ - int - ] = None # allow user to set TPM limit for all team members + team_member_budget: Optional[float] = ( + None # allow user to set a budget for all team members + ) + team_member_rpm_limit: Optional[int] = ( + None # allow user to set RPM limit for all team members + ) + team_member_tpm_limit: Optional[int] = ( + None # allow user to set TPM limit for all team members + ) team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m" allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None @@ -1615,9 +1617,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase): class AddTeamCallback(LiteLLMPydanticObjectBase): callback_name: str - callback_type: Optional[ - Literal["success", "failure", "success_and_failure"] - ] = "success_and_failure" + callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = ( + "success_and_failure" + ) callback_vars: Dict[str, str] @model_validator(mode="before") @@ -1945,9 +1947,9 @@ class ConfigList(LiteLLMPydanticObjectBase): stored_in_db: Optional[bool] field_default_value: Any premium_field: bool = False - nested_fields: Optional[ - List[FieldDetail] - ] = None # For nested dictionary or Pydantic fields + nested_fields: Optional[List[FieldDetail]] = ( + None # For nested dictionary or Pydantic fields + ) class UserHeaderMapping(LiteLLMPydanticObjectBase): @@ -2386,9 +2388,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): budget_id: Optional[str] = None created_at: datetime updated_at: datetime - user: Optional[ - Any - ] = None # You might want to replace 'Any' with a more specific type if available + user: Optional[Any] = ( + None # You might want to replace 'Any' with a more specific type if available + ) litellm_budget_table: Optional[LiteLLM_BudgetTable] = None model_config = ConfigDict(protected_namespaces=()) @@ -3364,9 +3366,9 @@ class TeamModelDeleteRequest(BaseModel): # Organization Member Requests class OrganizationMemberAddRequest(OrgMemberAddRequest): organization_id: str - max_budget_in_organization: Optional[ - float - ] = None # Users max budget within the organization + max_budget_in_organization: Optional[float] = ( + None # Users max budget within the organization + ) class OrganizationMemberDeleteRequest(MemberDeleteRequest): @@ -3584,9 +3586,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase): Maps provider names to their budget configs. """ - providers: Dict[ - str, ProviderBudgetResponseObject - ] = {} # Dictionary mapping provider names to their budget configurations + providers: Dict[str, ProviderBudgetResponseObject] = ( + {} + ) # Dictionary mapping provider names to their budget configurations class ProxyStateVariables(TypedDict): @@ -3729,9 +3731,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): enforce_rbac: bool = False roles_jwt_field: Optional[str] = None # v2 on role mappings role_mappings: Optional[List[RoleMapping]] = None - object_id_jwt_field: Optional[ - str - ] = None # can be either user / team, inferred from the role mapping + object_id_jwt_field: Optional[str] = ( + None # can be either user / team, inferred from the role mapping + ) scope_mappings: Optional[List[ScopeMapping]] = None enforce_scope_based_access: bool = False enforce_team_based_model_access: bool = False diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 90c2f7fdf62..597521ae773 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -262,6 +262,67 @@ if MCP_AVAILABLE: ) -> List[LiteLLM_MCPServerTable]: return [_redact_mcp_credentials(server) for server in mcp_servers] + def _is_restricted_virtual_key_request(user_api_key_dict: UserAPIKeyAuth) -> bool: + """Best-effort detection for route-restricted virtual keys. + + We treat a requestor as a "restricted" virtual key if `allowed_routes` + is a non-empty list. This matches the auth gate that blocks routes with + the error: "Virtual key is not allowed to call this route...". + """ + + allowed_routes = getattr(user_api_key_dict, "allowed_routes", None) + return isinstance(allowed_routes, list) and len(allowed_routes) > 0 + + def _sanitize_mcp_server_for_virtual_key( + mcp_server: LiteLLM_MCPServerTable, + ) -> LiteLLM_MCPServerTable: + """Return a minimally sufficient MCP server view for virtual keys. + + Security model: + - Virtual keys should be able to *discover* accessible servers. + - They should NOT receive sensitive configuration details like upstream + URLs, env vars, headers, commands/args, access-group names, or + credentials. + """ + + sanitized = _redact_mcp_credentials(mcp_server) + + # Remove potentially sensitive config + identity fields. + sanitized.url = None + sanitized.static_headers = None + sanitized.env = {} + sanitized.command = None + sanitized.args = [] + sanitized.extra_headers = [] + sanitized.allowed_tools = [] + sanitized.mcp_access_groups = [] + sanitized.teams = [] + + sanitized.authorization_url = None + sanitized.token_url = None + sanitized.registration_url = None + + sanitized.health_check_error = None + sanitized.last_health_check = None + + sanitized.created_by = None + sanitized.updated_by = None + sanitized.created_at = None + sanitized.updated_at = None + + # `mcp_info` is arbitrary metadata; keep only an explicit safe subset. + is_public = False + if isinstance(sanitized.mcp_info, dict): + is_public = bool(sanitized.mcp_info.get("is_public")) + sanitized.mcp_info = {"is_public": True} if is_public else None + + return sanitized + + def _sanitize_mcp_server_list_for_virtual_key( + mcp_servers: Iterable[LiteLLM_MCPServerTable], + ) -> List[LiteLLM_MCPServerTable]: + return [_sanitize_mcp_server_for_virtual_key(server) for server in mcp_servers] + def _inherit_credentials_from_existing_server( payload: NewMCPServerRequest, ) -> NewMCPServerRequest: @@ -504,8 +565,11 @@ if MCP_AVAILABLE: """ user_mcp_management_mode = _get_user_mcp_management_mode() + is_restricted_virtual_key = _is_restricted_virtual_key_request( + user_api_key_dict + ) - if user_mcp_management_mode == "view_all": + if user_mcp_management_mode == "view_all" and not is_restricted_virtual_key: servers = await global_mcp_server_manager.get_all_mcp_servers_unfiltered() redacted_mcp_servers = _redact_mcp_credentials_list(servers) else: @@ -531,6 +595,11 @@ if MCP_AVAILABLE: if server.mcp_info is None: server.mcp_info = {} server.mcp_info["is_public"] = True + + # Virtual keys only get a sanitized discovery view. + if is_restricted_virtual_key: + return _sanitize_mcp_server_list_for_virtual_key(redacted_mcp_servers) + return redacted_mcp_servers @router.get( @@ -625,6 +694,34 @@ if MCP_AVAILABLE: detail={"error": f"MCP Server with id {server_id} not found"}, ) + # Implement authz restriction from requested user + is_admin_view = _user_has_admin_view(user_api_key_dict) + is_restricted_virtual_key = _is_restricted_virtual_key_request( + user_api_key_dict + ) + + if not is_admin_view: + # Perform authz check BEFORE any health check (avoid side-effects for + # unauthorized callers). + mcp_server_records = await get_all_mcp_servers_for_user( + prisma_client, user_api_key_dict + ) + exists = does_mcp_server_exist(mcp_server_records, server_id) + + if not exists: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + f"User does not have permission to view mcp server with id {server_id}. " + "You can only view mcp servers that you have access to." + ) + }, + ) + + # At this point caller is authorized to view the server. + await global_mcp_server_manager.add_server(mcp_server) + # Perform health check on the server using server manager try: health_result = await global_mcp_server_manager.health_check_server( @@ -644,26 +741,10 @@ if MCP_AVAILABLE: mcp_server.last_health_check = datetime.now() mcp_server.health_check_error = str(e) - # Implement authz restriction from requested user - if _user_has_admin_view(user_api_key_dict): - return _redact_mcp_credentials(mcp_server) - - # Perform authz check to filter the mcp servers user has access to - mcp_server_records = await get_all_mcp_servers_for_user( - prisma_client, user_api_key_dict - ) - exists = does_mcp_server_exist(mcp_server_records, server_id) - - if exists: - await global_mcp_server_manager.add_server(mcp_server) - return _redact_mcp_credentials(mcp_server) - else: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail={ - "error": f"User does not have permission to view mcp server with id {server_id}. You can only view mcp servers that you have access to." - }, - ) + redacted = _redact_mcp_credentials(mcp_server) + if is_restricted_virtual_key: + return _sanitize_mcp_server_for_virtual_key(redacted) + return redacted @router.post( "/server", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index a745ac3de13..65bb329e7ea 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -108,6 +108,22 @@ def test_virtual_key_allowed_routes_with_litellm_routes_member_name_allowed(): assert result is True +def test_virtual_key_mcp_routes_allows_v1_mcp_server(): + """Regression test for #20325: allow virtual keys to list MCP servers.""" + + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["mcp_routes"], + ) + + result = RouteChecks.is_virtual_key_allowed_to_call_route( + route="/v1/mcp/server", + valid_token=valid_token, + ) + + assert result is True + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" @@ -199,14 +215,14 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route): def test_google_routes_with_dynamic_model_names_recognized_as_llm_api_route(route): """ Test that Google routes with dynamic model names (including custom names) are recognized as LLM API routes. - + This test verifies the fix for the issue where routes like: /v1beta/models/google-gemini-2-5-pro-code-reviewer-k8s:generateContent were incorrectly classified as "custom admin only route" instead of LLM API routes. - + The fix adds pattern matching for Google routes with placeholders like {model_name}. """ - + # Test that the route is recognized as an LLM API route assert RouteChecks.is_llm_api_route(route) is True @@ -214,28 +230,28 @@ def test_google_routes_with_dynamic_model_names_recognized_as_llm_api_route(rout def test_google_routes_with_dynamic_model_names_accessible_to_internal_users(): """ Test that internal users can access Google routes with dynamic model names. - + This ensures that routes like /v1beta/models/{model_name}:generateContent are properly accessible to internal users and not blocked as admin-only routes. """ - + # Create an internal user object user_obj = LiteLLM_UserTable( user_id="test_user", user_email="test@example.com", user_role=LitellmUserRoles.INTERNAL_USER.value, ) - + # Create an internal user API key auth valid_token = UserAPIKeyAuth( user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER.value, ) - + # Create a mock request request = MagicMock(spec=Request) request.query_params = {} - + # Test that calling Google route with dynamic model name does NOT raise an exception try: RouteChecks.non_proxy_admin_allowed_routes_check( @@ -263,11 +279,13 @@ def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names(): # Test that routes from both groups are allowed result1 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/chat/completions", valid_token=valid_token # This is in openai_routes + route="/chat/completions", + valid_token=valid_token, # This is in openai_routes ) result2 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/user/info", valid_token=valid_token # This is in info_routes + route="/user/info", + valid_token=valid_token, # This is in info_routes ) assert result1 is True @@ -288,11 +306,13 @@ def test_virtual_key_allowed_routes_with_mixed_member_names_and_explicit_routes( # Test that both info routes and explicit custom route are allowed result1 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/user/info", valid_token=valid_token # This is in info_routes + route="/user/info", + valid_token=valid_token, # This is in info_routes ) result2 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/custom/route", valid_token=valid_token # This is explicitly listed + route="/custom/route", + valid_token=valid_token, # This is explicitly listed ) assert result1 is True @@ -323,7 +343,8 @@ def test_virtual_key_allowed_routes_with_no_member_names_only_explicit(): # Test that non-allowed route raises HTTPException with pytest.raises(HTTPException) as exc_info: RouteChecks.is_virtual_key_allowed_to_call_route( - route="/user/info", valid_token=valid_token # Not in allowed routes + route="/user/info", + valid_token=valid_token, # Not in allowed routes ) assert exc_info.value.status_code == 403 @@ -372,12 +393,15 @@ def test_virtual_key_llm_api_routes_allows_registered_pass_through_endpoints(): }, } - with patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", - mock_registered_routes, - ), patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path", - return_value="/", + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + mock_registered_routes, + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path", + return_value="/", + ), ): # Create a virtual key with llm_api_routes permission valid_token = UserAPIKeyAuth( @@ -421,12 +445,15 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through(): }, } - with patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", - mock_registered_routes, - ), patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path", - return_value="/", + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + mock_registered_routes, + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path", + return_value="/", + ), ): # Create a virtual key without llm_api_routes permission valid_token = UserAPIKeyAuth( @@ -442,7 +469,9 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through(): ) assert exc_info.value.status_code == 403 - assert "Virtual key is not allowed to call this route" in str(exc_info.value.detail) + assert "Virtual key is not allowed to call this route" in str( + exc_info.value.detail + ) def test_check_passthrough_route_access_key_metadata_exact_match(): @@ -833,6 +862,7 @@ def test_videos_route_with_virtual_key_llm_api_routes(): result is True ), f"Virtual key with llm_api_routes should be able to access {route}" + def test_non_proxy_admin_wildcard_allowed_routes(): """Test that nonproxy admin users can still use wildcard routes""" @@ -847,7 +877,7 @@ def test_non_proxy_admin_wildcard_allowed_routes(): user_role=LitellmUserRoles.INTERNAL_USER.value, allowed_routes=["/scim/*"], ) - + request = MagicMock(spec=Request) request.query_params = {} @@ -864,14 +894,14 @@ def test_non_proxy_admin_wildcard_allowed_routes(): def test_proxy_admin_viewer_can_access_global_spend_tags(): """ Test that proxy_admin_viewer can access /global/spend/tags endpoint. - + This test verifies the fix for the issue where proxy_admin_viewer was getting 403 errors when trying to access /global/spend/tags endpoint. - + Related: Slack thread from 10/9/2025 - Erik Kristensen reported this issue. proxy_admin_viewer role should have access to "view all spend" endpoints. """ - + # Create a proxy admin viewer user object user_obj = LiteLLM_UserTable( user_id="viewer_user", @@ -912,8 +942,12 @@ def test_route_in_additional_public_routes_wildcard_match(): """ from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes - with patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}), \ - patch("litellm.proxy.proxy_server.premium_user", True): + with ( + patch( + "litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]} + ), + patch("litellm.proxy.proxy_server.premium_user", True), + ): # Wildcard should match subpaths assert route_in_additonal_public_routes("/api/users") is True assert route_in_additonal_public_routes("/api/users/123") is True @@ -927,11 +961,15 @@ def test_route_in_additional_public_routes_exact_match(): """ from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes - with patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/health", "/status"]}), \ - patch("litellm.proxy.proxy_server.premium_user", True): + with ( + patch( + "litellm.proxy.proxy_server.general_settings", + {"public_routes": ["/health", "/status"]}, + ), + patch("litellm.proxy.proxy_server.premium_user", True), + ): # Exact matches should work assert route_in_additonal_public_routes("/health") is True assert route_in_additonal_public_routes("/status") is True # Non-matching routes should fail assert route_in_additonal_public_routes("/other") is False - diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 83914e30354..b4c8d5f07cc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1,4 +1,3 @@ -import json import os import sys import types @@ -204,25 +203,28 @@ class TestListMCPServers: transport="http", ), ] - mock_manager.get_all_allowed_mcp_servers = AsyncMock( - return_value=mock_servers - ) + mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=mock_servers) for idx, server in enumerate(mock_servers): server.credentials = {"auth_value": f"secret_{idx}"} - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", - return_value=True, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): # Import and call the function from litellm.proxy.management_endpoints.mcp_management_endpoints import ( @@ -269,12 +271,15 @@ class TestListMCPServers: return_value=mock_servers ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", - return_value="view_all", - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), ): from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_all_mcp_servers, @@ -285,6 +290,79 @@ class TestListMCPServers: assert len(result) == 2 assert {server.server_id for server in result} == {"server-1", "server-2"} + @pytest.mark.asyncio + async def test_list_mcp_servers_view_all_mode_virtual_key_is_sanitized(self): + """Issue #20325: virtual keys should get a safe discovery view.""" + + mock_user_auth = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test_user_id", + api_key="test_api_key", + allowed_routes=["mcp_routes"], + ) + + mock_servers = [ + generate_mock_mcp_server_db_record(server_id="server-1", alias="One"), + generate_mock_mcp_server_db_record(server_id="server-2", alias="Two"), + ] + for idx, server in enumerate(mock_servers): + server.credentials = {"auth_value": f"secret_{idx}"} + server.env = {"API_KEY": "super-secret"} + server.static_headers = {"Authorization": "Bearer super-secret"} + server.mcp_access_groups = ["group-a"] + server.teams = [{"team_id": "team-1", "team_alias": "Team 1"}] + server.command = "bash" + server.args = ["-lc", "echo hi"] + server.extra_headers = ["Authorization"] + + mock_manager = MagicMock() + mock_manager.get_all_mcp_servers_unfiltered = AsyncMock( + return_value=mock_servers + ) + mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=mock_servers) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth) + + # Ensure we did not bypass filtering via view_all for restricted virtual keys. + mock_manager.get_all_mcp_servers_unfiltered.assert_not_called() + + assert len(result) == 2 + assert {server.server_id for server in result} == {"server-1", "server-2"} + + for server in result: + assert server.credentials is None + assert server.url is None + assert server.static_headers is None + assert server.env == {} + assert server.command is None + assert server.args == [] + assert server.extra_headers == [] + assert server.allowed_tools == [] + assert server.mcp_access_groups == [] + assert server.teams == [] + @pytest.mark.asyncio async def test_list_mcp_servers_combined_config_and_db(self): """ @@ -374,25 +452,28 @@ class TestListMCPServers: transport="http", ), ] - mock_manager.get_all_allowed_mcp_servers = AsyncMock( - return_value=mock_servers - ) + mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=mock_servers) for idx, server in enumerate(mock_servers): server.credentials = {"auth_value": f"secret_{idx}"} - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", - return_value=True, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): # Import and call the function from litellm.proxy.management_endpoints.mcp_management_endpoints import ( @@ -494,25 +575,28 @@ class TestListMCPServers: url="https://actions.zapier.com/mcp/sse", ), ] - mock_manager.get_all_allowed_mcp_servers = AsyncMock( - return_value=mock_servers - ) + mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=mock_servers) for idx, server in enumerate(mock_servers): server.credentials = {"auth_value": f"secret_{idx}"} - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", - return_value=False, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=False, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): # Import and call the function from litellm.proxy.management_endpoints.mcp_management_endpoints import ( @@ -540,7 +624,6 @@ class TestListMCPServers: assert server.alias == "Allowed Zapier MCP" assert server.url == "https://actions.zapier.com/mcp/sse" - @pytest.mark.asyncio async def test_fetch_single_mcp_server_redacts_credentials(self): mock_server = generate_mock_mcp_server_db_record( @@ -562,18 +645,23 @@ class TestListMCPServers: user_role=LitellmUserRoles.PROXY_ADMIN ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", - AsyncMock(return_value=mock_server), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", - AsyncMock(return_value=mock_health_result), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", - return_value=True, + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=mock_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", + AsyncMock(return_value=mock_health_result), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), ): from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_mcp_server, @@ -610,18 +698,23 @@ class TestListMCPServers: user_role=LitellmUserRoles.PROXY_ADMIN ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", - AsyncMock(return_value=mock_server), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", - AsyncMock(return_value=mock_health_result), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", - return_value=True, + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=mock_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", + AsyncMock(return_value=mock_health_result), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), ): from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_mcp_server, @@ -763,16 +856,20 @@ class TestTemporaryMCPSessionEndpoints: mock_manager.get_mcp_server_by_id.return_value = inherited_server mock_manager.build_mcp_server_from_table = AsyncMock(return_value=built_server) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", - MagicMock(), - ) as validate_mock, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._cache_temporary_mcp_server", - MagicMock(), - ) as cache_mock: + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", + MagicMock(), + ) as validate_mock, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._cache_temporary_mcp_server", + MagicMock(), + ) as cache_mock, + ): response = await add_session_mcp_server( payload=payload, user_api_key_dict=user_auth, @@ -832,13 +929,16 @@ class TestTemporaryMCPSessionEndpoints: server = generate_mock_mcp_server_config_record(server_id="server-1") authorize_response = MagicMock() - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", - return_value=server, - ) as get_server, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.authorize_with_server", - AsyncMock(return_value=authorize_response), - ) as authorize_mock: + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", + return_value=server, + ) as get_server, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.authorize_with_server", + AsyncMock(return_value=authorize_response), + ) as authorize_mock, + ): result = await mcp_authorize( request=request, server_id="server-1", @@ -875,13 +975,16 @@ class TestTemporaryMCPSessionEndpoints: server = generate_mock_mcp_server_config_record(server_id="server-1") exchange_response = {"access_token": "token"} - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", - return_value=server, - ) as get_server, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.exchange_token_with_server", - AsyncMock(return_value=exchange_response), - ) as exchange_mock: + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", + return_value=server, + ) as get_server, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.exchange_token_with_server", + AsyncMock(return_value=exchange_response), + ) as exchange_mock, + ): result = await mcp_token( request=request, server_id="server-1", @@ -922,16 +1025,20 @@ class TestTemporaryMCPSessionEndpoints: "token_endpoint_auth_method": "client_secret_basic", } - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", - return_value=server, - ) as get_server, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._read_request_body", - AsyncMock(return_value=request_body), - ) as read_body, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.register_client_with_server", - AsyncMock(return_value=register_response), - ) as register_mock: + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", + return_value=server, + ) as get_server, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._read_request_body", + AsyncMock(return_value=request_body), + ) as read_body, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.register_client_with_server", + AsyncMock(return_value=register_response), + ) as register_mock, + ): result = await mcp_register(request=request, server_id="server-1") assert result is register_response @@ -947,6 +1054,7 @@ class TestTemporaryMCPSessionEndpoints: fallback_client_id="server-1", ) + class TestUpdateMCPServer: """Test suite for update MCP server functionality""" @@ -954,7 +1062,7 @@ class TestUpdateMCPServer: async def test_update_mcp_server_respects_extra_headers(self): """ Test that updating an MCP server with extra_headers properly saves the field. - + This test ensures that extra_headers field in UpdateMCPServerRequest is properly handled and persisted when updating an MCP server. """ @@ -999,21 +1107,27 @@ class TestUpdateMCPServer: ) # Mock the update_mcp_server function to capture the call - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=mock_prisma_client, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", - MagicMock(), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.update_mcp_server", - AsyncMock(return_value=updated_server), - ) as update_mock, patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.add_server", - AsyncMock(), - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.reload_servers_from_database", - AsyncMock(), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", + MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.update_mcp_server", + AsyncMock(return_value=updated_server), + ) as update_mock, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.add_server", + AsyncMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.reload_servers_from_database", + AsyncMock(), + ), ): # Import and call the function from litellm.proxy.management_endpoints.mcp_management_endpoints import ( @@ -1030,7 +1144,10 @@ class TestUpdateMCPServer: # First arg is prisma_client, second is the payload (UpdateMCPServerRequest) called_payload = call_args[0][1] assert called_payload.server_id == "test-server-1" - assert called_payload.extra_headers == ["X-Custom-Header", "X-Another-Header"] + assert called_payload.extra_headers == [ + "X-Custom-Header", + "X-Another-Header", + ] assert called_payload.alias == "Updated Test Server" # Verify the result includes extra_headers @@ -1081,12 +1198,15 @@ class TestHealthCheckServers: return_value=[mock_health_result_1, mock_health_result_2] ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): result = await health_check_servers( server_ids=None, @@ -1133,9 +1253,12 @@ class TestMCPRegistryEndpoint: # The registry endpoint uses get_filtered_registry (filters by client IP) mock_manager.get_filtered_registry.return_value = {mock_server.server_id: mock_server} - with patch_proxy_general_settings({"enable_mcp_registry": True}), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, + with ( + patch_proxy_general_settings({"enable_mcp_registry": True}), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), ): response = client.get("/v1/mcp/registry.json") @@ -1182,12 +1305,15 @@ class TestMCPRegistryEndpoint: return_value=[mock_health_result] ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): result = await health_check_servers( server_ids=["server-1"], @@ -1245,12 +1371,15 @@ class TestManagementPayloadValidation: return_value=[health_result_one, health_result_two] ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", - return_value="view_all", - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), ): result = await health_check_servers( server_ids=None, @@ -1295,12 +1424,15 @@ class TestManagementPayloadValidation: return_value=[mock_health_result] # Only server-1 is returned (accessible) ) - with patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", - mock_manager, - ), patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", - AsyncMock(return_value=[mock_user_auth]), + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), ): result = await health_check_servers( server_ids=["server-1", "server-unauthorized"], From 90cd6538d475fc7b3a8197fa19dc523ed244a5c9 Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Wed, 4 Feb 2026 23:34:40 -0800 Subject: [PATCH 085/117] added functionality to propagate bedrock guardrail errors down to litellm (#20395) --- .../guardrail_hooks/bedrock_guardrails.py | 60 ++++++++++++++++++- .../test_bedrock_apply_guardrail.py | 8 +-- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 528857f5dd1..f8fba5f5984 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -461,12 +461,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): data=prepared_request.body, # type: ignore headers=prepared_request.headers, # type: ignore ) + except HTTPException: + # Propagate HTTPException (e.g. from non-200 path) as-is + raise except Exception as e: + # If this is an HTTP error with a response body (e.g. httpx.HTTPStatusError), + # extract the AWS error message and propagate it + response = getattr(e, "response", None) + if isinstance(response, httpx.Response): + try: + status_code, detail_message = ( + self._parse_bedrock_guardrail_error_response(response) + ) + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_provider=self.guardrail_provider, + guardrail_json_response={"error": detail_message}, + request_data=request_data or {}, + guardrail_status="guardrail_failed_to_respond", + start_time=start_time.timestamp(), + end_time=datetime.now().timestamp(), + duration=(datetime.now() - start_time).total_seconds(), + event_type=event_type, + ) + raise HTTPException( + status_code=status_code, detail=detail_message + ) from e + except HTTPException: + raise # Endpoint down, timeout, or other HTTP/network errors verbose_proxy_logger.error( "Bedrock AI: failed to make guardrail request: %s", str(e) ) - # Add guardrail information with failure status self.add_standard_logging_guardrail_information_to_request_data( guardrail_provider=self.guardrail_provider, guardrail_json_response={"error": str(e)}, @@ -477,7 +502,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): duration=(datetime.now() - start_time).total_seconds(), event_type=event_type, ) - # Re-raise the exception to maintain existing behavior raise ######################################################### @@ -509,11 +533,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): bedrock_guardrail_response ) else: + status_code, detail_message = self._parse_bedrock_guardrail_error_response( + httpx_response + ) verbose_proxy_logger.error( "Bedrock AI: error in response. Status code: %s, response: %s", httpx_response.status_code, httpx_response.text, ) + raise HTTPException(status_code=status_code, detail=detail_message) return bedrock_guardrail_response @@ -579,6 +607,34 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return "success" return "guardrail_failed_to_respond" + def _parse_bedrock_guardrail_error_response( + self, response: httpx.Response + ) -> Tuple[int, str]: + """ + Parse AWS Bedrock guardrail error response body to extract status code and message. + + AWS may return shapes like {"message": "..."} or {"error": {"message": "..."}}. + Returns (status_code, message) for use in HTTPException. + """ + status_code = response.status_code + message = "Bedrock guardrail request failed" + try: + body = response.json() + except Exception: + text = getattr(response, "text", None) or "" + if isinstance(text, str) and text.strip(): + return (status_code, text.strip()) + return (status_code, message) + if isinstance(body, dict): + if isinstance(body.get("message"), str): + return (status_code, body["message"]) + err = body.get("error") + if isinstance(err, dict) and isinstance(err.get("message"), str): + return (status_code, err["message"]) + if isinstance(err, str): + return (status_code, err) + return (status_code, message) + def _get_http_exception_for_blocked_guardrail( self, response: BedrockGuardrailResponse ) -> Union[HTTPException, GuardrailInterventionNormalStringError]: diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index 4bea2255b56..dff444168c2 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -78,7 +78,7 @@ async def test_bedrock_apply_guardrail_blocked(): }, ) - # HTTPException must propagate as-is (not wrapped in a generic Exception) + # Test the apply_guardrail method propagates HTTPException (AWS error) to the client with pytest.raises(HTTPException) as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["This is blocked content"]}, @@ -276,7 +276,7 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable }, ) - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(HTTPException, match="policy") as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["blocked"]}, request_data=request_data, @@ -286,9 +286,9 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable assert mock_api.called _, kwargs = mock_api.call_args assert kwargs["messages"] == [request_messages[-1]] - # HTTPException must propagate as-is (not wrapped) + # HTTPException from guardrail is propagated so the client gets the AWS message assert exc_info.value.status_code == 400 - assert "Violated guardrail policy" in str(exc_info.value.detail) + assert "policy" in str(exc_info.value.detail) def test_bedrock_guardrail_filters_latest_user_message_when_enabled(): From def910b08692f515cb65c22845d2dc6d204ae254 Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Thu, 5 Feb 2026 13:02:34 +0530 Subject: [PATCH 086/117] fix: Sanitize empty text content blocks for databricks provider (#20384) * fix(databricks): sanitize empty text content blocks for Anthropic Messages API * test(databricks): add tests for empty content block sanitization --- .../llms/databricks/chat/transformation.py | 33 ++++++++++++++ .../test_databricks_chat_transformation.py | 43 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index e9ae94307d4..2699b0618bd 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -60,6 +60,38 @@ from ...anthropic.chat.transformation import AnthropicConfig from ...openai_like.chat.transformation import OpenAILikeChatConfig from ..common_utils import DatabricksBase, DatabricksException +def _sanitize_empty_content(message_dict: dict) -> None: + """ + Remove or filter content so empty text blocks are not sent. + Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks. + """ + content = message_dict.get("content") + if content is None: + message_dict.pop("content", None) + return + if isinstance(content, str): + if not content.strip(): + message_dict.pop("content") + return + if isinstance(content, list): + if not content: + message_dict.pop("content") + return + filtered = [ + block + for block in content + if not ( + isinstance(block, dict) + and block.get("type") == "text" + and not (block.get("text") or "").strip() + ) + ] + if not filtered: + message_dict.pop("content") + else: + message_dict["content"] = filtered + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -350,6 +382,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): # Move message-level cache_control into a content block when content is a string. if "cache_control" in _message and isinstance(_message.get("content"), str): _message = self._move_cache_control_into_string_content_block(_message) + _sanitize_empty_content(_message) new_messages.append(_message) if is_async: diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index f437b8405f7..f9b5b5fe29c 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -13,6 +13,7 @@ from unittest.mock import MagicMock, patch from litellm.llms.databricks.chat.transformation import ( DatabricksChatResponseIterator, DatabricksConfig, + _sanitize_empty_content, ) @@ -215,3 +216,45 @@ def test_chunk_parser_with_citation(): "end_char_index": 50, } } + + +def test_sanitize_empty_content_pops_none(): + message = {"role": "user", "content": None} + _sanitize_empty_content(message) + assert "content" not in message + + +def test_sanitize_empty_content_pops_empty_string(): + message = {"role": "user", "content": ""} + _sanitize_empty_content(message) + assert "content" not in message + + +def test_sanitize_empty_content_pops_single_empty_text_block(): + message = {"role": "user", "content": [{"type": "text", "text": ""}]} + _sanitize_empty_content(message) + assert "content" not in message + + +def test_sanitize_empty_content_filters_empty_blocks_keeps_non_empty(): + message = { + "role": "user", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " "}, + ], + } + _sanitize_empty_content(message) + assert message["content"] == [{"type": "text", "text": "Hello"}] + + +def test_transform_messages_sanitizes_empty_content(): + config = DatabricksConfig() + messages = [ + {"role": "user", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": "Hi"}, + ] + result = config._transform_messages(messages=messages, model="databricks-claude", is_async=False) + assert "content" not in result[0] + assert result[1]["content"] == "Hi" From dfbc0e21568aeed32b10111055205a3105b12d1c Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Thu, 5 Feb 2026 12:53:17 +0530 Subject: [PATCH 087/117] fix:Parse embedded JSON in the message field of logs (#20366) * fix:Parse embedded JSON in the message field of logs * Update litellm/_logging.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/_logging.py | 96 +++++++++++++++-- tests/test_litellm/test_logging.py | 162 ++++++++++++++++++++++++----- 2 files changed, 224 insertions(+), 34 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index e222627e76c..fd833f7056a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,9 +1,13 @@ -import json +import ast import logging import os import sys from datetime import datetime from logging import Formatter +from typing import Any, Dict, Optional + +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.safe_json_loads import safe_json_loads set_verbose = False @@ -19,6 +23,67 @@ handler = logging.StreamHandler() handler.setLevel(numeric_level) +def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]: + """ + Try to parse a log message as JSON. Returns parsed dict if valid, else None. + Handles messages that are entirely valid JSON (e.g. json.dumps output). + Uses shared safe_json_loads for consistent error handling. + """ + if not message or not isinstance(message, str): + return None + msg_stripped = message.strip() + if not (msg_stripped.startswith("{") or msg_stripped.startswith("[")): + return None + parsed = safe_json_loads(message, default=None) + if parsed is None or not isinstance(parsed, dict): + return None + return parsed + + +def _try_parse_embedded_python_dict(message: str) -> Optional[Dict[str, Any]]: + """ + Try to find and parse a Python dict repr (e.g. str(d) or repr(d)) embedded in + the message. Handles patterns like: + "get_available_deployment for model: X, Selected deployment: {'model_name': '...', ...} for model: X" + Uses ast.literal_eval for safe parsing. Returns the parsed dict or None. + """ + if not message or not isinstance(message, str) or "{" not in message: + return None + i = 0 + while i < len(message): + start = message.find("{", i) + if start == -1: + break + depth = 0 + for j in range(start, len(message)): + c = message[j] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + substr = message[start : j + 1] + try: + result = ast.literal_eval(substr) + if isinstance(result, dict) and len(result) > 0: + return result + except (ValueError, SyntaxError, TypeError): + pass + break + i = start + 1 + return None + + +# Standard LogRecord attribute names - used to identify 'extra' fields. +# Derived at runtime so we automatically include version-specific attrs (e.g. taskName). +def _get_standard_record_attrs() -> frozenset: + """Standard LogRecord attribute names - excludes extra keys from logger.debug(..., extra={...}).""" + return frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__.keys()) + + +_STANDARD_RECORD_ATTRS = _get_standard_record_attrs() + + class JsonFormatter(Formatter): def __init__(self): super(JsonFormatter, self).__init__() @@ -29,16 +94,31 @@ class JsonFormatter(Formatter): return dt.isoformat() def format(self, record): - json_record = { - "message": record.getMessage(), + message_str = record.getMessage() + json_record: Dict[str, Any] = { + "message": message_str, "level": record.levelname, "timestamp": self.formatTime(record), } + # Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties + parsed = _try_parse_json_message(message_str) + if parsed is None: + parsed = _try_parse_embedded_python_dict(message_str) + if parsed is not None: + for key, value in parsed.items(): + if key not in json_record: + json_record[key] = value + + # Include extra attributes passed via logger.debug("msg", extra={...}) + for key, value in record.__dict__.items(): + if key not in _STANDARD_RECORD_ATTRS and key not in json_record: + json_record[key] = value + if record.exc_info: json_record["stacktrace"] = self.formatException(record.exc_info) - return json.dumps(json_record) + return safe_dumps(json_record) # Function to set up exception handlers for JSON logging @@ -169,15 +249,15 @@ def _initialize_loggers_with_handler(handler: logging.Handler): def _get_uvicorn_json_log_config(): """ Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers. - + This ensures that uvicorn's access logs, error logs, and all application logs are formatted as JSON when json_logs is enabled. """ json_formatter_class = "litellm._logging.JsonFormatter" - + # Use the module-level log_level variable for consistency uvicorn_log_level = log_level.upper() - + log_config = { "version": 1, "disable_existing_loggers": False, @@ -222,7 +302,7 @@ def _get_uvicorn_json_log_config(): }, }, } - + return log_config diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 7e5931d8c0f..6f65ada7459 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,27 +1,21 @@ import asyncio -import datetime import json import os import sys -import unittest -from typing import List, Optional, Tuple -from unittest.mock import ANY, MagicMock, Mock, patch +from typing import List -import httpx import pytest sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system-path -import io import logging import sys -import unittest -from contextlib import redirect_stdout import litellm from litellm._logging import ( ALL_LOGGERS, + JsonFormatter, _initialize_loggers_with_handler, _turn_on_json, verbose_logger, @@ -72,6 +66,117 @@ def test_json_mode_emits_one_record_per_logger(capfd): assert "timestamp" in obj, "`timestamp` key missing" +def test_json_formatter_parses_embedded_json_message(): + """ + Test that JsonFormatter parses embedded JSON in the message field and promotes + sub-fields to first-class JSON properties for downstream querying. + """ + formatter = JsonFormatter() + record = logging.LogRecord( + name="LiteLLM", + level=logging.DEBUG, + pathname="", + lineno=0, + msg='{"event": "giveup", "exception": "Connection failed", "model_name": "gpt-4"}', + args=(), + exc_info=None, + ) + output = formatter.format(record) + obj = json.loads(output) + # Standard fields preserved + assert "message" in obj + assert obj["level"] == "DEBUG" + assert "timestamp" in obj + # Embedded JSON fields promoted to top-level for querying + assert obj["event"] == "giveup" + assert obj["exception"] == "Connection failed" + assert obj["model_name"] == "gpt-4" + + +def test_json_formatter_includes_extra_attributes(): + """ + Test that JsonFormatter includes extra attributes from logger.debug("msg", extra={...}). + """ + formatter = JsonFormatter() + record = logging.LogRecord( + name="LiteLLM", + level=logging.DEBUG, + pathname="", + lineno=0, + msg="POST Request Sent from LiteLLM", + args=(), + exc_info=None, + ) + record.api_base = "https://api.openai.com" + record.authorization = "Bearer sk-***" + output = formatter.format(record) + obj = json.loads(output) + assert obj["message"] == "POST Request Sent from LiteLLM" + assert obj["api_base"] == "https://api.openai.com" + assert obj["authorization"] == "Bearer sk-***" + + +def test_json_formatter_plain_message_unchanged(): + """ + Test that non-JSON messages are passed through as-is in the message field. + """ + formatter = JsonFormatter() + record = logging.LogRecord( + name="LiteLLM", + level=logging.INFO, + pathname="", + lineno=0, + msg="Cache hit!", + args=(), + exc_info=None, + ) + output = formatter.format(record) + obj = json.loads(output) + assert obj["message"] == "Cache hit!" + assert "event" not in obj + assert "exception" not in obj + + +def test_json_formatter_parses_embedded_python_dict_repr(): + """ + Test that JsonFormatter parses Python dict repr (str/deployment) embedded in + plain text, e.g. from get_available_deployment logs. + Reproduces Roni's reported case. + """ + formatter = JsonFormatter() + msg = ( + "get_available_deployment for model: text-embedding-3-large, " + "Selected deployment: {'model_name': 'text-embedding-3-large', " + "'litellm_params': {'api_key': 'sk**********', 'tpm': 1000000, 'rpm': 2000, " + "'use_in_pass_through': False, 'use_litellm_proxy': False, " + "'merge_reasoning_content_in_choices': False, 'model': 'text-embedding-3-large'}, " + "'model_info': {'id': 'a624b057aec64ada48311', 'db_model': False}} " + "for model: text-embedding-3-large" + ) + record = logging.LogRecord( + name="LiteLLM Router", + level=logging.INFO, + pathname="", + lineno=0, + msg=msg, + args=(), + exc_info=None, + ) + output = formatter.format(record) + obj = json.loads(output) + assert "message" in obj + assert obj["level"] == "INFO" + # Python dict parsed and promoted to first-class properties + assert obj["model_name"] == "text-embedding-3-large" + assert "litellm_params" in obj + assert obj["litellm_params"]["api_key"] == "sk**********" + assert obj["litellm_params"]["tpm"] == 1000000 + assert obj["litellm_params"]["use_in_pass_through"] is False + assert "model_info" in obj + assert obj["model_info"]["id"] == "a624b057aec64ada48311" + assert obj["model_info"]["db_model"] is False + + def test_initialize_loggers_with_handler_sets_propagate_false(): """ Test that the initialize_loggers_with_handler function sets propagate to False for all loggers @@ -96,7 +201,7 @@ async def test_cache_hit_includes_custom_llm_provider(): test_custom_logger = CacheHitCustomLogger() original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] litellm.callbacks = [test_custom_logger] - + try: # First call - should be a cache miss response1 = await litellm.acompletion( @@ -105,10 +210,10 @@ async def test_cache_hit_includes_custom_llm_provider(): mock_response="test response", caching=True, ) - + # Wait for logging to complete await asyncio.sleep(0.5) - + # Second identical call - should be a cache hit response2 = await litellm.acompletion( model="gpt-3.5-turbo", @@ -116,38 +221,43 @@ async def test_cache_hit_includes_custom_llm_provider(): mock_response="test response", caching=True, ) - + # Wait for logging to complete await asyncio.sleep(0.5) - + # Verify we have logged events - assert len(test_custom_logger.logged_standard_logging_payloads) >= 2, \ - f"Expected at least 2 logged events, got {len(test_custom_logger.logged_standard_logging_payloads)}" - + assert ( + len(test_custom_logger.logged_standard_logging_payloads) >= 2 + ), f"Expected at least 2 logged events, got {len(test_custom_logger.logged_standard_logging_payloads)}" + # Find the cache hit event (should be the second call) cache_hit_payload = None for payload in test_custom_logger.logged_standard_logging_payloads: if payload.get("cache_hit") is True: cache_hit_payload = payload break - + # Verify cache hit event was found - assert cache_hit_payload is not None, "No cache hit event found in logged payloads" - + assert ( + cache_hit_payload is not None + ), "No cache hit event found in logged payloads" + # Verify custom_llm_provider is included in the cache hit payload - assert "custom_llm_provider" in cache_hit_payload, \ - "custom_llm_provider missing from cache hit standard logging payload" - + assert ( + "custom_llm_provider" in cache_hit_payload + ), "custom_llm_provider missing from cache hit standard logging payload" + # Verify custom_llm_provider has a valid value (should be "openai" for gpt-3.5-turbo) custom_llm_provider = cache_hit_payload["custom_llm_provider"] - assert custom_llm_provider is not None and custom_llm_provider != "", \ - f"custom_llm_provider should not be None or empty, got: {custom_llm_provider}" - + assert ( + custom_llm_provider is not None and custom_llm_provider != "" + ), f"custom_llm_provider should not be None or empty, got: {custom_llm_provider}" + print( f"Cache hit standard logging payload with custom_llm_provider: {custom_llm_provider}", json.dumps(cache_hit_payload, indent=2), ) - + finally: # Clean up litellm.callbacks = original_callbacks From aa3325c1d5ea50e911977bfbe6435d6b0e8a20a0 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Thu, 5 Feb 2026 09:16:56 +0200 Subject: [PATCH 088/117] fix: map global location to us-central1 for vertex's claude models count_tokens endpoint (#20348) * fix: map global location to us-east5 for Claude count_tokens endpoint - Vertex AI doesn't support count_tokens endpoint for Claude models with global location - Map global -> us-east5 for count_tokens only, keeping global for inference - Fixes 404 error when calling count_tokens with vertex_location: global - Reference: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens * Update handler.py --- .../vertex_ai_partner_models/count_tokens/handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 3842159fd7b..c6914ac3d6b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -107,6 +107,11 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): vertex_project = self.get_vertex_ai_project(litellm_params) vertex_location = self.get_vertex_ai_location(litellm_params) + # Map empty location/cluade models to a supported region for count-tokens endpoint + # https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens + if not vertex_location or "claude" in model.lower(): + vertex_location = "us-central1" + # Get access token and resolved project ID access_token, project_id = await self._ensure_access_token_async( credentials=vertex_credentials, @@ -118,7 +123,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): endpoint_url = self._build_count_tokens_endpoint( model=model, project_id=project_id, - vertex_location=vertex_location or "us-central1", + vertex_location=vertex_location, api_base=litellm_params.get("api_base"), ) From d56a0a97f8ddc348ab761ec0cf270fe09cee2dba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 5 Feb 2026 16:16:00 +0900 Subject: [PATCH 089/117] fix(ui): allow editing MCP stdio transport config (#20241) * fix(ui): enable stdio transport edits for MCP servers * fix(ui): use antd Input in MCP edit stdio Align MCP Server Edit with UI guidelines by replacing deprecated Tremor TextInput, and relax stdio args validation to match create flow while improving test stability. * fix(otel): make semantic log LogRecord import mypy-safe Prefer the OTEL >=1.39.0 LogRecord import path and keep an ignored fallback for older versions so MyPy doesn't fail on newer SDK stubs. * fix(otel): tolerate LogRecord ctor changes across SDK versions Create semantic LogRecords via a best-effort wrapper that falls back when the `resource` kwarg is unsupported (OTEL >= 1.39), and avoid MyPy overload/no-redef failures. * fix(otel): silence mypy no-redef on versioned LogRecord import MyPy sees both branches of the version-compat import and flags a redefinition. Ignore no-redef on the legacy import path to keep CI passing. * fix(ui): ensure mcp_info.server_name is always populated When using stdio transport there may be no URL to fall back on; prefer existing server_name/url/alias to avoid sending an empty mcp_info.server_name on update. * chore(otel): format opentelemetry; ignore ui export output * fix: guard optional a2a resolver + make OTEL semantic logs mypy-safe * chore: format A2A resolver and OTEL semantic logs * fix: address review feedback for MCP stdio edit * fix: keep MCP stdio edit PR scoped * fix(otel): make semantic logs mypy-safe --- litellm/integrations/opentelemetry.py | 235 +++++++--- .../mcp_tools/StdioConfiguration.tsx | 9 +- .../mcp_tools/mcp_server_columns.tsx | 6 +- .../mcp_tools/mcp_server_edit.test.tsx | 154 +++++++ .../components/mcp_tools/mcp_server_edit.tsx | 410 ++++++++++++++++-- .../components/mcp_tools/mcp_server_view.tsx | 6 +- .../src/components/mcp_tools/types.tsx | 12 +- 7 files changed, 711 insertions(+), 121 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 296a88f9a0b..138d508db4b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -5,6 +5,10 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast import litellm from litellm._logging import verbose_logger +from litellm.integrations._types.open_inference import ( + OpenInferenceSpanKindValues, + SpanAttributes, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.secret_managers.main import get_secret_bool @@ -17,10 +21,6 @@ from litellm.types.utils import ( StandardCallbackDynamicParams, StandardLoggingPayload, ) -from litellm.integrations._types.open_inference import ( - OpenInferenceSpanKindValues, - SpanAttributes, -) # OpenTelemetry imports moved to individual functions to avoid import errors when not installed @@ -40,7 +40,9 @@ if TYPE_CHECKING: Context = Union[_Context, Any] SpanExporter = Union[_SpanExporter, Any] UserAPIKeyAuth = Union[_UserAPIKeyAuth, Any] - ManagementEndpointLoggingPayload = Union[_ManagementEndpointLoggingPayload, Any] + ManagementEndpointLoggingPayload = Union[ + _ManagementEndpointLoggingPayload, Any + ] else: Span = Any Tracer = Any @@ -95,12 +97,16 @@ class OpenTelemetryConfig: exporter = os.getenv( "OTEL_EXPORTER_OTLP_PROTOCOL", os.getenv("OTEL_EXPORTER", "console") ) - endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT")) + endpoint = os.getenv( + "OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT") + ) headers = os.getenv( "OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS") ) # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" enable_metrics: bool = ( - os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower() + os.getenv( + "LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false" + ).lower() == "true" ) enable_events: bool = ( @@ -108,7 +114,9 @@ class OpenTelemetryConfig: == "true" ) service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") - deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production") + deployment_environment = os.getenv( + "OTEL_ENVIRONMENT_NAME", "production" + ) model_id = os.getenv("OTEL_MODEL_ID", service_name) if exporter == "in_memory": @@ -157,7 +165,9 @@ class OpenTelemetry(CustomLogger): logging.getLogger(__name__) # Enable OpenTelemetry logging - otel_exporter_logger = logging.getLogger("opentelemetry.sdk.trace.export") + otel_exporter_logger = logging.getLogger( + "opentelemetry.sdk.trace.export" + ) otel_exporter_logger.setLevel(logging.DEBUG) # init CustomLogger params @@ -253,7 +263,9 @@ class OpenTelemetry(CustomLogger): # Don't call set_provider to preserve existing context else: # Default proxy provider or unknown type, create our own - verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name) + verbose_logger.debug( + "OpenTelemetry: Creating new %s", provider_name + ) provider = create_new_provider_fn() set_provider_fn(provider) except Exception as e: @@ -274,7 +286,9 @@ class OpenTelemetry(CustomLogger): from opentelemetry.trace import SpanKind def create_tracer_provider(): - provider = TracerProvider(resource=self._get_litellm_resource(self.config)) + provider = TracerProvider( + resource=self._get_litellm_resource(self.config) + ) provider.add_span_processor(self._get_span_processor()) return provider @@ -388,10 +402,14 @@ class OpenTelemetry(CustomLogger): def log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event( + self, kwargs, response_obj, start_time, end_time + ): self._handle_success(kwargs, response_obj, start_time, end_time) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event( + self, kwargs, response_obj, start_time, end_time + ): self._handle_failure(kwargs, response_obj, start_time, end_time) async def async_service_success_hook( @@ -588,7 +606,9 @@ class OpenTelemetry(CustomLogger): if dynamic_headers is not None: # Create spans using a temporary tracer with dynamic headers - tracer_to_use = self._get_tracer_with_dynamic_headers(dynamic_headers) + tracer_to_use = self._get_tracer_with_dynamic_headers( + dynamic_headers + ) verbose_logger.debug( "Using dynamic headers for this request: %s", dynamic_headers ) @@ -624,7 +644,9 @@ class OpenTelemetry(CustomLogger): ) # Create a temporary tracer provider with dynamic headers - temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config)) + temp_provider = TracerProvider( + resource=self._get_litellm_resource(self.config) + ) temp_provider.add_span_processor( self._get_span_processor(dynamic_headers=dynamic_headers) ) @@ -755,7 +777,9 @@ class OpenTelemetry(CustomLogger): metadata = litellm_params.get("metadata") or {} generation_name = metadata.get("generation_name") - raw_span_name = generation_name if generation_name else RAW_REQUEST_SPAN_NAME + raw_span_name = ( + generation_name if generation_name else RAW_REQUEST_SPAN_NAME + ) otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) raw_span = otel_tracer.start_span( @@ -780,7 +804,9 @@ class OpenTelemetry(CustomLogger): } std_log = kwargs.get("standard_logging_object") - md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {}) + md = getattr(std_log, "metadata", None) or (std_log or {}).get( + "metadata", {} + ) for key in [ "user_api_key_hash", "user_api_key_alias", @@ -802,9 +828,9 @@ class OpenTelemetry(CustomLogger): common_attrs[f"metadata.{key}"] = str(md[key]) # get hidden params - hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get( - "hidden_params", {} - ) + hidden_params = getattr(std_log, "hidden_params", None) or ( + std_log or {} + ).get("hidden_params", {}) if hidden_params: common_attrs["hidden_params"] = safe_dumps(hidden_params) @@ -838,7 +864,9 @@ class OpenTelemetry(CustomLogger): self._record_response_duration_metric(kwargs, end_time, common_attrs) @staticmethod - def _to_timestamp(val: Optional[Union[datetime, float, str]]) -> Optional[float]: + def _to_timestamp( + val: Optional[Union[datetime, float, str]], + ) -> Optional[float]: """Convert datetime/float/string to timestamp.""" if val is None: return None @@ -855,7 +883,9 @@ class OpenTelemetry(CustomLogger): except ValueError: return None - def _record_time_to_first_token_metric(self, kwargs: dict, common_attrs: dict): + def _record_time_to_first_token_metric( + self, kwargs: dict, common_attrs: dict + ): """Record Time to First Token (TTFT) metric for streaming requests.""" optional_params = kwargs.get("optional_params", {}) is_streaming = optional_params.get("stream", False) @@ -868,7 +898,10 @@ class OpenTelemetry(CustomLogger): api_call_start_time = kwargs.get("api_call_start_time", None) completion_start_time = kwargs.get("completion_start_time", None) - if api_call_start_time is not None and completion_start_time is not None: + if ( + api_call_start_time is not None + and completion_start_time is not None + ): # Convert to timestamps if needed (handles datetime, float, and string) api_call_start_ts = self._to_timestamp(api_call_start_time) completion_start_ts = self._to_timestamp(completion_start_time) @@ -876,7 +909,9 @@ class OpenTelemetry(CustomLogger): if api_call_start_ts is None or completion_start_ts is None: return # Skip recording if conversion failed - time_to_first_token_seconds = completion_start_ts - api_call_start_ts + time_to_first_token_seconds = ( + completion_start_ts - api_call_start_ts + ) self._time_to_first_token_histogram.record( time_to_first_token_seconds, attributes=common_attrs ) @@ -946,7 +981,9 @@ class OpenTelemetry(CustomLogger): generation_time_seconds = duration_s if generation_time_seconds > 0: - time_per_output_token_seconds = generation_time_seconds / completion_tokens + time_per_output_token_seconds = ( + generation_time_seconds / completion_tokens + ) self._time_per_output_token_histogram.record( time_per_output_token_seconds, attributes=common_attrs ) @@ -1007,21 +1044,26 @@ class OpenTelemetry(CustomLogger): # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676 # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords - from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider + from opentelemetry._logs import ( + SeverityNumber, + get_logger, + ) - try: - from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0 - except ImportError: - from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0 + # MyPy evaluates both branches of try/except imports and can fail when + # newer OTEL stubs remove/relocate symbols. Gate the typing import so + # only the canonical location is type-checked. + if TYPE_CHECKING: + from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord + else: + try: + from opentelemetry.sdk._logs import ( + LogRecord as SdkLogRecord, # type: ignore[attr-defined] + ) + except ImportError: + from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord otel_logger = get_logger(LITELLM_LOGGER_NAME) - # Get the resource from the logger provider - logger_provider = get_logger_provider() - resource = getattr( - logger_provider, "_resource", None - ) or self._get_litellm_resource(self.config) - parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( "custom_llm_provider", "Unknown" @@ -1030,7 +1072,10 @@ class OpenTelemetry(CustomLogger): # per-message events for msg in kwargs.get("messages", []): role = msg.get("role", "user") - attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider} + attrs = { + "event_name": "gen_ai.content.prompt", + "gen_ai.system": provider, + } if role == "tool" and msg.get("id"): attrs["id"] = msg["id"] if self.message_logging and msg.get("content"): @@ -1044,7 +1089,6 @@ class OpenTelemetry(CustomLogger): severity_number=SeverityNumber.INFO, severity_text="INFO", body=msg.copy(), - resource=resource, attributes=attrs, ) otel_logger.emit(log_record) @@ -1076,7 +1120,6 @@ class OpenTelemetry(CustomLogger): severity_number=SeverityNumber.INFO, severity_text="INFO", body=body, - resource=resource, attributes=attrs, ) otel_logger.emit(log_record) @@ -1146,7 +1189,9 @@ class OpenTelemetry(CustomLogger): value=guardrail_information.get("guardrail_mode"), ) - masked_entity_count = guardrail_information.get("masked_entity_count") + masked_entity_count = guardrail_information.get( + "masked_entity_count" + ) if masked_entity_count is not None: guardrail_span.set_attribute( "masked_entity_count", safe_dumps(masked_entity_count) @@ -1173,8 +1218,9 @@ class OpenTelemetry(CustomLogger): # Decide whether to create a primary span # Always create if no parent span exists (backward compatibility) # OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled - should_create_primary_span = parent_otel_span is None or get_secret_bool( - "USE_OTEL_LITELLM_REQUEST_SPAN" + should_create_primary_span = ( + parent_otel_span is None + or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN") ) if should_create_primary_span: @@ -1200,7 +1246,9 @@ class OpenTelemetry(CustomLogger): if parent_otel_span.is_recording(): parent_otel_span.set_status(Status(StatusCode.ERROR)) self.set_attributes(parent_otel_span, kwargs, response_obj) - self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) + self._record_exception_on_span( + span=parent_otel_span, kwargs=kwargs + ) # Create span for guardrail information self._create_guardrail_span(kwargs=kwargs, context=_parent_context) @@ -1223,7 +1271,9 @@ class OpenTelemetry(CustomLogger): 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation """ try: - from litellm.integrations._types.open_inference import ErrorAttributes + from litellm.integrations._types.open_inference import ( + ErrorAttributes, + ) # Get the exception object if available exception = kwargs.get("exception") @@ -1233,15 +1283,17 @@ class OpenTelemetry(CustomLogger): span.record_exception(exception) # Get StandardLoggingPayload for structured error information - standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( - "standard_logging_object" + standard_logging_payload: Optional[StandardLoggingPayload] = ( + kwargs.get("standard_logging_object") ) if standard_logging_payload is None: return # Extract error_information from StandardLoggingPayload - error_information = standard_logging_payload.get("error_information") + error_information = standard_logging_payload.get( + "error_information" + ) if error_information is None: # Fallback to error_str if error_information is not available @@ -1331,7 +1383,9 @@ class OpenTelemetry(CustomLogger): ) pass - def cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]: + def cast_as_primitive_value_type( + self, value + ) -> Union[str, bool, int, float]: """ Casts the value to a primitive OTEL type if it is not already a primitive type. @@ -1401,8 +1455,8 @@ class OpenTelemetry(CustomLogger): optional_params = kwargs.get("optional_params", {}) litellm_params = kwargs.get("litellm_params", {}) or {} - standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( - "standard_logging_object" + standard_logging_payload: Optional[StandardLoggingPayload] = ( + kwargs.get("standard_logging_object") ) if standard_logging_payload is None: raise ValueError("standard_logging_object not found in kwargs") @@ -1424,11 +1478,13 @@ class OpenTelemetry(CustomLogger): ) or (standard_logging_payload or {}).get("hidden_params", {}) if hidden_params: self.safe_set_attribute( - span=span, key="hidden_params", value=safe_dumps(hidden_params) + span=span, + key="hidden_params", + value=safe_dumps(hidden_params), ) # Cost breakdown tracking - cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get( - "cost_breakdown" + cost_breakdown: Optional[CostBreakdown] = ( + standard_logging_payload.get("cost_breakdown") ) if cost_breakdown: for key, value in cost_breakdown.items(): @@ -1504,7 +1560,9 @@ class OpenTelemetry(CustomLogger): # The unique identifier for the completion. if response_obj and response_obj.get("id"): self.safe_set_attribute( - span=span, key="gen_ai.response.id", value=response_obj.get("id") + span=span, + key="gen_ai.response.id", + value=response_obj.get("id"), ) # The model used to generate the response. @@ -1639,7 +1697,9 @@ class OpenTelemetry(CustomLogger): "OpenTelemetry logging error in set_attributes %s", str(e) ) - def _cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]: + def _cast_as_primitive_value_type( + self, value + ) -> Union[str, bool, int, float]: """ Casts the value to a primitive OTEL type if it is not already a primitive type. @@ -1673,7 +1733,10 @@ class OpenTelemetry(CustomLogger): if isinstance(messages, str): # Handle system_instructions passed as a string return [ - {"role": "system", "parts": [{"type": "text", "content": messages}]} + { + "role": "system", + "parts": [{"type": "text", "content": messages}], + } ] transformed = [] @@ -1714,9 +1777,11 @@ class OpenTelemetry(CustomLogger): message = choice.get("message") or {} finish_reason = choice.get("finish_reason") - transformed_msg = self._transform_messages_to_otel_semantic_conventions( - [message] - )[0] + transformed_msg = ( + self._transform_messages_to_otel_semantic_conventions( + [message] + )[0] + ) if finish_reason: transformed_msg["finish_reason"] = finish_reason @@ -1728,7 +1793,9 @@ class OpenTelemetry(CustomLogger): self.set_attributes(span, kwargs, response_obj) kwargs.get("optional_params", {}) litellm_params = kwargs.get("litellm_params", {}) or {} - custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") + custom_llm_provider = litellm_params.get( + "custom_llm_provider", "Unknown" + ) _raw_response = kwargs.get("original_response") _additional_args = kwargs.get("additional_args", {}) or {} @@ -1741,7 +1808,9 @@ class OpenTelemetry(CustomLogger): if complete_input_dict and isinstance(complete_input_dict, dict): for param, val in complete_input_dict.items(): self.safe_set_attribute( - span=span, key=f"llm.{custom_llm_provider}.{param}", value=val + span=span, + key=f"llm.{custom_llm_provider}.{param}", + value=val, ) ############################################# @@ -1773,7 +1842,8 @@ class OpenTelemetry(CustomLogger): ) except Exception as e: verbose_logger.exception( - "OpenTelemetry logging error in set_raw_request_attributes %s", str(e) + "OpenTelemetry logging error in set_raw_request_attributes %s", + str(e), ) def _to_ns(self, dt): @@ -1813,7 +1883,9 @@ class OpenTelemetry(CustomLogger): ) litellm_params = kwargs.get("litellm_params", {}) or {} - proxy_server_request = litellm_params.get("proxy_server_request", {}) or {} + proxy_server_request = ( + litellm_params.get("proxy_server_request", {}) or {} + ) headers = proxy_server_request.get("headers", {}) or {} traceparent = headers.get("traceparent", None) _metadata = litellm_params.get("metadata", {}) or {} @@ -1832,7 +1904,10 @@ class OpenTelemetry(CustomLogger): "OpenTelemetry: Using traceparent header for context propagation" ) carrier = {"traceparent": traceparent} - return TraceContextTextMapPropagator().extract(carrier=carrier), None + return ( + TraceContextTextMapPropagator().extract(carrier=carrier), + None, + ) # Priority 3: Active span from global context (auto-detection) try: @@ -1960,10 +2035,14 @@ class OpenTelemetry(CustomLogger): self.OTEL_HEADERS, ) - _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + _split_otel_headers = OpenTelemetry._get_headers_dictionary( + self.OTEL_HEADERS + ) # Normalize endpoint for logs - ensure it points to /v1/logs instead of /v1/traces - normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "logs") + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "logs" + ) verbose_logger.debug( "OpenTelemetry: Log endpoint normalized from %s to %s", @@ -2051,14 +2130,18 @@ class OpenTelemetry(CustomLogger): self.OTEL_HEADERS, ) - _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + _split_otel_headers = OpenTelemetry._get_headers_dictionary( + self.OTEL_HEADERS + ) normalized_endpoint = self._normalize_otel_endpoint( self.OTEL_ENDPOINT, "metrics" ) if self.OTEL_EXPORTER == "console": exporter = ConsoleMetricExporter() - return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + return PeriodicExportingMetricReader( + exporter, export_interval_millis=5000 + ) elif ( self.OTEL_EXPORTER == "otlp_http" @@ -2074,7 +2157,9 @@ class OpenTelemetry(CustomLogger): headers=_split_otel_headers, preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) - return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + return PeriodicExportingMetricReader( + exporter, export_interval_millis=5000 + ) elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": try: @@ -2092,7 +2177,9 @@ class OpenTelemetry(CustomLogger): headers=_split_otel_headers, preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) - return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + return PeriodicExportingMetricReader( + exporter, export_interval_millis=5000 + ) else: verbose_logger.warning( @@ -2100,7 +2187,9 @@ class OpenTelemetry(CustomLogger): self.OTEL_EXPORTER, ) exporter = ConsoleMetricExporter() - return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + return PeriodicExportingMetricReader( + exporter, export_interval_millis=5000 + ) def _normalize_otel_endpoint( self, endpoint: Optional[str], signal_type: str @@ -2171,7 +2260,9 @@ class OpenTelemetry(CustomLogger): return endpoint @staticmethod - def _get_headers_dictionary(headers: Optional[Union[str, dict]]) -> Dict[str, str]: + def _get_headers_dictionary( + headers: Optional[Union[str, dict]], + ) -> Dict[str, str]: """ Convert a string or dictionary of headers into a dictionary of headers. """ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/StdioConfiguration.tsx b/ui/litellm-dashboard/src/components/mcp_tools/StdioConfiguration.tsx index 23f5f84fead..476a5b61683 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/StdioConfiguration.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/StdioConfiguration.tsx @@ -4,9 +4,14 @@ import { InfoCircleOutlined } from "@ant-design/icons"; interface StdioConfigurationProps { isVisible: boolean; + /** + * When true, stdio_config is required + validated as JSON. + * Edit screen can set this to false when using dedicated command/args/env fields. + */ + required?: boolean; } -const StdioConfiguration: React.FC = ({ isVisible }) => { +const StdioConfiguration: React.FC = ({ isVisible, required = true }) => { if (!isVisible) return null; return ( @@ -21,7 +26,7 @@ const StdioConfiguration: React.FC = ({ isVisible }) => } name="stdio_config" rules={[ - { required: true, message: "Please enter stdio configuration" }, + ...(required ? [{ required: true, message: "Please enter stdio configuration" }] : []), { validator: (_, value) => { if (!value) return Promise.resolve(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx index ac611c633aa..78e9c6f465a 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx @@ -36,7 +36,11 @@ export const mcpServerColumns = ( id: "url", header: "URL", cell: ({ row }) => { - const { maskedUrl } = getMaskedAndFullUrl(row.original.url); + const url = row.original.url; + if (!url) { + return —; + } + const { maskedUrl } = getMaskedAndFullUrl(url); return {maskedUrl}; }, }, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx new file mode 100644 index 00000000000..e33e2fff491 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -0,0 +1,154 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, waitFor, fireEvent, act } from "@testing-library/react"; +import MCPServerEdit from "./mcp_server_edit"; +import * as networking from "../networking"; + +vi.mock("../networking", () => ({ + updateMCPServer: vi.fn(), + testMCPToolsListRequest: vi.fn().mockResolvedValue({ tools: [], error: null }), +})); + +vi.mock("../molecules/notifications_manager", () => ({ + default: { + success: vi.fn(), + fromBackend: vi.fn(), + }, +})); + +vi.mock("@/hooks/useMcpOAuthFlow", () => ({ + useMcpOAuthFlow: () => ({ + startOAuthFlow: vi.fn(), + status: "idle", + error: null, + tokenResponse: null, + }), +})); + +vi.mock("./mcp_server_cost_config", () => ({ + default: () =>
, +})); + +vi.mock("./MCPPermissionManagement", () => ({ + default: () =>
, +})); + +vi.mock("./mcp_tool_configuration", () => ({ + default: () =>
, +})); + +describe("MCPServerEdit (stdio)", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render without crashing", () => { + render( + , + ); + + expect(screen.getByRole("tab", { name: "Server Configuration" })).toBeInTheDocument(); + }); + + it("should allow updating stdio transport configuration", async () => { + const onCancel = vi.fn(); + const onSuccess = vi.fn(); + + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + server_id: "server-1", + server_name: "TestServer", + alias: "test", + transport: "stdio", + url: null, + command: "npx", + args: ["-y", "@circleci/mcp-server-circleci"], + env: { CIRCLECI_TOKEN: "***" }, + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }); + + render( + , + ); + + // Stdio section should be visible + expect(screen.getByLabelText("Command")).toBeInTheDocument(); + + // URL field should not be visible when transport=stdio + expect(screen.queryByText("MCP Server URL")).not.toBeInTheDocument(); + + // Update env_json + const envTextarea = screen.getByLabelText("Environment (JSON object)"); + await act(async () => { + fireEvent.change(envTextarea, { + target: { + value: JSON.stringify({ CIRCLECI_TOKEN: "new-token", CIRCLECI_BASE_URL: "https://circleci.com" }, null, 2), + }, + }); + }); + + const saveButtons = screen.getAllByRole("button", { name: "Save Changes" }); + const saveButton = saveButtons[0]; + await act(async () => { + fireEvent.click(saveButton); + }); + + await waitFor(() => { + expect(networking.updateMCPServer).toHaveBeenCalledTimes(1); + }); + + const [_token, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + expect(_token).toBe("access-token"); + expect(payload.transport).toBe("stdio"); + expect(payload.command).toBe("npx"); + expect(payload.args).toEqual(["-y", "@circleci/mcp-server-circleci"]); + expect(payload.env).toEqual({ CIRCLECI_TOKEN: "new-token", CIRCLECI_BASE_URL: "https://circleci.com" }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 05a5a3cf4bf..fa46521e19d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import { Form, Select, Button as AntdButton, Tooltip } from "antd"; +import { Form, Select, Button as AntdButton, Tooltip, Input } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TextInput, TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; import { AUTH_TYPE, OAUTH_FLOW, MCPServer, MCPServerCostInfo } from "./types"; @@ -8,6 +8,7 @@ import { updateMCPServer, testMCPToolsListRequest } from "../networking"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; import MCPToolConfiguration from "./mcp_tool_configuration"; +import StdioConfiguration from "./StdioConfiguration"; import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "../molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; @@ -40,6 +41,8 @@ const MCPServerEdit: React.FC = ({ const [allowedTools, setAllowedTools] = useState([]); const [pendingRestoredValues, setPendingRestoredValues] = useState | null>(null); const authType = Form.useWatch("auth_type", form) as string | undefined; + const transportType = Form.useWatch("transport", form) as string | undefined; + const isStdioTransport = transportType === "stdio"; const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false; const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2; const oauthFlowTypeValue = Form.useWatch("oauth_flow_type", form) as string | undefined; @@ -127,13 +130,26 @@ const MCPServerEdit: React.FC = ({ })); }, [mcpServer.static_headers]); + const initialEnvJson = React.useMemo(() => { + const env = mcpServer.env ?? undefined; + if (!env || Object.keys(env).length === 0) { + return ""; + } + try { + return JSON.stringify(env, null, 2); + } catch { + return ""; + } + }, [mcpServer.env]); + + const initialValues = React.useMemo( () => ({ ...mcpServer, static_headers: initialStaticHeaders, oauth_flow_type: mcpServer.token_url ? OAUTH_FLOW.M2M : OAUTH_FLOW.INTERACTIVE, }), - [mcpServer, initialStaticHeaders], + [mcpServer, initialStaticHeaders, initialEnvJson], ); // Initialize cost config from existing server data @@ -214,9 +230,10 @@ const MCPServerEdit: React.FC = ({ }, [mcpServer, accessToken, oauthAccessToken]); const fetchTools = async () => { - if (!accessToken || !mcpServer.url) { - return; - } + if (!accessToken) return; + + // HTTP/SSE requires a URL; stdio does not. + if (mcpServer.transport !== "stdio" && !mcpServer.url) return; const isM2M = mcpServer.auth_type === AUTH_TYPE.OAUTH2 && !!mcpServer.token_url; if (mcpServer.auth_type === AUTH_TYPE.OAUTH2 && !isM2M && !oauthAccessToken) { @@ -237,6 +254,9 @@ const MCPServerEdit: React.FC = ({ authorization_url: mcpServer.authorization_url, token_url: mcpServer.token_url, registration_url: mcpServer.registration_url, + command: mcpServer.command, + args: mcpServer.args, + env: mcpServer.env, }; const toolsResponse = await testMCPToolsListRequest(accessToken, mcpServerConfig, oauthAccessToken); @@ -287,6 +307,27 @@ const MCPServerEdit: React.FC = ({ return existingOptions; }; + const handleTransportChange = (value: string) => { + // Clear fields that are not relevant for the selected transport. + if (value === "stdio") { + form.setFieldsValue({ + url: undefined, + auth_type: undefined, + credentials: undefined, + authorization_url: undefined, + token_url: undefined, + registration_url: undefined, + }); + } else { + form.setFieldsValue({ + command: undefined, + args: undefined, + env_json: undefined, + stdio_config: undefined, + }); + } + }; + const handleSave = async (values: Record) => { if (!accessToken) return; try { @@ -294,6 +335,10 @@ const MCPServerEdit: React.FC = ({ const { static_headers: staticHeadersList, credentials: credentialValues, + stdio_config: rawStdioConfig, + env_json: rawEnvJson, + command: rawCommand, + args: rawArgs, allow_all_keys: allowAllKeysRaw, available_on_public_internet: availableOnPublicInternetRaw, ...restValues @@ -334,12 +379,104 @@ const MCPServerEdit: React.FC = ({ }, {}) : undefined; + let stdioFields: Record = {}; + + if (restValues.transport === "stdio") { + // Prefer JSON config if provided (matches Create screen behavior) + if (rawStdioConfig) { + try { + const stdioConfig = JSON.parse(rawStdioConfig); + + let actualConfig = stdioConfig; + if (stdioConfig?.mcpServers && typeof stdioConfig.mcpServers === "object") { + const serverNames = Object.keys(stdioConfig.mcpServers); + if (serverNames.length > 0) { + actualConfig = stdioConfig.mcpServers[serverNames[0]]; + } + } + + const parsedArgs = Array.isArray(actualConfig?.args) + ? actualConfig.args.map((v: any) => String(v)).filter((v: string) => v.trim() !== "") + : []; + + const parsedEnv = + actualConfig?.env && typeof actualConfig.env === "object" && !Array.isArray(actualConfig.env) + ? Object.entries(actualConfig.env).reduce((acc: Record, [k, v]) => { + if (k == null || String(k).trim() === "") return acc; + acc[String(k)] = v == null ? "" : String(v); + return acc; + }, {}) + : {}; + + stdioFields = { + command: actualConfig?.command ? String(actualConfig.command) : undefined, + args: parsedArgs, + env: parsedEnv, + }; + + if (!stdioFields.command) { + NotificationsManager.fromBackend("Stdio configuration must include a command"); + return; + } + } catch { + NotificationsManager.fromBackend("Invalid JSON in stdio configuration"); + return; + } + } else { + // Dedicated fields path (command/args + env JSON) + let parsedEnv: Record = {}; + if (rawEnvJson) { + try { + const env = JSON.parse(rawEnvJson); + if (env && typeof env === "object" && !Array.isArray(env)) { + parsedEnv = Object.entries(env).reduce((acc: Record, [k, v]) => { + if (k == null || String(k).trim() === "") return acc; + acc[String(k)] = v == null ? "" : String(v); + return acc; + }, {}); + } + } catch { + NotificationsManager.fromBackend("Invalid JSON in stdio env configuration"); + return; + } + } + const parsedArgs = Array.isArray(rawArgs) + ? rawArgs.map((v: any) => String(v)).filter((v: string) => v.trim() !== "") + : []; + + const parsedCommand = rawCommand ? String(rawCommand).trim() : ""; + if (!parsedCommand) { + NotificationsManager.fromBackend("Stdio transport requires a command"); + return; + } + + stdioFields = { + command: parsedCommand, + args: parsedArgs, + env: parsedEnv, + }; + } + } + // Prepare the payload with cost configuration and permission fields + const mcpInfoServerName = + restValues.server_name || + restValues.url || + mcpServer.server_name || + mcpServer.url || + restValues.alias || + mcpServer.alias || + "unknown"; + const payload: Record = { ...restValues, + ...stdioFields, + // Remove UI-only fields + stdio_config: undefined, + env_json: undefined, server_id: mcpServer.server_id, mcp_info: { - server_name: restValues.server_name || restValues.url, + server_name: mcpInfoServerName, description: restValues.description, mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, }, @@ -386,7 +523,7 @@ const MCPServerEdit: React.FC = ({ }, ]} > - + = ({ }, ]} > - setAliasManuallyEdited(true)} /> + setAliasManuallyEdited(true)} + className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" + /> - - - validateMCPServerUrl(value) }, - ]} - > - + - Server-Sent Events (SSE) HTTP - - - - - {shouldShowAuthValueField && ( + {/* URL/Auth fields are only applicable for HTTP/SSE */} + {!isStdioTransport && ( + validateMCPServerUrl(value) }, + ]} + > + + + )} + + {!isStdioTransport && ( + + + + )} + + {isStdioTransport && ( +
+

+ Configure the stdio transport used to launch the MCP server process. You can either fill in the fields + below or paste a JSON configuration. +

+ + + + + + + + + + Authorization URL Override (optional) + + + + + } + name="authorization_url" + > + + + + Token URL Override (optional) + + + + + } + name="token_url" + > + + + + Registration URL Override (optional) + + + + + } + name="registration_url" + > + + +
+

Use OAuth to fetch a fresh access token and temporarily save it in the session as the authentication value.

+ + {oauthError &&

{oauthError}

} + {oauthStatus === "success" && oauthTokenResponse?.access_token && ( +

+ Token fetched. Expires in {oauthTokenResponse.expires_in ?? "?"} seconds. +

+ )} +
+ )} {/* Permission Management / Access Control Section */} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx index 960bd4a5183..635c787f30d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx @@ -43,9 +43,11 @@ export const MCPServerView: React.FC = ({ onBack(); }; - const { maskedUrl, hasToken } = getMaskedAndFullUrl(mcpServer.url); + const urlValue = mcpServer.url ?? ""; + const { maskedUrl, hasToken } = urlValue ? getMaskedAndFullUrl(urlValue) : { maskedUrl: "—", hasToken: false }; - const renderUrlWithToggle = (url: string, showFull: boolean) => { + const renderUrlWithToggle = (url: string | null | undefined, showFull: boolean) => { + if (!url) return "—"; if (!hasToken) return url; return showFull ? url : maskedUrl; }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 5cb840ec7d4..ecc4171a8ac 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -21,6 +21,7 @@ export const OAUTH_FLOW = { export const TRANSPORT = { SSE: "sse", HTTP: "http", + STDIO: "stdio", }; export const handleTransport = (transport?: string | null): string => { @@ -137,7 +138,11 @@ export interface MCPServer { server_name?: string | null; alias?: string | null; description?: string | null; - url: string; + /** + * Only required for HTTP/SSE transports. + * For `stdio`, the backend can return null/undefined. + */ + url?: string | null; transport?: string | null; auth_type?: string | null; authorization_url?: string | null; @@ -158,6 +163,11 @@ export interface MCPServer { allowed_tools?: string[]; allow_all_keys?: boolean; available_on_public_internet?: boolean; + + /** Stdio-only fields (present when transport === 'stdio') */ + command?: string | null; + args?: string[] | null; + env?: Record | null; } export interface MCPServerProps { From 8663ec8610bf27d92f323ba945e63b5c9152b863 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 16:43:32 +0530 Subject: [PATCH 090/117] Add Kimi model pricing by region --- ...odel_prices_and_context_window_backup.json | 66 +++++++++++++++++++ model_prices_and_context_window.json | 66 +++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 815d29c7964..a27aa59376f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6091,6 +6091,17 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", @@ -6109,6 +6120,17 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.94e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6314,6 +6336,17 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6450,6 +6483,28 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6856,6 +6911,17 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 815d29c7964..a27aa59376f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6091,6 +6091,17 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", @@ -6109,6 +6120,17 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.94e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6314,6 +6336,17 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6450,6 +6483,28 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6856,6 +6911,17 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, From 849d6b7cdbcd51afa4e9fecd3c748c8090c49d6e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:00:49 +0530 Subject: [PATCH 091/117] Add perplexity response api class --- litellm/llms/perplexity/responses/__init__.py | 7 + .../perplexity/responses/transformation.py | 509 ++++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 litellm/llms/perplexity/responses/__init__.py create mode 100644 litellm/llms/perplexity/responses/transformation.py diff --git a/litellm/llms/perplexity/responses/__init__.py b/litellm/llms/perplexity/responses/__init__.py new file mode 100644 index 00000000000..9bdf810e839 --- /dev/null +++ b/litellm/llms/perplexity/responses/__init__.py @@ -0,0 +1,7 @@ +""" +Perplexity Agentic Research API (Responses API) module +""" + +from .transformation import PerplexityResponsesConfig + +__all__ = ["PerplexityResponsesConfig"] diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py new file mode 100644 index 00000000000..1dfee52d46e --- /dev/null +++ b/litellm/llms/perplexity/responses/transformation.py @@ -0,0 +1,509 @@ +""" +Transformation logic for Perplexity Agentic Research API (Responses API) + +This module handles the translation between OpenAI's Responses API format +and Perplexity's Responses API format, which supports: +- Third-party model access (OpenAI, Anthropic, Google, xAI, etc.) +- Presets for optimized configurations +- Web search and URL fetching tools +- Reasoning effort control +- Instructions parameter for system-level guidance +""" + +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.responses.main import DeleteResponseResult +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class PerplexityResponsesConfig(BaseResponsesAPIConfig): + """ + Configuration for Perplexity Agentic Research API (Responses API) + + Reference: https://docs.perplexity.ai/agentic-research/quickstart + """ + + @property + def custom_llm_provider(self) -> LlmProviders: + return "perplexity" + + def get_supported_openai_params(self, model: str) -> list: + """ + Perplexity Responses API supports a different set of parameters + + Ref: https://docs.perplexity.ai/api-reference/responses-post + """ + return [ + "max_output_tokens", + "stream", + "temperature", + "top_p", + "tools", + "reasoning", + "preset", + "instructions", + ] + + def validate_environment( + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """Validate environment and set up headers""" + # Get API key from environment + api_key = ( + get_secret_str("PERPLEXITYAI_API_KEY") + or get_secret_str("PERPLEXITY_API_KEY") + ) + + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + headers["Content-Type"] = "application/json" + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """Get the complete URL for the Perplexity Responses API""" + if api_base is None: + api_base = get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai" + + # Ensure api_base doesn't end with a slash + api_base = api_base.rstrip("/") + + # Add the responses endpoint + return f"{api_base}/v1/responses" + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """ + Map OpenAI Responses API parameters to Perplexity format + + Key differences: + - Supports 'preset' parameter for predefined configurations + - Supports 'instructions' parameter for system-level guidance + - Tools are specified differently (web_search, fetch_url) + """ + mapped_params = {} + + # Map standard parameters + if response_api_optional_params.get("max_output_tokens"): + mapped_params["max_output_tokens"] = response_api_optional_params["max_output_tokens"] + + if response_api_optional_params.get("temperature"): + mapped_params["temperature"] = response_api_optional_params["temperature"] + + if response_api_optional_params.get("top_p"): + mapped_params["top_p"] = response_api_optional_params["top_p"] + + if response_api_optional_params.get("stream"): + mapped_params["stream"] = response_api_optional_params["stream"] + + if response_api_optional_params.get("stream_options"): + mapped_params["stream_options"] = response_api_optional_params["stream_options"] + + # Map Perplexity-specific parameters + if response_api_optional_params.get("preset"): + mapped_params["preset"] = response_api_optional_params["preset"] + + if response_api_optional_params.get("instructions"): + mapped_params["instructions"] = response_api_optional_params["instructions"] + + if response_api_optional_params.get("reasoning"): + mapped_params["reasoning"] = response_api_optional_params["reasoning"] + + if response_api_optional_params.get("tools"): + mapped_params["tools"] = self._transform_tools(response_api_optional_params["tools"]) + + return mapped_params + + def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Transform tools to Perplexity format + + Perplexity supports: + - web_search: Performs web searches + - fetch_url: Fetches content from URLs + """ + perplexity_tools = [] + + for tool in tools: + if isinstance(tool, dict): + tool_type = tool.get("type") + + # Direct Perplexity tool format + if tool_type in ["web_search", "fetch_url"]: + perplexity_tools.append(tool) + + # OpenAI function format - try to map to Perplexity tools + elif tool_type == "function": + function = tool.get("function", {}) + function_name = function.get("name", "") + + if function_name == "web_search" or "search" in function_name.lower(): + perplexity_tools.append({"type": "web_search"}) + elif function_name == "fetch_url" or "fetch" in function_name.lower(): + perplexity_tools.append({"type": "fetch_url"}) + + return perplexity_tools + + def transform_responses_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Transform request to Perplexity Responses API format + """ + # Check if the model is a preset (format: preset/preset-name) + if model.startswith("preset/"): + preset_name = model.replace("preset/", "") + data = { + "preset": preset_name, + "input": self._format_input(input), + } + # Check if preset is explicitly provided in params + elif response_api_optional_request_params.get("preset"): + data = { + "preset": response_api_optional_request_params.pop("preset"), + "input": self._format_input(input), + } + else: + # Full request format for third-party models + data = { + "model": model, + "input": self._format_input(input), + } + + # Add all optional parameters + for key, value in response_api_optional_request_params.items(): + data[key] = value + + return data + + def _format_input(self, input: Union[str, ResponseInputParam]) -> Union[str, List[Dict[str, Any]]]: + """ + Format input for Perplexity Responses API + + The API accepts either: + - A simple string for single-turn queries + - An array of message objects for multi-turn conversations + """ + if isinstance(input, str): + return input + + # Handle ResponseInputParam format + if isinstance(input, list): + formatted_messages = [] + for item in input: + if isinstance(item, dict): + formatted_message = { + "type": "message", + "role": item.get("role"), + "content": item.get("content", ""), + } + formatted_messages.append(formatted_message) + return formatted_messages + + return str(input) + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Transform Perplexity Responses API response to OpenAI Responses API format + """ + try: + raw_response_json = raw_response.json() + except Exception as e: + raise BaseLLMException( + status_code=raw_response.status_code, + message=f"Failed to parse response: {str(e)}", + ) + + # Check for error status + status = raw_response_json.get("status") + if status == "failed": + error = raw_response_json.get("error", {}) + error_message = error.get("message", "Unknown error") + raise BaseLLMException( + status_code=raw_response.status_code, + message=error_message, + ) + + # Transform usage to handle Perplexity's cost structure + usage_data = raw_response_json.get("usage", {}) + transformed_usage = self._transform_usage(usage_data) + + # Map Perplexity response to OpenAI Responses API format + response = ResponsesAPIResponse( + id=raw_response_json.get("id", ""), + object="response", + created_at=raw_response_json.get("created_at", 0), + status=raw_response_json.get("status", "completed"), + model=raw_response_json.get("model", model), + output=raw_response_json.get("output", []), + usage=transformed_usage, + ) + + return response + + def _transform_usage(self, usage_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Transform Perplexity usage data to OpenAI format + + Perplexity returns: + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost": { + "currency": "USD", + "input_cost": 0.0001, + "output_cost": 0.0002, + "total_cost": 0.0003 + } + } + + OpenAI expects: + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300 + } + """ + transformed = { + "input_tokens": usage_data.get("input_tokens", 0), + "output_tokens": usage_data.get("output_tokens", 0), + "total_tokens": usage_data.get("total_tokens", 0), + } + + # Add input_tokens_details if present + if "input_tokens_details" in usage_data: + transformed["input_tokens_details"] = usage_data["input_tokens_details"] + + # Add output_tokens_details if present + if "output_tokens_details" in usage_data: + transformed["output_tokens_details"] = usage_data["output_tokens_details"] + + return transformed + + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse + """ + # Map Perplexity streaming chunk to OpenAI format + return ResponsesAPIStreamingResponse(**parsed_chunk) + + def transform_delete_response_api_request( + self, + response_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform delete response API request""" + # Perplexity may not support deleting responses + # Return appropriate URL and params + url = f"{api_base}/v1/responses/{response_id}" + return url, {} + + def transform_delete_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> DeleteResponseResult: + """Transform delete response API response""" + try: + response_json = raw_response.json() + return DeleteResponseResult( + id=response_json.get("id", ""), + object="response.deleted", + deleted=response_json.get("deleted", True), + ) + except Exception as e: + raise BaseLLMException( + status_code=raw_response.status_code, + message=f"Failed to parse delete response: {str(e)}", + ) + + def transform_get_response_api_request( + self, + response_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform get response API request""" + url = f"{api_base}/v1/responses/{response_id}" + return url, {} + + def transform_get_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """Transform get response API response""" + return self.transform_response_api_response( + model="", # Model will be in the response + raw_response=raw_response, + logging_obj=logging_obj, + ) + + def transform_list_input_items_request( + self, + response_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: Optional[str] = None, + before: Optional[str] = None, + include: Optional[List[str]] = None, + limit: int = 20, + order: Literal["asc", "desc"] = "desc", + ) -> Tuple[str, Dict]: + """Transform list input items request""" + url = f"{api_base}/v1/responses/{response_id}/input_items" + params = { + "limit": limit, + "order": order, + } + + if after: + params["after"] = after + if before: + params["before"] = before + if include: + params["include"] = include + + return url, params + + def transform_list_input_items_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Dict: + """Transform list input items response""" + try: + return raw_response.json() + except Exception as e: + raise BaseLLMException( + status_code=raw_response.status_code, + message=f"Failed to parse list input items response: {str(e)}", + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers] + ) -> BaseLLMException: + """Return appropriate error class based on status code""" + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + """Returns True if litellm should fake a stream for the given model and stream value""" + return False + + ######################################################### + ########## CANCEL RESPONSE API TRANSFORMATION ########## + ######################################################### + def transform_cancel_response_api_request( + self, + response_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform cancel response API request""" + # Perplexity may not support canceling responses + # Return appropriate URL and params + url = f"{api_base}/v1/responses/{response_id}/cancel" + return url, {} + + def transform_cancel_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """Transform cancel response API response""" + return self.transform_response_api_response( + model="", # Model will be in the response + raw_response=raw_response, + logging_obj=logging_obj, + ) + + ######################################################### + ########## COMPACT RESPONSE API TRANSFORMATION ########## + ######################################################### + def transform_compact_response_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform compact response API request""" + # Perplexity may not support compact responses + # Return standard URL and transformed request + url = f"{api_base}/v1/responses" + request_data = self.transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + return url, request_data + + def transform_compact_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """Transform compact response API response""" + return self.transform_response_api_response( + model="", # Model will be in the response + raw_response=raw_response, + logging_obj=logging_obj, + ) From be0ebb153e646a774df06bb3f8a2c2e7a922eabd Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:01:08 +0530 Subject: [PATCH 092/117] Add perplexity response api routing --- litellm/_lazy_imports_registry.py | 5 ++ litellm/utils.py | 2 + model_prices_and_context_window.json | 70 ++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index a01fe9c11db..791a129880a 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -274,6 +274,7 @@ LLM_CONFIG_NAMES = ( "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", + "PerplexityResponsesConfig", "AzureOpenAIO1Config", "IBMWatsonXAIConfig", "IBMWatsonXChatConfig", @@ -1033,6 +1034,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.perplexity.chat.transformation", "PerplexityChatConfig", ), + "PerplexityResponsesConfig": ( + ".llms.perplexity.responses.transformation", + "PerplexityResponsesConfig", + ), "AzureOpenAIO1Config": ( ".llms.azure.chat.o_series_transformation", "AzureOpenAIO1Config", diff --git a/litellm/utils.py b/litellm/utils.py index 6fdd2d88bca..ed0d6ee930d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8243,6 +8243,8 @@ class ProviderConfigManager: return litellm.VolcEngineResponsesAPIConfig() elif litellm.LlmProviders.MANUS == provider: return litellm.ManusResponsesAPIConfig() + elif litellm.LlmProviders.PERPLEXITY == provider: + return litellm.PerplexityResponsesConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 815d29c7964..6076d290739 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25614,6 +25614,76 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "perplexity/preset/pro-search": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_preset": true, + "preset_name": "pro-search" + }, + "perplexity/openai/gpt-4o": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "openai" + }, + "perplexity/openai/gpt-4o-mini": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "openai" + }, + "perplexity/openai/gpt-5.2": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": true, + "third_party_provider": "openai" + }, + "perplexity/anthropic/claude-3-5-sonnet-20241022": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "anthropic" + }, + "perplexity/anthropic/claude-3-5-haiku-20241022": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "anthropic" + }, + "perplexity/google/gemini-2.0-flash-exp": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "google" + }, + "perplexity/google/gemini-2.0-flash-thinking-exp": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": true, + "third_party_provider": "google" + }, + "perplexity/xai/grok-2-1212": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "xai" + }, + "perplexity/xai/grok-2-vision-1212": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "third_party_provider": "xai" + }, "publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": { "input_cost_per_token": 0.0, "litellm_provider": "publicai", From ac65524d9f5cfc353585f9be0152691a548e5447 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:37:08 +0530 Subject: [PATCH 093/117] Use openai base config --- .../perplexity/responses/transformation.py | 259 +++++------------- 1 file changed, 75 insertions(+), 184 deletions(-) diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 1dfee52d46e..27b78bac998 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -10,15 +10,14 @@ and Perplexity's Responses API format, which supports: - Instructions parameter for system-level guidance """ -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import httpx -import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( ResponseInputParam, @@ -26,14 +25,14 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamingResponse, ) -from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -class PerplexityResponsesConfig(BaseResponsesAPIConfig): +class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): """ Configuration for Perplexity Agentic Research API (Responses API) + Reference: https://docs.perplexity.ai/agentic-research/quickstart """ @@ -57,6 +56,7 @@ class PerplexityResponsesConfig(BaseResponsesAPIConfig): "reasoning", "preset", "instructions", + "models", # Model fallback support ] def validate_environment( @@ -296,7 +296,8 @@ class PerplexityResponsesConfig(BaseResponsesAPIConfig): { "input_tokens": 100, "output_tokens": 200, - "total_tokens": 300 + "total_tokens": 300, + "cost": 0.0003 } """ transformed = { @@ -305,6 +306,19 @@ class PerplexityResponsesConfig(BaseResponsesAPIConfig): "total_tokens": usage_data.get("total_tokens", 0), } + # Transform cost from Perplexity format (dict) to OpenAI format (float) + cost_obj = usage_data.get("cost") + if isinstance(cost_obj, dict) and "total_cost" in cost_obj: + transformed["cost"] = cost_obj["total_cost"] + verbose_logger.debug( + "Transformed Perplexity cost object to float: %s -> %s", + cost_obj, + cost_obj["total_cost"] + ) + elif cost_obj is not None: + # If cost is already a float/number, use it as-is + transformed["cost"] = cost_obj + # Add input_tokens_details if present if "input_tokens_details" in usage_data: transformed["input_tokens_details"] = usage_data["input_tokens_details"] @@ -324,186 +338,63 @@ class PerplexityResponsesConfig(BaseResponsesAPIConfig): """ Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse """ - # Map Perplexity streaming chunk to OpenAI format - return ResponsesAPIStreamingResponse(**parsed_chunk) - - def transform_delete_response_api_request( - self, - response_id: str, - api_base: str, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> Tuple[str, Dict]: - """Transform delete response API request""" - # Perplexity may not support deleting responses - # Return appropriate URL and params - url = f"{api_base}/v1/responses/{response_id}" - return url, {} - - def transform_delete_response_api_response( - self, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> DeleteResponseResult: - """Transform delete response API response""" - try: - response_json = raw_response.json() - return DeleteResponseResult( - id=response_json.get("id", ""), - object="response.deleted", - deleted=response_json.get("deleted", True), - ) - except Exception as e: - raise BaseLLMException( - status_code=raw_response.status_code, - message=f"Failed to parse delete response: {str(e)}", - ) - - def transform_get_response_api_request( - self, - response_id: str, - api_base: str, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> Tuple[str, Dict]: - """Transform get response API request""" - url = f"{api_base}/v1/responses/{response_id}" - return url, {} - - def transform_get_response_api_response( - self, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> ResponsesAPIResponse: - """Transform get response API response""" - return self.transform_response_api_response( - model="", # Model will be in the response - raw_response=raw_response, - logging_obj=logging_obj, + # Get the event type from the chunk + verbose_logger.debug("Raw Perplexity Chunk=%s", parsed_chunk) + event_type = str(parsed_chunk.get("type")) + event_pydantic_model = PerplexityResponsesConfig.get_event_model_class( + event_type=event_type ) - - def transform_list_input_items_request( - self, - response_id: str, - api_base: str, - litellm_params: GenericLiteLLMParams, - headers: dict, - after: Optional[str] = None, - before: Optional[str] = None, - include: Optional[List[str]] = None, - limit: int = 20, - order: Literal["asc", "desc"] = "desc", - ) -> Tuple[str, Dict]: - """Transform list input items request""" - url = f"{api_base}/v1/responses/{response_id}/input_items" - params = { - "limit": limit, - "order": order, - } - if after: - params["after"] = after - if before: - params["before"] = before - if include: - params["include"] = include + # Transform Perplexity-specific fields to OpenAI format + parsed_chunk = self._transform_perplexity_chunk(parsed_chunk) - return url, params - - def transform_list_input_items_response( - self, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> Dict: - """Transform list input items response""" + # Defensive: Handle error.code being null (similar to OpenAI implementation) try: - return raw_response.json() + error_obj = parsed_chunk.get("error") + if isinstance(error_obj, dict) and error_obj.get("code") is None: + # Preserve other fields, but ensure `code` is a non-null string + parsed_chunk = dict(parsed_chunk) + parsed_chunk["error"] = dict(error_obj) + parsed_chunk["error"]["code"] = "unknown_error" + except Exception: + # If anything unexpected happens here, fall back to attempting + # instantiation and let higher-level handlers manage errors. + verbose_logger.debug("Failed to coalesce error.code in parsed_chunk") + + return event_pydantic_model(**parsed_chunk) + + def _transform_perplexity_chunk(self, chunk: dict) -> dict: + """ + Transform Perplexity-specific fields in a streaming chunk to OpenAI format. + + This handles: + - Converting Perplexity's cost object to a simple float + """ + # Make a copy to avoid modifying the original + chunk = dict(chunk) + + # Transform usage.cost from Perplexity format to OpenAI format + # Perplexity: {"currency": "USD", "input_cost": 0.0001, "output_cost": 0.0002, "total_cost": 0.0003} + # OpenAI: 0.0003 (just the total_cost as a float) + try: + response_obj = chunk.get("response") + if isinstance(response_obj, dict): + usage_obj = response_obj.get("usage") + if isinstance(usage_obj, dict): + cost_obj = usage_obj.get("cost") + if isinstance(cost_obj, dict) and "total_cost" in cost_obj: + # Replace the cost object with just the total_cost value + chunk = dict(chunk) + chunk["response"] = dict(response_obj) + chunk["response"]["usage"] = dict(usage_obj) + chunk["response"]["usage"]["cost"] = cost_obj["total_cost"] + verbose_logger.debug( + "Transformed Perplexity cost object to float: %s -> %s", + cost_obj, + cost_obj["total_cost"] + ) except Exception as e: - raise BaseLLMException( - status_code=raw_response.status_code, - message=f"Failed to parse list input items response: {str(e)}", - ) - - def get_error_class( - self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers] - ) -> BaseLLMException: - """Return appropriate error class based on status code""" - return BaseLLMException( - status_code=status_code, - message=error_message, - headers=headers, - ) - - def should_fake_stream( - self, - model: Optional[str], - stream: Optional[bool], - custom_llm_provider: Optional[str] = None, - ) -> bool: - """Returns True if litellm should fake a stream for the given model and stream value""" - return False - - ######################################################### - ########## CANCEL RESPONSE API TRANSFORMATION ########## - ######################################################### - def transform_cancel_response_api_request( - self, - response_id: str, - api_base: str, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> Tuple[str, Dict]: - """Transform cancel response API request""" - # Perplexity may not support canceling responses - # Return appropriate URL and params - url = f"{api_base}/v1/responses/{response_id}/cancel" - return url, {} - - def transform_cancel_response_api_response( - self, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> ResponsesAPIResponse: - """Transform cancel response API response""" - return self.transform_response_api_response( - model="", # Model will be in the response - raw_response=raw_response, - logging_obj=logging_obj, - ) - - ######################################################### - ########## COMPACT RESPONSE API TRANSFORMATION ########## - ######################################################### - def transform_compact_response_api_request( - self, - model: str, - input: Union[str, ResponseInputParam], - response_api_optional_request_params: Dict, - api_base: str, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> Tuple[str, Dict]: - """Transform compact response API request""" - # Perplexity may not support compact responses - # Return standard URL and transformed request - url = f"{api_base}/v1/responses" - request_data = self.transform_responses_api_request( - model=model, - input=input, - response_api_optional_request_params=response_api_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ) - return url, request_data - - def transform_compact_response_api_response( - self, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> ResponsesAPIResponse: - """Transform compact response API response""" - return self.transform_response_api_response( - model="", # Model will be in the response - raw_response=raw_response, - logging_obj=logging_obj, - ) + # If transformation fails, log and continue with original chunk + verbose_logger.debug("Failed to transform Perplexity cost object: %s", e) + + return chunk From 9c1bf847298f8d836165cb6cfbf042a5dfbde167 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:41:03 +0530 Subject: [PATCH 094/117] Fix mypy issues --- .../perplexity/responses/transformation.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 27b78bac998..178e76ea970 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -20,6 +20,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( + ResponseAPIUsage, ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -39,7 +40,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: - return "perplexity" + return LlmProviders.PERPLEXITY def get_supported_openai_params(self, model: str) -> list: """ @@ -105,7 +106,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): - Supports 'instructions' parameter for system-level guidance - Tools are specified differently (web_search, fetch_url) """ - mapped_params = {} + mapped_params: Dict[str, Any] = {} # Map standard parameters if response_api_optional_params.get("max_output_tokens"): @@ -123,18 +124,23 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): if response_api_optional_params.get("stream_options"): mapped_params["stream_options"] = response_api_optional_params["stream_options"] - # Map Perplexity-specific parameters - if response_api_optional_params.get("preset"): - mapped_params["preset"] = response_api_optional_params["preset"] + # Map Perplexity-specific parameters (using .get() with Any dict access) + preset = response_api_optional_params.get("preset") # type: ignore + if preset: + mapped_params["preset"] = preset - if response_api_optional_params.get("instructions"): - mapped_params["instructions"] = response_api_optional_params["instructions"] + instructions = response_api_optional_params.get("instructions") # type: ignore + if instructions: + mapped_params["instructions"] = instructions if response_api_optional_params.get("reasoning"): mapped_params["reasoning"] = response_api_optional_params["reasoning"] - if response_api_optional_params.get("tools"): - mapped_params["tools"] = self._transform_tools(response_api_optional_params["tools"]) + tools = response_api_optional_params.get("tools") + if tools: + # Convert tools to list of dicts for transformation + tools_list = [dict(tool) if hasattr(tool, '__dict__') else tool for tool in tools] # type: ignore + mapped_params["tools"] = self._transform_tools(tools_list) # type: ignore return mapped_params @@ -260,7 +266,10 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): # Transform usage to handle Perplexity's cost structure usage_data = raw_response_json.get("usage", {}) - transformed_usage = self._transform_usage(usage_data) + transformed_usage_dict = self._transform_usage(usage_data) + + # Convert usage dict to ResponseAPIUsage object + usage_obj = ResponseAPIUsage(**transformed_usage_dict) if transformed_usage_dict else None # Map Perplexity response to OpenAI Responses API format response = ResponsesAPIResponse( @@ -270,7 +279,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): status=raw_response_json.get("status", "completed"), model=raw_response_json.get("model", model), output=raw_response_json.get("output", []), - usage=transformed_usage, + usage=usage_obj, ) return response From 2eb52db3e916e979f9466fad0fb3a55c199102df Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:44:00 +0530 Subject: [PATCH 095/117] Add documentation for perplexity --- docs/my-website/docs/providers/perplexity.md | 287 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 60 ++++ model_prices_and_context_window.json | 30 +- 3 files changed, 357 insertions(+), 20 deletions(-) diff --git a/docs/my-website/docs/providers/perplexity.md b/docs/my-website/docs/providers/perplexity.md index 2fcb49c60fa..68adf9939c6 100644 --- a/docs/my-website/docs/providers/perplexity.md +++ b/docs/my-website/docs/providers/perplexity.md @@ -120,6 +120,293 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported +## Agentic Research API (Responses API) + +Requires v1.72.6+ + + +### Using Presets + +Presets provide optimized defaults for specific use cases. Start with a preset for quick setup: + + + + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +# Using the pro-search preset +response = responses( + model="perplexity/preset/pro-search", + input="What are the latest developments in AI?", + custom_llm_provider="perplexity", +) + +print(response.output) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: perplexity-pro-search + litellm_params: + model: perplexity/preset/pro-search + api_key: os.environ/PERPLEXITY_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer anything" \ + -d '{ + "model": "perplexity-pro-search", + "input": "What are the latest developments in AI?" + }' +``` + + + + +### Using Third-Party Models + +Access models from OpenAI, Anthropic, Google, xAI, and other providers through Perplexity's unified API: + + + + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/openai/gpt-4o", + input="Explain quantum computing in simple terms", + custom_llm_provider="perplexity", + max_output_tokens=500, +) + +print(response.output) +``` + + + + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/anthropic/claude-3-5-sonnet-20241022", + input="Write a short story about a robot learning to paint", + custom_llm_provider="perplexity", + max_output_tokens=500, +) + +print(response.output) +``` + + + + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/google/gemini-2.0-flash-exp", + input="Explain the concept of neural networks", + custom_llm_provider="perplexity", + max_output_tokens=500, +) + +print(response.output) +``` + + + + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/xai/grok-2-1212", + input="What makes a good AI assistant?", + custom_llm_provider="perplexity", + max_output_tokens=500, +) + +print(response.output) +``` + + + + +### Web Search Tool + +Enable web search capabilities to access real-time information: + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/openai/gpt-4o", + input="What's the weather in San Francisco today?", + custom_llm_provider="perplexity", + tools=[{"type": "web_search"}], + instructions="You have access to a web_search tool. Use it for questions about current events.", +) + +print(response.output) +``` + + +### Reasoning Effort (Responses API) + +Control the reasoning effort level for reasoning-capable models: + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/openai/gpt-5.2", + input="Solve this complex problem step by step", + custom_llm_provider="perplexity", + reasoning={"effort": "high"}, # Options: low, medium, high + max_output_tokens=1000, +) + +print(response.output) +``` + +### Multi-Turn Conversations + +Use message arrays for multi-turn conversations with context: + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/anthropic/claude-3-5-sonnet-20241022", + input=[ + {"type": "message", "role": "system", "content": "You are a helpful assistant."}, + {"type": "message", "role": "user", "content": "What are the latest AI developments?"}, + ], + custom_llm_provider="perplexity", + instructions="Provide detailed, well-researched answers.", + max_output_tokens=800, +) + +print(response.output) +``` + +### Streaming Responses + +Stream responses for real-time output: + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +response = responses( + model="perplexity/openai/gpt-4o", + input="Tell me a story about space exploration", + custom_llm_provider="perplexity", + stream=True, + max_output_tokens=500, +) + +for chunk in response: + if hasattr(chunk, 'type'): + if chunk.type == "response.output_text.delta": + print(chunk.delta, end="", flush=True) +``` + +### Supported Third-Party Models + +| Provider | Model Name | Function Call | +|----------|------------|---------------| +| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` | +| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` | +| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` | +| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` | +| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` | +| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` | +| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` | +| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` | +| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` | + +### Available Presets + +| Preset Name | Function Call | +|----------------|--------------------------------------------------------| +| fast-search | `responses(model="perplexity/preset/fast-search", ...)`| +| pro-search | `responses(model="perplexity/preset/pro-search", ...)` | +| deep-research | `responses(model="perplexity/preset/deep-research", ...)`| + +### Complete Example + +```python +from litellm import responses +import os + +os.environ['PERPLEXITY_API_KEY'] = "" + +# Comprehensive example with multiple features +response = responses( + model="perplexity/openai/gpt-4o", + input="Research the latest developments in quantum computing and provide sources", + custom_llm_provider="perplexity", + tools=[ + {"type": "web_search"}, + {"type": "fetch_url"} + ], + instructions="Use web_search to find relevant information and fetch_url to retrieve detailed content from sources. Provide citations for all claims.", + max_output_tokens=1000, + temperature=0.7, +) + +print(f"Response ID: {response.id}") +print(f"Model: {response.model}") +print(f"Status: {response.status}") +print(f"Output: {response.output}") +print(f"Usage: {response.usage}") +``` + :::info For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 815d29c7964..9b45dfd893f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25614,6 +25614,66 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "perplexity/preset/pro-search": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_preset": true + }, + "perplexity/openai/gpt-4o": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/openai/gpt-4o-mini": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/openai/gpt-5.2": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": true + }, + "perplexity/anthropic/claude-3-5-sonnet-20241022": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/anthropic/claude-3-5-haiku-20241022": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/google/gemini-2.0-flash-exp": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/google/gemini-2.0-flash-thinking-exp": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": true + }, + "perplexity/xai/grok-2-1212": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, + "perplexity/xai/grok-2-vision-1212": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false + }, "publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": { "input_cost_per_token": 0.0, "litellm_provider": "publicai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6076d290739..9b45dfd893f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25618,71 +25618,61 @@ "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_preset": true, - "preset_name": "pro-search" + "supports_preset": true }, "perplexity/openai/gpt-4o": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "openai" + "supports_reasoning": false }, "perplexity/openai/gpt-4o-mini": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "openai" + "supports_reasoning": false }, "perplexity/openai/gpt-5.2": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": true, - "third_party_provider": "openai" + "supports_reasoning": true }, "perplexity/anthropic/claude-3-5-sonnet-20241022": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "anthropic" + "supports_reasoning": false }, "perplexity/anthropic/claude-3-5-haiku-20241022": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "anthropic" + "supports_reasoning": false }, "perplexity/google/gemini-2.0-flash-exp": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "google" + "supports_reasoning": false }, "perplexity/google/gemini-2.0-flash-thinking-exp": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": true, - "third_party_provider": "google" + "supports_reasoning": true }, "perplexity/xai/grok-2-1212": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "xai" + "supports_reasoning": false }, "perplexity/xai/grok-2-vision-1212": { "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, - "supports_reasoning": false, - "third_party_provider": "xai" + "supports_reasoning": false }, "publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": { "input_cost_per_token": 0.0, From f4ae6ed1810f9e2bc63c1df85b6d932ff990e5e3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:48:41 +0530 Subject: [PATCH 096/117] Fix mypy issues --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index fbb6f380a54..7274b389a92 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -482,7 +482,7 @@ def generic_response_convertor( ) # Build extra_fields dict from GENERIC_USER_EXTRA_ATTRIBUTES if specified - extra_fields = None + extra_fields: Optional[Dict[str, Any]] = None if generic_user_extra_attributes: extra_fields = {} for attr_name in generic_user_extra_attributes.split(","): From 19628f6187e1af18ab3aff3872ba36e46cbe350d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:52:17 +0530 Subject: [PATCH 097/117] Fix mypy issues --- litellm/llms/databricks/chat/transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2699b0618bd..7c2a9569c58 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -60,7 +60,7 @@ from ...anthropic.chat.transformation import AnthropicConfig from ...openai_like.chat.transformation import OpenAILikeChatConfig from ..common_utils import DatabricksBase, DatabricksException -def _sanitize_empty_content(message_dict: dict) -> None: +def _sanitize_empty_content(message_dict: dict[str, Any]) -> None: """ Remove or filter content so empty text blocks are not sent. Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks. @@ -382,7 +382,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): # Move message-level cache_control into a content block when content is a string. if "cache_control" in _message and isinstance(_message.get("content"), str): _message = self._move_cache_control_into_string_content_block(_message) - _sanitize_empty_content(_message) + _sanitize_empty_content(cast(dict[str, Any], _message)) new_messages.append(_message) if is_async: From 7d5141c28c3eed3f35cbf56449d636e864d5ea83 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 17:58:15 +0530 Subject: [PATCH 098/117] Fix mypy issues --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8174b9d2655..dacb928e8a7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1393,6 +1393,7 @@ if TYPE_CHECKING: from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig + from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 791a129880a..051b957ed1d 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -226,6 +226,7 @@ LLM_CONFIG_NAMES = ( "XAIResponsesAPIConfig", "LiteLLMProxyResponsesAPIConfig", "VolcEngineResponsesAPIConfig", + "PerplexityResponsesConfig", "GoogleAIStudioInteractionsConfig", "OpenAIOSeriesConfig", "AnthropicSkillsConfig", @@ -902,6 +903,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.manus.responses.transformation", "ManusResponsesAPIConfig", ), + "PerplexityResponsesConfig": ( + ".llms.perplexity.responses.transformation", + "PerplexityResponsesConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", From 63eedc26b0efb795f45e3d36c444303bc628b1dc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 18:08:23 +0530 Subject: [PATCH 099/117] Fix test_async_post_call_success_hook_for_unified_finetuning_job --- .../litellm_enterprise/proxy/hooks/test_managed_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index ff957ed395b..58efa854e7c 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -192,7 +192,7 @@ async def test_async_post_call_success_hook_for_unified_finetuning_job(): "model_id": "gpt-3.5-turbo-0613", } proxy_managed_files = _PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() + DualCache(), prisma_client=AsyncMock() ) data = { "user_api_key_dict": {"parent_otel_span": MagicMock()}, From 7166ca8c5983ff712f1f931b3aec9ed9255772c6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 18:10:37 +0530 Subject: [PATCH 100/117] Add moonshotai.kimi-k2.5 --- litellm/model_prices_and_context_window_backup.json | 11 +++++++++++ model_prices_and_context_window.json | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a27aa59376f..3d5f9bc01e5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6102,6 +6102,17 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a27aa59376f..3d5f9bc01e5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6102,6 +6102,17 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", From f6228fda3ee75508d6ecf7cce4db3b8e7b8bb075 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Feb 2026 18:18:41 +0530 Subject: [PATCH 101/117] Fix mypy issues --- litellm/llms/vertex_ai/gemini/transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 1d55629f079..00b461dcda0 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -445,13 +445,13 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 if isinstance(image_item, dict): image_url_obj = image_item.get("image_url") if isinstance(image_url_obj, dict): - image_url = image_url_obj.get("url") + assistant_image_url = image_url_obj.get("url") format = image_url_obj.get("format") detail = image_url_obj.get("detail") media_resolution_enum = _convert_detail_to_media_resolution_enum(detail) - if image_url: + if assistant_image_url: _part = _process_gemini_media( - image_url=image_url, + image_url=assistant_image_url, format=format, media_resolution_enum=media_resolution_enum, model=model, From 1afe3032fdda19eb96e8a36e61cc87f9f84ba4ed Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Tue, 10 Feb 2026 18:33:16 +0100 Subject: [PATCH 102/117] fix(otel): auto-infer otlp_http exporter when endpoint is configured (#20438) When OpenTelemetry is configured via the UI, only OTEL_ENDPOINT and OTEL_HEADERS are set, but OTEL_EXPORTER is not specified. This caused the exporter to default to "console", meaning traces were printed to stdout instead of being sent to the configured endpoint. This fix adds logic in OpenTelemetryConfig.__post_init__ to automatically infer "otlp_http" as the exporter when an endpoint is specified but the exporter is still the default "console". Fixes issue reported by Elastic team where traces weren't being sent to their OTEL endpoint when configured through the LiteLLM UI. --- litellm/integrations/opentelemetry.py | 7 +++++++ .../integrations/test_opentelemetry.py | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 138d508db4b..b847180174a 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -72,6 +72,13 @@ class OpenTelemetryConfig: model_id: Optional[str] = None def __post_init__(self) -> None: + # If endpoint is specified but exporter is still the default "console", + # automatically infer "otlp_http" to send traces to the endpoint. + # This fixes an issue where UI-configured OTEL settings would default + # to console output instead of sending traces to the configured endpoint. + if self.endpoint and isinstance(self.exporter, str) and self.exporter == "console": + self.exporter = "otlp_http" + if not self.service_name: self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") if not self.deployment_environment: diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 95fa6ed8f60..3da9d9857c9 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -274,6 +274,26 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(config.deployment_environment, "production") self.assertEqual(config.model_id, "custom-service") + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_auto_infer_otlp_http_when_endpoint_set(self): + """When endpoint is set but exporter is default 'console', auto-infer 'otlp_http'. + + This fixes an issue where UI-configured OTEL settings would default to console + output instead of sending traces to the configured endpoint. + See: https://github.com/BerriAI/litellm/issues/XXXX + """ + # When endpoint is specified without explicit exporter, should auto-infer otlp_http + config = OpenTelemetryConfig(endpoint="https://otel-collector.example.com:443") + self.assertEqual(config.exporter, "otlp_http") + + # When exporter is explicitly set to something other than console, should not override + config_grpc = OpenTelemetryConfig(exporter="grpc", endpoint="https://otel-collector.example.com:443") + self.assertEqual(config_grpc.exporter, "grpc") + + # When no endpoint is set, should keep console as default + config_no_endpoint = OpenTelemetryConfig() + self.assertEqual(config_no_endpoint.exporter, "console") + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): """Poll until we see at least one span with an attribute key starting with `prefix`.""" deadline = time.time() + self.POLL_TIMEOUT From 3bbc25a3f006225f96976b6aef421da84e34611d Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Tue, 10 Feb 2026 19:17:35 +0100 Subject: [PATCH 103/117] fix(aiohttp): respect ssl_verify with shared sessions (#20349) * fix(aiohttp): respect ssl_verify with shared sessions * fix(aiohttp): resolve mypy error for ssl parameter type Pass ssl kwarg conditionally to aiohttp request() only when explicitly configured, since None is not a valid value for the ssl parameter (expected SSLContext | bool | Fingerprint). --- .../llms/custom_httpx/aiohttp_transport.py | 23 +++++- litellm/llms/custom_httpx/http_handler.py | 16 +++- .../llms/custom_httpx/test_http_handler.py | 77 +++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index a7b83d8c802..1b03ec47643 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -1,6 +1,7 @@ import asyncio import contextlib import os +import ssl import typing import urllib.request from typing import Callable, Dict, Optional, Union @@ -139,8 +140,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport): Credit to: https://github.com/karpetrosyan/httpx-aiohttp for this implementation """ - def __init__(self, client: Union[ClientSession, Callable[[], ClientSession]]): + def __init__( + self, + client: Union[ClientSession, Callable[[], ClientSession]], + ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None, + ): self.client = client + self._ssl_verify = ssl_verify # Store for per-request SSL override super().__init__(client=client) # Store the client factory for recreating sessions when needed if callable(client): @@ -214,6 +220,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): timeout: dict, proxy: Optional[str], sni_hostname: Optional[str], + ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None, ) -> ClientResponse: """ Helper function to make an aiohttp request with the given parameters. @@ -224,6 +231,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): timeout: Timeout settings dict with 'connect', 'read', 'pool' keys proxy: Optional proxy URL sni_hostname: Optional SNI hostname for SSL + ssl_verify: Optional SSL verification setting (False to disable, SSLContext for custom) Returns: ClientResponse from aiohttp @@ -237,6 +245,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport): data = request.stream # type: ignore request.headers.pop("transfer-encoding", None) # handled by aiohttp + # Only pass ssl kwarg when explicitly configured, to avoid + # overriding the session/connector defaults with None (which is + # not a valid value for aiohttp's ssl parameter). + ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {} + if ssl_verify is not None: + ssl_kwargs["ssl"] = ssl_verify + response = await client_session.request( method=request.method, url=YarlURL(str(request.url), encoded=True), @@ -251,6 +266,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): ), proxy=proxy, server_hostname=sni_hostname, + **ssl_kwargs, ).__aenter__() return response @@ -268,6 +284,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Resolve proxy settings from environment variables proxy = await self._get_proxy_settings(request) + # Use stored SSL configuration for per-request override + ssl_config = self._ssl_verify + try: with map_aiohttp_exceptions(): response = await self._make_aiohttp_request( @@ -276,6 +295,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): timeout=timeout, proxy=proxy, sni_hostname=sni_hostname, + ssl_verify=ssl_config, ) except RuntimeError as e: # Handle the case where session was closed between our check and actual use @@ -296,6 +316,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): timeout=timeout, proxy=proxy, sni_hostname=sni_hostname, + ssl_verify=ssl_config, ) else: # Re-raise if it's a different RuntimeError diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index ac9dd5998e2..95f411c397c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -846,6 +846,16 @@ class AsyncHTTPHandler: if str_to_bool(os.getenv("AIOHTTP_TRUST_ENV", "False")) is True: trust_env = True + ######################################################### + # Determine SSL config to pass to transport for per-request override + # This ensures ssl_verify works even with shared sessions + ######################################################### + ssl_for_transport: Optional[Union[bool, ssl.SSLContext]] = None + if ssl_context is not None: + ssl_for_transport = ssl_context + elif ssl_verify is False: + ssl_for_transport = False + verbose_logger.debug("Creating AiohttpTransport...") # Use shared session if provided and valid @@ -853,7 +863,10 @@ class AsyncHTTPHandler: verbose_logger.debug( f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})" ) - return LiteLLMAiohttpTransport(client=shared_session) + return LiteLLMAiohttpTransport( + client=shared_session, + ssl_verify=ssl_for_transport, + ) # Create new session only if none provided or existing one is invalid verbose_logger.debug( @@ -877,6 +890,7 @@ class AsyncHTTPHandler: connector=TCPConnector(**transport_connector_kwargs), trust_env=trust_env, ), + ssl_verify=ssl_for_transport, ) @staticmethod diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 65f08ef5021..c249bd9970c 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -140,6 +140,83 @@ async def test_ssl_verification_with_aiohttp_transport(): litellm.disable_aiohttp_transport = original_disable +@pytest.mark.asyncio +async def test_ssl_verification_with_shared_session(): + """ + Test that ssl_verify=False is respected even with shared sessions. + + This was a bug where shared sessions bypassed SSL configuration because + _create_aiohttp_transport returned immediately without passing ssl_verify + to the LiteLLMAiohttpTransport constructor. + + The fix stores ssl_verify in the transport and passes it per-request. + """ + import aiohttp + + # Ensure aiohttp transport is enabled for this test + original_disable = litellm.disable_aiohttp_transport + litellm.disable_aiohttp_transport = False + + try: + # Create a shared session (simulating what happens in production) + shared_session = aiohttp.ClientSession() + + try: + # Create transport with shared session and ssl_verify=False + transport = AsyncHTTPHandler._create_aiohttp_transport( + ssl_verify=False, + shared_session=shared_session, + ) + + # Verify the transport uses the shared session + assert transport.client is shared_session + + # Verify the SSL setting is stored in the transport for per-request use + assert transport._ssl_verify is False + finally: + await shared_session.close() + finally: + # Restore original setting + litellm.disable_aiohttp_transport = original_disable + + +@pytest.mark.asyncio +async def test_ssl_context_with_shared_session(): + """ + Test that ssl_context is respected even with shared sessions. + """ + import aiohttp + + # Ensure aiohttp transport is enabled for this test + original_disable = litellm.disable_aiohttp_transport + litellm.disable_aiohttp_transport = False + + try: + # Create a custom SSL context + custom_ssl_context = ssl.create_default_context() + + # Create a shared session + shared_session = aiohttp.ClientSession() + + try: + # Create transport with shared session and custom ssl_context + transport = AsyncHTTPHandler._create_aiohttp_transport( + ssl_context=custom_ssl_context, + shared_session=shared_session, + ) + + # Verify the transport uses the shared session + assert transport.client is shared_session + + # Verify the SSL context is stored in the transport for per-request use + assert transport._ssl_verify is custom_ssl_context + finally: + await shared_session.close() + finally: + # Restore original setting + litellm.disable_aiohttp_transport = original_disable + + @pytest.mark.asyncio async def test_aiohttp_transport_trust_env_setting(monkeypatch): """Test that trust_env setting is properly configured in aiohttp transport""" From 7fd8c0e16045302f0b9982910ecf55fb7451924d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 12:44:38 -0800 Subject: [PATCH 104/117] Searchable Paginated Model Select For Spend Logs --- .../spend_management_endpoints.py | 6 + .../test_spend_management_endpoints.py | 79 +++++ .../hooks/models/useModels.test.ts | 242 +++++++++++++- .../app/(dashboard)/hooks/models/useModels.ts | 38 ++- .../PaginatedModelSelect.test.tsx | 301 ++++++++++++++++++ .../PaginatedModelSelect.tsx | 143 +++++++++ .../src/components/molecules/filter.tsx | 22 +- .../src/components/networking.tsx | 2 + .../src/components/view_logs/index.tsx | 12 +- .../components/view_logs/log_filter_logic.tsx | 1 + 10 files changed, 824 insertions(+), 22 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 6e49e4244e7..c05eda85c81 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1672,6 +1672,9 @@ async def ui_view_spend_logs( # noqa: PLR0915 model: Optional[str] = fastapi.Query( default=None, description="Filter logs by model" ), + model_id: Optional[str] = fastapi.Query( + default=None, description="Filter logs by model ID (litellm model deployment id)" + ), key_alias: Optional[str] = fastapi.Query( default=None, description="Filter logs by key alias" ), @@ -1763,6 +1766,9 @@ async def ui_view_spend_logs( # noqa: PLR0915 if model is not None: where_conditions["model"] = model + if model_id is not None: + where_conditions["model_id"] = model_id + # Build metadata filters metadata_filters = [] if key_alias is not None: diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 54a276bc97f..dd34cb47da4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1025,6 +1025,85 @@ async def test_ui_view_spend_logs_with_model(client, monkeypatch): assert data["data"][0]["model"] == "gpt-3.5-turbo" +@pytest.mark.asyncio +async def test_ui_view_spend_logs_with_model_id(client, monkeypatch): + """Test that the model_id query param filters spend logs by litellm model deployment id.""" + mock_spend_logs = [ + { + "id": "log1", + "request_id": "req1", + "api_key": "sk-test-key", + "user": "test_user_1", + "team_id": "team1", + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-3.5-turbo", + "model_id": "deployment-id-1", + "status": "success", + }, + { + "id": "log2", + "request_id": "req2", + "api_key": "sk-test-key", + "user": "test_user_2", + "team_id": "team1", + "spend": 0.10, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + "model_id": "deployment-id-2", + "status": "success", + }, + ] + + class MockDB: + async def find_many(self, *args, **kwargs): + if ( + "where" in kwargs + and "model_id" in kwargs["where"] + and kwargs["where"]["model_id"] == "deployment-id-1" + ): + return [mock_spend_logs[0]] + return mock_spend_logs + + async def count(self, *args, **kwargs): + if ( + "where" in kwargs + and "model_id" in kwargs["where"] + and kwargs["where"]["model_id"] == "deployment-id-1" + ): + return 1 + return len(mock_spend_logs) + + class MockPrismaClient: + def __init__(self): + self.db = MockDB() + self.db.litellm_spendlogs = self.db + + mock_prisma_client = MockPrismaClient() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + start_date = ( + datetime.datetime.now(timezone.utc) - datetime.timedelta(days=7) + ).strftime("%Y-%m-%d %H:%M:%S") + end_date = datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + response = client.get( + "/spend/logs/ui", + params={ + "model_id": "deployment-id-1", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert len(data["data"]) == 1 + assert data["data"][0]["model_id"] == "deployment-id-1" + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_key_hash(client, monkeypatch): # Mock data for the test diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts index 4985206092f..2539cc63f95 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts @@ -3,13 +3,14 @@ import { renderHook, waitFor } from "@testing-library/react"; import React, { ReactNode } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { - useModelsInfo, - useModelHub, useAllProxyModels, + useInfiniteModelInfo, + useModelHub, + useModelsInfo, useSelectedTeamModels, - type ProxyModel, type AllProxyModelsResponse, type PaginatedModelInfoResponse, + type ProxyModel, } from "./useModels"; vi.mock("@/components/networking", () => ({ @@ -23,7 +24,7 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockUseAuthorized(), })); -import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; +import { modelAvailableCall, modelHubCall, modelInfoCall } from "@/components/networking"; const mockProxyModel: ProxyModel = { id: "model-1", @@ -106,7 +107,7 @@ describe("useModelsInfo", () => { undefined, undefined, undefined, - undefined + undefined, ); expect(modelInfoCall).toHaveBeenCalledTimes(1); }); @@ -130,7 +131,7 @@ describe("useModelsInfo", () => { undefined, undefined, undefined, - undefined + undefined, ); }); @@ -393,7 +394,7 @@ describe("useAllProxyModels", () => { null, true, false, - "expand" + "expand", ); expect(modelAvailableCall).toHaveBeenCalledTimes(1); }); @@ -531,13 +532,7 @@ describe("useSelectedTeamModels", () => { expect(result.current.data).toEqual(mockAllProxyModelsResponse); expect(result.current.error).toBeNull(); - expect(modelAvailableCall).toHaveBeenCalledWith( - "test-access-token", - "test-user-id", - "Admin", - true, - "team-1" - ); + expect(modelAvailableCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", true, "team-1"); expect(modelAvailableCall).toHaveBeenCalledTimes(1); }); @@ -639,3 +634,222 @@ describe("useSelectedTeamModels", () => { expect(modelAvailableCall).not.toHaveBeenCalled(); }); }); + +describe("useInfiniteModelInfo", () => { + let queryClient: QueryClient; + + const mockPageOneResponse: PaginatedModelInfoResponse = { + data: [{ model_name: "gpt-4", model_info: { id: "model-1" } }], + total_count: 2, + current_page: 1, + total_pages: 2, + size: 50, + }; + + const mockPageTwoResponse: PaginatedModelInfoResponse = { + data: [{ model_name: "claude-3", model_info: { id: "model-2" } }], + total_count: 2, + current_page: 2, + total_pages: 2, + size: 50, + }; + + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); + + vi.clearAllMocks(); + + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + }); + + const wrapper = ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + + it("should return defined result", () => { + (modelInfoCall as any).mockResolvedValue(mockPageOneResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current).toBeDefined(); + expect(result.current).toHaveProperty("data"); + expect(result.current).toHaveProperty("fetchNextPage"); + expect(result.current).toHaveProperty("hasNextPage"); + expect(result.current).toHaveProperty("isFetchingNextPage"); + expect(result.current).toHaveProperty("isLoading"); + }); + + it("should return paginated data and call modelInfoCall with page 1 initially", async () => { + (modelInfoCall as any).mockResolvedValue(mockPageOneResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current.isLoading).toBe(true); + + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data?.pages).toHaveLength(1); + expect(result.current.data?.pages[0]).toEqual(mockPageOneResponse); + expect(result.current.hasNextPage).toBe(true); + expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 1, 50, undefined); + expect(modelInfoCall).toHaveBeenCalledTimes(1); + }); + + it("should use custom size parameter", async () => { + (modelInfoCall as any).mockResolvedValue(mockPageOneResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(25), { wrapper }); + + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + }); + + expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 1, 25, undefined); + }); + + it("should pass search parameter to modelInfoCall", async () => { + (modelInfoCall as any).mockResolvedValue(mockPageOneResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(50, "gpt"), { wrapper }); + + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + }); + + expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 1, 50, "gpt"); + }); + + it("should fetch next page when fetchNextPage is called", async () => { + (modelInfoCall as any).mockResolvedValueOnce(mockPageOneResponse).mockResolvedValueOnce(mockPageTwoResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + expect(result.current.hasNextPage).toBe(true); + }); + + await result.current.fetchNextPage(); + + await waitFor(() => { + expect(result.current.data?.pages).toHaveLength(2); + expect(result.current.data?.pages[1]).toEqual(mockPageTwoResponse); + expect(result.current.hasNextPage).toBe(false); + }); + + expect(modelInfoCall).toHaveBeenNthCalledWith(2, "test-access-token", "test-user-id", "Admin", 2, 50, undefined); + }); + + it("should return undefined for hasNextPage when on last page", async () => { + const lastPageResponse: PaginatedModelInfoResponse = { + ...mockPageOneResponse, + current_page: 1, + total_pages: 1, + }; + (modelInfoCall as any).mockResolvedValue(lastPageResponse); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.hasNextPage).toBe(false); + }); + + it("should handle error when modelInfoCall fails", async () => { + const errorMessage = "Failed to fetch models"; + const testError = new Error(errorMessage); + (modelInfoCall as any).mockRejectedValue(testError); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current.isLoading).toBe(true); + + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toEqual(testError); + expect(result.current.data).toBeUndefined(); + expect(modelInfoCall).toHaveBeenCalledTimes(1); + }); + + it("should not execute query when accessToken is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: null, + userId: "test-user-id", + userRole: "Admin", + token: null, + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(modelInfoCall).not.toHaveBeenCalled(); + }); + + it("should not execute query when userId is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: null, + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(modelInfoCall).not.toHaveBeenCalled(); + }); + + it("should not execute query when userRole is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: null, + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useInfiniteModelInfo(), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(modelInfoCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index c57de675e0e..fe1afdcc39f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -1,4 +1,4 @@ -import { useQuery } from "@tanstack/react-query"; +import { useQuery, useInfiniteQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "../useAuthorized"; @@ -26,6 +26,7 @@ const modelKeys = createQueryKeys("models"); const modelHubKeys = createQueryKeys("modelHub"); const allProxyModelsKeys = createQueryKeys("allProxyModels"); const selectedTeamModelsKeys = createQueryKeys("selectedTeamModels"); +const infiniteModelKeys = createQueryKeys("infiniteModels"); export const useModelsInfo = (page: number = 1, size: number = 50, search?: string, modelId?: string, teamId?: string, sortBy?: string, sortOrder?: string) => { const { accessToken, userId, userRole } = useAuthorized(); @@ -74,3 +75,38 @@ export const useSelectedTeamModels = (teamID: string | null) => { enabled: Boolean(accessToken && userId && userRole && teamID), }); }; + +export const useInfiniteModelInfo = ( + size: number = 50, + search?: string, +) => { + const { accessToken, userId, userRole } = useAuthorized(); + return useInfiniteQuery({ + queryKey: infiniteModelKeys.list({ + filters: { + ...(userId && { userId }), + ...(userRole && { userRole }), + size, + ...(search && { search }), + }, + }), + queryFn: async ({ pageParam }) => { + return await modelInfoCall( + accessToken!, + userId!, + userRole!, + pageParam as number, + size, + search, + ); + }, + initialPageParam: 1, + getNextPageParam: (lastPage) => { + if (lastPage.current_page < lastPage.total_pages) { + return lastPage.current_page + 1; + } + return undefined; + }, + enabled: Boolean(accessToken && userId && userRole), + }); +}; diff --git a/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.test.tsx b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.test.tsx new file mode 100644 index 00000000000..b91f97885e6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.test.tsx @@ -0,0 +1,301 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../../../tests/test-utils"; +import { PaginatedModelSelect } from "./PaginatedModelSelect"; + +const mockFetchNextPage = vi.fn(); + +vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ + useInfiniteModelInfo: vi.fn(), +})); + +vi.mock("@tanstack/react-pacer/debouncer", () => { + const React = require("react"); + return { + useDebouncedState: (initial: string) => { + const [value, setValue] = React.useState(initial); + return [value, setValue]; + }, + }; +}); + +import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; + +const mockUseInfiniteModelInfo = vi.mocked(useInfiniteModelInfo); + +const mockPagesWithModels = { + pages: [ + { + data: [ + { model_name: "GPT-4", model_info: { id: "model-1" } }, + { model_name: "Claude-3", model_info: { id: "model-2" } }, + ], + total_count: 2, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], +}; + +const mockEmptyPages = { + pages: [{ data: [], total_count: 0, current_page: 1, total_pages: 1, size: 50 }], +}; + +describe("PaginatedModelSelect", () => { + const mockOnChange = vi.fn(); + + const defaultHookReturn = { + data: mockPagesWithModels, + fetchNextPage: mockFetchNextPage, + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockUseInfiniteModelInfo.mockReturnValue(defaultHookReturn as any); + }); + + it("should render", () => { + renderWithProviders(); + + expect(screen.getByRole("combobox")).toBeInTheDocument(); + expect(screen.getByText("Select a model")).toBeInTheDocument(); + }); + + it("should display custom placeholder when provided", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("Choose model")).toBeInTheDocument(); + }); + + it("should display model options when data is loaded", async () => { + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "GPT-4 (model-1)" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Claude-3 (model-2)" })).toBeInTheDocument(); + }); + }); + + it("should call onChange when user selects a model", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + + const visibleOption = await screen.findByTitle("GPT-4 (model-1)"); + await user.click(visibleOption); + + await waitFor(() => { + expect(mockOnChange).toHaveBeenCalledWith("model-1"); + }); + }); + + it("should display selected value when value prop is provided", async () => { + renderWithProviders( + , + ); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "GPT-4 (model-1)" })).toBeInTheDocument(); + }); + }); + + it("should show loading state when isLoading is true", () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + isLoading: true, + } as any); + + renderWithProviders(); + + expect(screen.getByRole("combobox")).toHaveAttribute("aria-expanded", "false"); + }); + + it("should pass pageSize to useInfiniteModelInfo", () => { + renderWithProviders(); + + expect(mockUseInfiniteModelInfo).toHaveBeenCalledWith(25, undefined); + }); + + it("should pass search to useInfiniteModelInfo when user types", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + await user.keyboard("gpt"); + + await waitFor(() => { + expect(mockUseInfiniteModelInfo).toHaveBeenCalledWith(50, "gpt"); + }); + }); + + it("should have scroll container for infinite loading when hasNextPage is true", async () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + hasNextPage: true, + isFetchingNextPage: false, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "GPT-4 (model-1)" })).toBeInTheDocument(); + }); + + const scrollableContainer = document.querySelector( + ".ant-select-dropdown .rc-virtual-list-holder", + ); + expect(scrollableContainer).toBeInTheDocument(); + expect(scrollableContainer).toHaveAttribute("style"); + }); + + it("should deduplicate models with same id across pages", async () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + data: { + pages: [ + { + data: [ + { model_name: "GPT-4", model_info: { id: "model-1" } }, + { model_name: "GPT-4 Dupe", model_info: { id: "model-1" } }, + ], + total_count: 2, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], + }, + fetchNextPage: mockFetchNextPage, + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + const model1Options = screen.queryAllByRole("option", { name: /model-1/ }); + expect(model1Options.length).toBe(1); + }); + }); + + it("should skip models without model_info id", async () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + data: { + pages: [ + { + data: [ + { model_name: "Valid Model", model_info: { id: "valid-id" } }, + { model_name: "No ID", model_info: null }, + { model_name: "Empty ID", model_info: { id: "" } }, + ], + total_count: 3, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], + }, + fetchNextPage: mockFetchNextPage, + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "Valid Model (valid-id)" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "No ID" })).not.toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Empty ID" })).not.toBeInTheDocument(); + }); + }); + + it("should show model ID only when model_name is empty", async () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + data: { + pages: [ + { + data: [{ model_name: "", model_info: { id: "id-only" } }], + total_count: 1, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], + }, + fetchNextPage: mockFetchNextPage, + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "id-only" })).toBeInTheDocument(); + }); + }); + + it("should respect allowClear prop", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("combobox")).toBeInTheDocument(); + }); + + it("should respect disabled prop", () => { + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + expect(combobox.closest(".ant-select")).toHaveClass("ant-select-disabled"); + }); + + it("should not call fetchNextPage when hasNextPage is false", async () => { + mockUseInfiniteModelInfo.mockReturnValue({ + ...defaultHookReturn, + hasNextPage: false, + } as any); + + renderWithProviders(); + + await userEvent.click(screen.getByRole("combobox")); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "GPT-4 (model-1)" })).toBeInTheDocument(); + }); + + expect(mockFetchNextPage).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx new file mode 100644 index 00000000000..285f4e94d68 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx @@ -0,0 +1,143 @@ +import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; +import { LoadingOutlined } from "@ant-design/icons"; +import { useDebouncedState } from "@tanstack/react-pacer/debouncer"; +import { Select, Space, Typography } from "antd"; +import { useMemo, useState, type UIEvent } from "react"; + +const { Text } = Typography; + +export interface PaginatedModelSelectProps { + value?: string; + onChange?: (value: string) => void; + placeholder?: string; + style?: React.CSSProperties; + pageSize?: number; + allowClear?: boolean; + disabled?: boolean; +} + +const SCROLL_THRESHOLD = 0.8; +const DEBOUNCE_MS = 300; + +export const PaginatedModelSelect = ({ + value, + onChange, + placeholder = "Select a model", + style, + pageSize = 50, + allowClear = true, + disabled = false, +}: PaginatedModelSelectProps) => { + const [searchInput, setSearchInput] = useState(""); + const [debouncedSearch, setDebouncedSearch] = useDebouncedState("", { + wait: DEBOUNCE_MS, + }); + + const { + data, + fetchNextPage, + hasNextPage, + isFetchingNextPage, + isLoading, + } = useInfiniteModelInfo(pageSize, debouncedSearch || undefined); + + const options = useMemo(() => { + if (!data?.pages) return []; + + const seen = new Set(); + const result: { label: string; value: string; modelName: string; modelId: string }[] = []; + + for (const page of data.pages) { + for (const model of page.data) { + const modelId = model.model_info?.id ?? ""; + const modelName = model.model_name ?? ""; + + // Dedupe by id - skip models without id (can't uniquely identify) + if (!modelId || seen.has(modelId)) continue; + seen.add(modelId); + + result.push({ + label: modelName ? `${modelName} (${modelId})` : modelId, + value: modelId, + modelName, + modelId, + }); + } + } + + return result; + }, [data]); + + const optionRender = (option: { data: { modelName: string; modelId: string; label: string } }) => { + const { modelName, modelId } = option.data; + + return ( + + {modelName ? ( + <> + + Model name: + {modelName} + + + Model ID: {modelId} + + + ) : ( + Model ID: {modelId} + )} + + ); + }; + + const handlePopupScroll = (e: UIEvent) => { + const target = e.currentTarget; + const scrollRatio = + (target.scrollTop + target.clientHeight) / target.scrollHeight; + + if (scrollRatio >= SCROLL_THRESHOLD && hasNextPage && !isFetchingNextPage) { + fetchNextPage(); + } + }; + + const handleSearch = (value: string) => { + setSearchInput(value); + setDebouncedSearch(value); + }; + + const handleChange = (v: string | string[] | null) => { + const normalized = + typeof v === "string" ? v : Array.isArray(v) ? v[0] ?? "" : ""; + onChange?.(normalized); + }; + + return ( + + ) : option.customComponent ? ( + (() => { + const CustomComponent = option.customComponent; + return ( + handleFilterChange(option.name, value ?? "")} + placeholder={`Select ${option.label || option.name}...`} + /> + ); + })() ) : ( Date: Tue, 10 Feb 2026 12:53:51 -0800 Subject: [PATCH 105/117] fixing build --- .../PaginatedModelSelect.tsx | 14 +- .../src/components/team/team_info.tsx | 1296 ----------------- 2 files changed, 7 insertions(+), 1303 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/team/team_info.tsx diff --git a/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx index 285f4e94d68..9b22fd1bd87 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx @@ -72,21 +72,21 @@ export const PaginatedModelSelect = ({ const { modelName, modelId } = option.data; return ( - + <> {modelName ? ( - <> + Model name: - {modelName} + {modelName} - + Model ID: {modelId} - + ) : ( - Model ID: {modelId} + Model ID: {modelId} )} - + ); }; diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx deleted file mode 100644 index 014f8fb9010..00000000000 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ /dev/null @@ -1,1296 +0,0 @@ -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import UserSearchModal from "@/components/common_components/user_search_modal"; -import { - getGuardrailsList, - getPoliciesList, - getPolicyInfoWithGuardrails, - Member, - Organization, - organizationInfoCall, - teamInfoCall, - teamMemberAddCall, - teamMemberDeleteCall, - teamMemberUpdateCall, - teamUpdateCall, -} from "@/components/networking"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; -import { isProxyAdminRole } from "@/utils/roles"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { ArrowLeftIcon } from "@heroicons/react/outline"; -import { - Badge, - Card, - Grid, - Tab, - TabGroup, - TabList, - TabPanel, - TabPanels, - Text, - TextInput, - Title, - Button as TremorButton, -} from "@tremor/react"; -import { Button, Form, Input, message, Select, Switch, Tooltip } from "antd"; -import { CheckIcon, CopyIcon } from "lucide-react"; -import React, { useEffect, useMemo, useState } from "react"; -import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; -import AgentSelector from "../agent_management/AgentSelector"; -import DeleteResourceModal from "../common_components/DeleteResourceModal"; -import DurationSelect from "../common_components/DurationSelect"; -import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; -import LoggingSettingsView from "../logging_settings_view"; -import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; -import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; -import { ModelSelect } from "../ModelSelect/ModelSelect"; -import NotificationsManager from "../molecules/notifications_manager"; -import { fetchMCPAccessGroups } from "../networking"; -import ObjectPermissionsView from "../object_permissions_view"; -import NumericalInput from "../shared/numerical_input"; -import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; -import EditLoggingSettings from "./EditLoggingSettings"; -import MemberModal from "./EditMembership"; -import MemberPermissions from "./member_permissions"; -import TeamMembersComponent from "./team_member_view"; - -export interface TeamMembership { - user_id: string; - team_id: string; - budget_id: string; - spend: number; - litellm_budget_table: { - budget_id: string; - soft_budget: number | null; - max_budget: number | null; - max_parallel_requests: number | null; - tpm_limit: number | null; - rpm_limit: number | null; - model_max_budget: Record | null; - budget_duration: string | null; - }; -} - -export interface TeamData { - team_id: string; - team_info: { - team_alias: string; - team_id: string; - organization_id: string | null; - admins: string[]; - members: string[]; - members_with_roles: Member[]; - metadata: Record; - tpm_limit: number | null; - rpm_limit: number | null; - max_budget: number | null; - soft_budget?: number | null; - budget_duration: string | null; - models: string[]; - blocked: boolean; - spend: number; - max_parallel_requests: number | null; - budget_reset_at: string | null; - model_id: string | null; - litellm_model_table: { - model_aliases: Record; - } | null; - created_at: string; - guardrails?: string[]; - policies?: string[]; - object_permission?: { - object_permission_id: string; - mcp_servers: string[]; - mcp_access_groups?: string[]; - mcp_tool_permissions?: Record; - vector_stores: string[]; - agents?: string[]; - agent_access_groups?: string[]; - }; - team_member_budget_table: { - max_budget: number; - budget_duration: string; - tpm_limit: number | null; - rpm_limit: number | null; - } | null; - }; - keys: any[]; - team_memberships: TeamMembership[]; -} - -export interface TeamInfoProps { - teamId: string; - onUpdate: (data: any) => void; - onClose: () => void; - accessToken: string | null; - is_team_admin: boolean; - is_proxy_admin: boolean; - userModels: string[]; - editTeam: boolean; - premiumUser?: boolean; -} - -const getOrganizationModels = (organization: Organization | null, userModels: string[]) => { - let tempModelsToPick = []; - - if (organization) { - // Check if organization has "all-proxy-models" in its models array - if (organization.models.includes("all-proxy-models")) { - // Treat as all-proxy-models (use userModels) - tempModelsToPick = userModels; - } else if (organization.models.length > 0) { - // Organization has specific models - tempModelsToPick = organization.models; - } else { - // Empty array [] is treated as all-proxy-models - tempModelsToPick = userModels; - } - } else { - // No organization, show all available models - tempModelsToPick = userModels; - } - - return unfurlWildcardModelsInList(tempModelsToPick, userModels); -}; - -const TeamInfoView: React.FC = ({ - teamId, - onClose, - accessToken, - is_team_admin, - is_proxy_admin, - userModels, - editTeam, - premiumUser = false, - onUpdate, -}) => { - const [teamData, setTeamData] = useState(null); - const [loading, setLoading] = useState(true); - const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false); - const [form] = Form.useForm(); - const [isEditMemberModalVisible, setIsEditMemberModalVisible] = useState(false); - const [selectedEditMember, setSelectedEditMember] = useState(null); - const [isEditing, setIsEditing] = useState(false); - const [mcpAccessGroups, setMcpAccessGroups] = useState([]); - const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); - const [copiedStates, setCopiedStates] = useState>({}); - const [guardrailsList, setGuardrailsList] = useState([]); - const [policiesList, setPoliciesList] = useState([]); - const [policyGuardrails, setPolicyGuardrails] = useState>({}); - const [loadingPolicies, setLoadingPolicies] = useState(false); - const [memberToDelete, setMemberToDelete] = useState(null); - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [isDeleting, setIsDeleting] = useState(false); - const [isTeamSaving, setIsTeamSaving] = useState(false); - const [organization, setOrganization] = useState(null); - const { userRole } = useAuthorized(); - - const canEditTeam = is_team_admin || is_proxy_admin; - - const fetchTeamInfo = async () => { - try { - setLoading(true); - if (!accessToken) return; - const response = await teamInfoCall(accessToken, teamId); - setTeamData(response); - } catch (error) { - NotificationsManager.fromBackend("Failed to load team information"); - console.error("Error fetching team info:", error); - } finally { - setLoading(false); - } - }; - - useEffect(() => { - fetchTeamInfo(); - }, [teamId, accessToken]); - - // Fetch organization data when team has organization_id - useEffect(() => { - const fetchOrganization = async () => { - if (!accessToken || !teamData?.team_info?.organization_id) { - setOrganization(null); - return; - } - - try { - const orgData = await organizationInfoCall(accessToken, teamData.team_info.organization_id); - setOrganization(orgData); - } catch (error) { - console.error("Error fetching organization info:", error); - setOrganization(null); - } - }; - - fetchOrganization(); - }, [accessToken, teamData?.team_info?.organization_id]); - - // Compute modelsToPick based on organization and userModels - const modelsToPick = useMemo(() => { - return getOrganizationModels(organization, userModels); - }, [organization, userModels]); - - const fetchMcpAccessGroups = async () => { - if (!accessToken) return; - if (mcpAccessGroupsLoaded) return; - try { - const groups = await fetchMCPAccessGroups(accessToken); - setMcpAccessGroups(groups); - setMcpAccessGroupsLoaded(true); - } catch (error) { - console.error("Failed to fetch MCP access groups:", error); - } - }; - - useEffect(() => { - const fetchGuardrails = async () => { - try { - if (!accessToken) return; - const response = await getGuardrailsList(accessToken); - const guardrailNames = response.guardrails.map((g: { guardrail_name: string }) => g.guardrail_name); - setGuardrailsList(guardrailNames); - } catch (error) { - console.error("Failed to fetch guardrails:", error); - } - }; - - const fetchPolicies = async () => { - try { - if (!accessToken) return; - const response = await getPoliciesList(accessToken); - const policyNames = response.policies.map((p: { policy_name: string }) => p.policy_name); - setPoliciesList(policyNames); - } catch (error) { - console.error("Failed to fetch policies:", error); - } - }; - - fetchGuardrails(); - fetchPolicies(); - }, [accessToken]); - - // Fetch resolved guardrails for all policies - useEffect(() => { - const fetchPolicyGuardrails = async () => { - if (!accessToken || !teamData?.team_info?.policies || teamData.team_info.policies.length === 0) { - return; - } - - setLoadingPolicies(true); - const guardrailsMap: Record = {}; - - try { - await Promise.all( - teamData.team_info.policies.map(async (policyName: string) => { - try { - const policyInfo = await getPolicyInfoWithGuardrails(accessToken, policyName); - guardrailsMap[policyName] = policyInfo.resolved_guardrails || []; - } catch (error) { - console.error(`Failed to fetch guardrails for policy ${policyName}:`, error); - guardrailsMap[policyName] = []; - } - }) - ); - setPolicyGuardrails(guardrailsMap); - } catch (error) { - console.error("Failed to fetch policy guardrails:", error); - } finally { - setLoadingPolicies(false); - } - }; - - fetchPolicyGuardrails(); - }, [accessToken, teamData?.team_info?.policies]); - - const handleMemberCreate = async (values: any) => { - try { - if (accessToken == null) return; - - const member: Member = { - user_email: values.user_email, - user_id: values.user_id, - role: values.role, - }; - - await teamMemberAddCall(accessToken, teamId, member); - - NotificationsManager.success("Team member added successfully"); - setIsAddMemberModalVisible(false); - form.resetFields(); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error: any) { - let errMsg = "Failed to add team member"; - - if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { - errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; - } else if (error?.message) { - errMsg = error.message; - } - - NotificationsManager.fromBackend(errMsg); - console.error("Error adding team member:", error); - } - }; - - const handleMemberUpdate = async (values: any) => { - try { - if (accessToken == null) { - return; - } - - const member: Member = { - user_email: values.user_email, - user_id: values.user_id, - role: values.role, - max_budget_in_team: values.max_budget_in_team, - tpm_limit: values.tpm_limit, - rpm_limit: values.rpm_limit, - }; - console.log("Updating member with values:", member); - message.destroy(); // Remove all existing toasts - - await teamMemberUpdateCall(accessToken, teamId, member); - - NotificationsManager.success("Team member updated successfully"); - setIsEditMemberModalVisible(false); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error: any) { - let errMsg = "Failed to update team member"; - if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { - errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; - } else if (error?.message) { - errMsg = error.message; - } - setIsEditMemberModalVisible(false); - - message.destroy(); // Remove all existing toasts - - NotificationsManager.fromBackend(errMsg); - console.error("Error updating team member:", error); - } - }; - - const handleMemberDelete = (member: Member) => { - setMemberToDelete(member); - setIsDeleteModalOpen(true); - }; - - const handleDeleteConfirm = async () => { - if (!memberToDelete || !accessToken) return; - - setIsDeleting(true); - try { - await teamMemberDeleteCall(accessToken, teamId, memberToDelete); - - NotificationsManager.success("Team member removed successfully"); - - // Fetch updated team info - const updatedTeamData = await teamInfoCall(accessToken, teamId); - setTeamData(updatedTeamData); - - // Notify parent component of the update - onUpdate(updatedTeamData); - } catch (error) { - NotificationsManager.fromBackend("Failed to remove team member"); - console.error("Error removing team member:", error); - } finally { - setIsDeleting(false); - setIsDeleteModalOpen(false); - setMemberToDelete(null); - } - }; - - const handleDeleteCancel = () => { - setIsDeleteModalOpen(false); - setMemberToDelete(null); - }; - - const handleTeamUpdate = async (values: any) => { - try { - if (!accessToken) return; - setIsTeamSaving(true); - - let parsedMetadata = {}; - try { - const rawMetadata = values.metadata ? JSON.parse(values.metadata) : {}; - // Exclude soft_budget_alerting_emails from parsed metadata since it's handled separately - const { soft_budget_alerting_emails, ...rest } = rawMetadata; - parsedMetadata = rest; - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in metadata field"); - return; - } - - let secretManagerSettings: Record | undefined; - if (typeof values.secret_manager_settings === "string") { - const trimmedSecretConfig = values.secret_manager_settings.trim(); - if (trimmedSecretConfig.length > 0) { - try { - secretManagerSettings = JSON.parse(values.secret_manager_settings); - } catch (e) { - NotificationsManager.fromBackend("Invalid JSON in secret manager settings"); - return; - } - } - } - - const sanitizeNumeric = (v: any) => { - if (v === null || v === undefined) return null; - if (typeof v === "string" && v.trim() === "") return null; - if (typeof v === "number" && Number.isNaN(v)) return null; - return v; - }; - - const updateData: any = { - team_id: teamId, - team_alias: values.team_alias, - models: values.models, - tpm_limit: sanitizeNumeric(values.tpm_limit), - rpm_limit: sanitizeNumeric(values.rpm_limit), - max_budget: values.max_budget, - soft_budget: sanitizeNumeric(values.soft_budget), - budget_duration: values.budget_duration, - metadata: { - ...parsedMetadata, - ...(values.guardrails?.length > 0 ? { guardrails: values.guardrails } : {}), - ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), - disable_global_guardrails: values.disable_global_guardrails || false, - soft_budget_alerting_emails: - typeof values.soft_budget_alerting_emails === "string" - ? values.soft_budget_alerting_emails - .split(",") - .map((email: string) => email.trim()) - .filter((email: string) => email.length > 0) - : values.soft_budget_alerting_emails || [], - ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), - }, - ...(values.policies?.length > 0 ? { policies: values.policies } : {}), - organization_id: values.organization_id, - }; - - updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); - updateData.team_member_budget_duration = values.team_member_budget_duration; - - if (values.team_member_budget !== undefined) { - updateData.team_member_budget = Number(values.team_member_budget); - } - - if (values.team_member_key_duration !== undefined) { - updateData.team_member_key_duration = values.team_member_key_duration; - } - - if (values.team_member_tpm_limit !== undefined || values.team_member_rpm_limit !== undefined) { - updateData.team_member_tpm_limit = sanitizeNumeric(values.team_member_tpm_limit); - updateData.team_member_rpm_limit = sanitizeNumeric(values.team_member_rpm_limit); - } - - // Handle object_permission updates - const { servers, accessGroups } = values.mcp_servers_and_groups || { - servers: [], - accessGroups: [], - }; - const serverIds = new Set(servers || []); - const mcpToolPermissions = Object.fromEntries( - Object.entries(values.mcp_tool_permissions || {}).filter(([serverId]) => serverIds.has(serverId)), - ); - - updateData.object_permission = {}; - if (servers) { - updateData.object_permission.mcp_servers = servers; - } - if (accessGroups) { - updateData.object_permission.mcp_access_groups = accessGroups; - } - if (mcpToolPermissions) { - updateData.object_permission.mcp_tool_permissions = mcpToolPermissions; - } - delete values.mcp_servers_and_groups; - delete values.mcp_tool_permissions; - - // Handle agent permissions - const { agents, accessGroups: agentAccessGroups } = values.agents_and_groups || { - agents: [], - accessGroups: [], - }; - if (agents && agents.length > 0) { - updateData.object_permission.agents = agents; - } - if (agentAccessGroups && agentAccessGroups.length > 0) { - updateData.object_permission.agent_access_groups = agentAccessGroups; - } - delete values.agents_and_groups; - - // Handle vector stores permissions - if (values.vector_stores && values.vector_stores.length > 0) { - updateData.object_permission.vector_stores = values.vector_stores; - } - - const response = await teamUpdateCall(accessToken, updateData); - - NotificationsManager.success("Team settings updated successfully"); - setIsEditing(false); - fetchTeamInfo(); - } catch (error) { - console.error("Error updating team:", error); - } finally { - setIsTeamSaving(false); - } - }; - - if (loading) { - return
Loading...
; - } - - if (!teamData?.team_info) { - return
Team not found
; - } - - const { team_info: info } = teamData; - - const copyToClipboard = async (text: string, key: string) => { - const success = await utilCopyToClipboard(text); - if (success) { - setCopiedStates((prev) => ({ ...prev, [key]: true })); - setTimeout(() => { - setCopiedStates((prev) => ({ ...prev, [key]: false })); - }, 2000); - } - }; - - return ( -
-
-
- - Back to Teams - - {info.team_alias} -
- {info.team_id} -
-
-
- - - - {[ - Overview, - ...(canEditTeam - ? [ - Members, - Member Permissions, - Settings, - ] - : []), - ]} - - - - {/* Overview Panel */} - - - - Budget Status -
- ${formatNumberWithCommas(info.spend, 4)} - - of {info.max_budget === null ? "Unlimited" : `$${formatNumberWithCommas(info.max_budget, 4)}`} - - {info.budget_duration && Reset: {info.budget_duration}} -
- {info.team_member_budget_table && ( - - Team Member Budget: ${formatNumberWithCommas(info.team_member_budget_table.max_budget, 4)} - - )} -
- - - - Rate Limits -
- TPM: {info.tpm_limit || "Unlimited"} - RPM: {info.rpm_limit || "Unlimited"} - {info.max_parallel_requests && Max Parallel Requests: {info.max_parallel_requests}} -
- - - - Models -
- {info.models.length === 0 ? ( - All proxy models - ) : ( - info.models.map((model, index) => ( - - {model} - - )) - )} -
- - - - Virtual Keys -
- User Keys: {teamData.keys.filter((key) => key.user_id).length} - Service Account Keys: {teamData.keys.filter((key) => !key.user_id).length} - Total: {teamData.keys.length} -
- - - - - - Guardrails - {info.guardrails && info.guardrails.length > 0 ? ( -
- {info.guardrails.map((guardrail: string, index: number) => ( - - {guardrail} - - ))} -
- ) : ( - No guardrails configured - )} - {info.metadata?.disable_global_guardrails && ( -
- Global Guardrails Disabled -
- )} - - - - Policies - {info.policies && info.policies.length > 0 ? ( -
- {info.policies.map((policy: string, index: number) => ( -
-
- {policy} - {loadingPolicies && Loading guardrails...} -
- {!loadingPolicies && policyGuardrails[policy] && policyGuardrails[policy].length > 0 && ( -
- Resolved Guardrails: -
- {policyGuardrails[policy].map((guardrail: string, gIndex: number) => ( - - {guardrail} - - ))} -
-
- )} -
- ))} -
- ) : ( - No policies configured - )} - - - - - - - {/* Members Panel */} - - - - - {/* Member Permissions Panel */} - {canEditTeam && ( - - - - )} - - {/* Settings Panel */} - - -
- Team Settings - {canEditTeam && !isEditing && ( - setIsEditing(true)}>Edit Settings - )} -
- - {isEditing ? ( -
rest)(info.metadata), - null, - 2, - ) - : "", - logging_settings: info.metadata?.logging || [], - secret_manager_settings: info.metadata?.secret_manager_settings - ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) - : "", - organization_id: info.organization_id, - vector_stores: info.object_permission?.vector_stores || [], - mcp_servers: info.object_permission?.mcp_servers || [], - mcp_access_groups: info.object_permission?.mcp_access_groups || [], - mcp_servers_and_groups: { - servers: info.object_permission?.mcp_servers || [], - accessGroups: info.object_permission?.mcp_access_groups || [], - }, - mcp_tool_permissions: info.object_permission?.mcp_tool_permissions || {}, - agents_and_groups: { - agents: info.object_permission?.agents || [], - accessGroups: info.object_permission?.agent_access_groups || [], - }, - }} - layout="vertical" - > - - - - - - form.setFieldValue("models", values)} - teamID={teamId} - organizationID={teamData?.team_info?.organization_id || undefined} - options={{ - includeSpecialOptions: true, - includeUserModels: !teamData?.team_info?.organization_id, - showAllProxyModelsOverride: isProxyAdminRole(userRole) && !teamData?.team_info?.organization_id, - }} - context="team" - dataTestId="models-select" - /> - - - - - - - - - - - - - - - - - - - - form.setFieldValue("team_member_budget_duration", value)} - value={form.getFieldValue("team_member_budget_duration")} - /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - Guardrails{" "} - - e.stopPropagation()} - > - - - - - } - name="guardrails" - help="Select existing guardrails or enter new ones" - > - ({ value: name, label: name }))} - /> - - - - form.setFieldValue("vector_stores", values)} - value={form.getFieldValue("vector_stores")} - accessToken={accessToken || ""} - placeholder="Select vector stores" - /> - - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes" - /> - - - - form.setFieldValue("mcp_servers_and_groups", val)} - value={form.getFieldValue("mcp_servers_and_groups")} - accessToken={accessToken || ""} - placeholder="Select MCP servers or access groups (optional)" - /> - - - {/* Hidden field to register mcp_tool_permissions with the form */} - - - - prevValues.mcp_servers_and_groups !== currentValues.mcp_servers_and_groups || - prevValues.mcp_tool_permissions !== currentValues.mcp_tool_permissions - } - > - {() => ( -
- form.setFieldsValue({ mcp_tool_permissions: toolPerms })} - /> -
- )} - - - - form.setFieldValue("agents_and_groups", val)} - value={form.getFieldValue("agents_and_groups")} - accessToken={accessToken || ""} - placeholder="Select agents or access groups (optional)" - /> - - - - - - - - form.setFieldValue("logging_settings", values)} - /> - - - { - if (!value) { - return Promise.resolve(); - } - try { - JSON.parse(value); - return Promise.resolve(); - } catch (error) { - return Promise.reject(new Error("Please enter valid JSON")); - } - }, - }, - ]} - > - - - - - - - -
-
- setIsEditing(false)} disabled={isTeamSaving}> - Cancel - - - Save Changes - -
-
-
- ) : ( -
-
- Team Name -
{info.team_alias}
-
-
- Team ID -
{info.team_id}
-
-
- Created At -
{new Date(info.created_at).toLocaleString()}
-
-
- Models -
- {info.models.map((model, index) => ( - - {model} - - ))} -
-
-
- Rate Limits -
TPM: {info.tpm_limit || "Unlimited"}
-
RPM: {info.rpm_limit || "Unlimited"}
-
-
- Team Budget -
- Max Budget:{" "} - {info.max_budget !== null ? `$${formatNumberWithCommas(info.max_budget, 4)}` : "No Limit"} -
-
- Soft Budget:{" "} - {info.soft_budget !== null && info.soft_budget !== undefined - ? `$${formatNumberWithCommas(info.soft_budget, 4)}` - : "No Limit"} -
-
Budget Reset: {info.budget_duration || "Never"}
- {info.metadata?.soft_budget_alerting_emails && - Array.isArray(info.metadata.soft_budget_alerting_emails) && - info.metadata.soft_budget_alerting_emails.length > 0 && ( -
- Soft Budget Alerting Emails: {info.metadata.soft_budget_alerting_emails.join(", ")} -
- )} -
-
- - Team Member Settings{" "} - - - - -
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
-
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
-
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
-
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
-
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
-
-
- Organization ID -
{info.organization_id}
-
-
- Status - {info.blocked ? "Blocked" : "Active"} -
- -
- Disable Global Guardrails -
- {info.metadata?.disable_global_guardrails === true ? ( - Enabled - Global guardrails bypassed - ) : ( - Disabled - Global guardrails active - )} -
-
- - - - - - {info.metadata?.secret_manager_settings && ( -
- Secret Manager Settings - 
-                        {JSON.stringify(info.metadata.secret_manager_settings, null, 2)}
-
-
- )} -
- )} - - - - - - setIsEditMemberModalVisible(false)} - onSubmit={handleMemberUpdate} - initialData={selectedEditMember} - mode="edit" - config={{ - title: "Edit Member", - showEmail: true, - showUserId: true, - roleOptions: [ - { label: "Admin", value: "admin" }, - { label: "User", value: "user" }, - ], - additionalFields: [ - { - name: "max_budget_in_team", - label: ( - - Team Member Budget (USD){" "} - - - - - ), - type: "numerical" as const, - step: 0.01, - min: 0, - placeholder: "Budget limit for this member within this team", - }, - { - name: "tpm_limit", - label: ( - - Team Member TPM Limit{" "} - - - - - ), - type: "numerical" as const, - step: 1, - min: 0, - placeholder: "Tokens per minute limit for this member in this team", - }, - { - name: "rpm_limit", - label: ( - - Team Member RPM Limit{" "} - - - - - ), - type: "numerical" as const, - step: 1, - min: 0, - placeholder: "Requests per minute limit for this member in this team", - }, - ], - }} - /> - - setIsAddMemberModalVisible(false)} - onSubmit={handleMemberCreate} - accessToken={accessToken} - /> - - {/* Delete Member Confirmation Modal */} - -
- ); -}; - -export default TeamInfoView; From ea38630e7cfb240290758543dcc319b2263692a1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 13:58:57 -0800 Subject: [PATCH 106/117] =?UTF-8?q?bump:=20version=200.4.33=20=E2=86=92=20?= =?UTF-8?q?0.4.34?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 8937858bbd9..e0a769a5edf 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.33" +version = "0.4.34" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.33" +version = "0.4.34" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index f26e49093de..eb68cd2f4a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.33", optional = true} +litellm-proxy-extras = {version = "0.4.34", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.31", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index f680de120c5..be4c6e2f7e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.33 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.34 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env From 7d2c874434577be17047372ebc8dcc4c48b226be Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 13:59:48 -0800 Subject: [PATCH 107/117] Fixing ci pypi build --- ...litellm_proxy_extras-0.4.34-py3-none-any.whl | Bin 0 -> 53171 bytes .../dist/litellm_proxy_extras-0.4.34.tar.gz | Bin 0 -> 24232 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..175d84543ec1d32172a52329fe2341a80f7ce006 GIT binary patch literal 53171 zcmcG$1yq%5*EUL*2nYx$DJ9LKK|~q>>6FezcSuW1Bi$k0ol18%(g;!l(v5(eXQ_LC zAH4fLKl}XSIL5skGGvT3=RM<^*PQc~c?AoH4Fv^-1e_o~eKSi#Lt_g= z;P-vyCEw4IU^HAPlVpC@?O8<>>P_<-c%y7=BWkG#E<2gZT5W?VAIRTy-_;KP0Mmxge#=hem(?QS7BG zGnuGo!X>H{D__%BnR^L- zuV4kkCQ9)u9ZW>{zN8#4PQ&1X>C>~Tj*jCH$TYDl>1ZF%%~qB4=reoB{fL3e5~}#% zGHNZn^OBD>vZn$O3jLUBBDJidHRl}pLmUupWR?qP-g^~dtWbxHrTulm<{E~oO38uVH@ zMxwHw5_b~@#SF|&w{i_OrM0(U>7}SvZnMf6Xb|clUihvooFU^m&5SL;jDPh zRn>XRS#}KfA+0AXS!iLX@Fx>I)YWp1idzyfeh9)(hB3EG!-Yc96ZPAEp2{wyi>gK~ zS6btcK$8+V)t4*~zOSzZb;7}*Xq2LS?U^c*m}FKz9IXzP$Covlo&2)OKE;z+#*D_@5!Ll;9G94 zkHh7g4s`;d1avsQ5@e5WvI>$!gc0wZ2K3F>(-EI{_KP(a+e^B*bW} zi|IziE=&TxO;rs;yhR;M*X}uO9WME?olO#lw8yOrwVN)jPH_nuNTiy5QEsv#>My2H zUaLG;wxykgdk|RXiQ8fXAFxz#>am{u(u#@_Y^;yY4;`{;D=mTH;}n2${E=;)ru`?6 z{*uPvw`#C>rfNkJIyw&j(EBh?#{#OaY-{=!YWOc7x!Tu@>Xig0Lqpd}JoyT1)z?S` z3nIpb4ZAnf8!o?PZE6f>3Nv~H9m)5){$xkhGkZC%Myv7dS+kSdAn2Zkg>Ni!T@r3E z6pE9LA-o3&MvBxQjtf)vs zyifQ|S8(mo8vU#zp9Wp)^Y^~4L8B>pue(_7OxT~xiMl6P7%=D}gz74R(Nmt^-jb=) zhO1OZ2-Rg;Ig%FT8G+i2DM}iZEpg~49wNd9!6Emfb_Mq4ptxYwZag!9n$*T0_2OGACyX?i zOc95#VY8<4AdW3%%SRz;^$?v0({vt&>bT7))QlWdWpSJDKPsITHW0zL9CdD@7|e4? z>klS!6UScifO9(EscoV3S=ip?#g=b=ZTRKOhf;lktk+%{KAU3~!OwlT?2LxQs7c0! zKbnL6!f9T0^7G2Jxj)gWx*yx{W4rRj!)1K0nt;Ieg>38!)Tk;xXxN|XL?cB-QZao%Q=vw!Y;_P7A%TnJ2U&>@?K(=Q%GV?-NvhfJ zMr-{15+{qL)cesLNj`>pLg9w3#u4SNOuDl7wpqqbxRh^# ziD6Lw^!DJh4cF9A#kyl&1zOb_!2_ZQal^;?G!cUZMl#*elX{tk>KPKtyZ&of=GY6i z%GBP1kFtpNSTe^fNR(IdZB3`&%hdSUe$C@hc)0PtjG{xmdM?FqUYfm zoZ#*B{R{ufXFFz*Ie6a$4L@ml=}9*F#OrQbi(cpZi5i6p7N3S` z_X=hdeYBK9p_u{MMk`I)F`}5Npsz$b!jsJMsxRi)wN9_ORFh?*3GQPOR&|hbNYey# zbRPD-To{Vk%Qf$-IZ_m`-QLM4H9GxJf6qkqe1}S3p3uc!E}gulkGv@8NnXH&x=5kOqB|0Z>EFo8hKS~~jrTKYQ1W=>l6wqP4AI~`rKTWK^|Ud_}Sgy(ju z43|nb#Wb~?l4L+eXM#&sDcwS^BiaC0m2q_%)XCXd8HoaYZt<-ysbU_c>4C~qqZ3V} zzNf<(7&5!ag4)aVSa^PP-3Ln%)#+lHxwZER?&S^*erEJ_K}mnzfUT!ID@_wY&-Jl2 z;Z&d5@TGOULkt^k$3xr;v(3nH=!~o7@i+E8f$93?c}+G930iN#bl|Q}KdAC25UA4k z;3M*$AuU$K>Z0sO@;ub^XDBH+<|@*BjgfOD-~ji7ir<5>zAzLIcz3A`t0rhN{T!+X&=^IUae z!U|DQJ?XxK40iLCZc8IeM@Yr}ahw>WHKUlhK&Asr_+XvQB2hB1GRY&Gg}_X)pS+D=2j{wn@_LUJ+CWe)DFiivar2uJkn@DN3E{+ zG5Xc%K=lqO9XCdt&X)_vcsf`on<58C^~U&0DkRTM7|*6)jSaP?egYw6Bbxix$2HTM@JIn9Ao3XG`4t11+&P4k{orUl8_&s#*69bGucb{|Htpvd#&%Na4t+s+F z@$1vEsfGCg({l6PAh;Ckr!>Ne!3Cy9h6{A^#9k>MKXyj=Ff944OwiGqj>y<57NPF? zaZf~DQLlW7(igT^U;93AwEC74n|-o%VlR-beU^oM%e7zYrZO%2y9u2DC6)sRVmSX; zxcv=}|3c6#Ow3GdtZeLBKwYP2s%2}hrw6vR)zY!h*D?Tt0$Bfltmt=;&@nXpCtZgG z|FR&yQ`e60X|K95<8-7+>?7vb8O=0|z_o~$rkY}@#^Zg>F{o_Pc)ypJ-t9jTf?pnJ z8&eC6pIE#e<<4KZke}#*d4`x1s*ArEl+3{>pSr3j9Hd%{a1y(X_3Rx+K=V0^i0;&G zzh$94fv|n?MoCLc9QsWqBq-rCp8>*K10}5dU6sJf^h*gWOuszGT*pGk5Rlsdkp7<} zO%7+W1boHU;W{ufM~GDZB_&qy$`a@JM^TUp0{#lSPp%}zkKIHBUr(BF)mNIXsS7U7 zvE*TmF0;;+7rl`Rw(khS=*baj1ixqXOBk5C99TTe#}|1*I!ySO0DeUcF500c%I1yB zbzvKmqhaVgIt#*aTi?&m#HuJm!o@i6+s~I=!GqPt0&YL>Jki`-4b5`ANSEQu&9Hbn z9MEE1JH!LRVpMH?*QD+n<8u^&$avz`^7#gQTK7#P4%eEf%qqL%zO*FllKjY{rrb{o z(>GEZAJp=zenp`vjwV2V{zl+DO<@dw)kIfvq(nH_@2tSZYSR6qJM2wGXSDG-nE?uR z1dhVp70nFdWa40B(Xs>Um}{AX&2@o{V+FP`H@3C?Ro4A&pOEs6qK6+6l|;U}N$p3v zG0G_GVJXY|g5a~FhFj@iLc}93GCtS1YE62cWD8H2D+|)S;9Kt}ijkLJV+t%j?*anS zJACQohcBz$s0{`ZW6mZb z4bemT%g|a$FrbE$OKL&0?eoa+AwAKQ&i!y36cLy+G6a_{-**--JSOcw*er{G4OgKe zw}q8^$x6b+CbHFlM%^^ud)#RCh=QueXXta1&3w*bP_1w41P+GPWOGOQ8w4-oh8pbNq+AfC$YD|1aYAE)p z9fAC`Z*Zo(6lGZ_k8J!fEN=*Rh1dR(S5Tv}uzbG97b1=-D&~ajNAv|tEIRc+R%1I0 zu@$fpQzoDB5qS<_q!xIyxt0s~d3=_UewU+8F8qQwM+sK#LWg_!{yTP*u-y>yxuUnh zX-mZSOFHO$C_|kzb`y^h3L9Ky2a9|Bxnn?&k-3CcT0LIxbk=HJ4wMQ8r@dU~$}bs? z+OmO{{|;lma1W_k|D`-?Cb%o|zDTjJ!y^2S$4I$#8a)xKIhV}i)nH;oaWc#MHIeYM zgXG}%&%)lbbhrqHpDG1w;;?8VK+gw`(;6d;e)y;s9(D4wOZFgH1exdP@gSl0>FYK^ zMMT4*g8R&-pA4FU`*%OQqMYj#SMpQQ>e*uCI_)ht;98o}3D6)`Kust3AaTFizZ7 zI#}4L#@dhAaPAX?tr=U2H)GIw#@S@)J?B{U5{P}O z0@kGM4(9V5>^jefen@?$U<{1lFu&rB9VTi&)|u{iF$l^Ty9c_Tpk?dCi$+s|hb%uo zO?htp?8qt-gK-MMnpJjhd#&xTX0OCgYCw0r0`=2Q86!iU^SB_~K?KSn%qJnL`Fav^ zwRdd;2N#>RzOj_Z0eJ3xk01OHNfEIkj9l*+iJ^GCu--j<*ctqd5?LT=E@NSZh~5wW zHCMs=SJ(=eDW|WZL|Rg#jN)H>G%P|M!J7{II+5C{T6uf{fDbFhWoP4gN1 ztY}5u%edG7N~~zn2KfM8X79+>Iz&e1podQEcz-{{L9D+&4^>P8ok4WqRRE)y#QPzs zB8Bp`C74@={(S=A41sU!034Hl^Q}4Am{@;DHXSoFO9u!_vb47Y+iLvXO<`#^!e^z)RAxxbmMEC^nvHyb-#wE4JV%XJe#U-|a zCpg3;m09vxF=lCN37%KN9|h0^fl3>@01xY=IB3 z?M-2Q!v{>-Ko~x|&Vj;D@e}0V^H3L~SWs|x+Zam;RK<{P@|sZ$*OV+}dJb2rOpUoF z%&QbBg-II0np}J#?ZGxo@HQ}Q@kAk(K{C$V-bVbSY}N`=EVI_MLP7R?ZP!F)@Ithc z8zrI1n5uWG3d7-L?zC_Ni}`H4_2ZK-U+oC%t-hQ4z1&LJw6dV&Mc_Sh7bHaZZdQt& zQc(W-y0D`CP3OBa=GgjC35$q}qx$>L51uOByRTA&%~;7L|I}}O{ig$`wfXbqC-zC} zrRZO(ar1DfXoJLQV{m_*Ob?s|XkoL{#7dZd^|gF*ehg17MvQcha8t`Aw9~Q>Enfp# zzH^#j19331GqL@_1?GAXw+9G^e@YX{vXT(mq49!NvtB_*pv){bxttDGr4o~@FehyD zUZfa)gx#<^ub~>pD+d*l*l*vj#4$hMRK3ir;XzOUkvCU8%|)?ro{@N5&&;)hvy(y9Q@1CpBdCpScZBCbUNO z`ZYxdDCm;fI#jk$*-ebPCHFawHkccvV+zRKYNw7`nrPiwIZg8(#g5WkX*}ywp%Bv) zDR2}|ufiw{Ga)Sro?Uho8vDt(t9#jlZ~cjZhFVmqyJ;}99}z7@PM_-mz9Sgl|7|iI z+EN$E*{$mMn&N(Z9H3u9K)-r-wc@|@`x}HBfpyI6jI;nG3e~D|_aFfS_~##=a>3cwRO$%q(_(WYg^JpF`h+f_4x zu#gf<#JR$CSfX(8mWxxhj}4t1Y18A=H(ikn9p7w|h(>uO`;l%yVTzt4a4sYKp-ijbkJA`tlpM*o7O( ztejeWqea!rI3ehW=ge@a@p3HeO5qwg@#mtiA`Pqh5i#NT$YO%I2#9&z-Nu6CwyWmj z({t^5>DsV=HvlHZ-a(!pVf@o^{BhU z`eIm3nnnZ}lB4DOqqJaL1_s*7I9m%j~o^4lMr{=t3 zWhbHON2K)=&QW)`WvrXGm^Ohg2uVMZz`1h;$p+#CaWFH1SbnD=I~_awUl#qVo&uW6 zx9k}L75b^g=uyNZdc{_4j*zYI%OP3Y9at;=4HkZZg#8|AMkP5bJ8N01lT+m5V|gg( ztfh$d2)PfA{-#3yY`D0$kVD+JlC@6Yi?aYVk=ET!kc$bZi&)tpOskfzu_4e!x6?8J z(#!Aq=uc4yiHEo3_#Dvy_$9WlL(0-Sm6YZ@Bqs8QAyF*><2ECmFwrqBHL;5;^!3$n~Hkz+(Jn$!BTF|esW?4 zr@BE~&$U|@;eig^ln$<7Tw!DseZ8R&s*eEeZ0G&urC?#ddOH0Km#+Q?IIuixpLiEl z(WWPfLXt(}G?7@dH>1g3-*?kHhf#Ib1St|8{OH?xThvf9)MlK7wqxTxzwj|JOS^jS zeO=ux1|jH)!3pH<4S;iJ$jHV7Vq#@xV`aSsLjXY(}vbeXPA@3F)sfrN09r`~U*d zo!NH)J_2Ge1C{_`-wo|`Z1imawhb`l|773Y!ucU-@EyQn5_3_HkA-q)vEy&(ZY@2n z%n7642oruf(R#A`P3Rtdg{OLi->GB@qZr9{{Wi}`&7xR7mmpgALhqW(MOW7nc&Y-qlBV2hsZ?e`>P4~M`Dd> zBYiEXlOx=D1g&`noo*`26hMqDX0V@b{nG_Ui)LTd2E(N7wC~}h(!lGuf>kHs8rw6M z=36-RyRvS!GN9G8f791sV+Q`PfPgaeF9c_)YXV@nf6{D`+=vC>Rc_yCEgV~gQcl)V zXvTetu$5n>GiCFu;j(87FGjdt**`44)J{Z#PFd|20x{r1^lFy2{!Zx zoVVntzyr)ckP1bKINGR$an$|p)0QAn0y)m-E@iS}X_;~EGtK*P4n;J&@f5yiuU|z6 zfP2W~KC%sOpt?TiWBTfWCm6X=ILUHs8ll0!*!f&F=DSypAO00loB#4V`1JEcRqO>? z&X{b8tTdR%(Y%zIrkiz&pCtNZP3OrHv<)(}_?&gT%f1?8kG!QPT`)g5R{Cs9Kqw`S zLgeGvF3V|0^z7+ek1(SFsyu^KE=n6$#yx5L0NPsHx?-0Hgx2q1dS|zA&g*C{-q9gF zNNi?F@vtpz)U$tUfLE9~L(CFvyoafWl>5G`XiixcnTIaD2yt;-$Q6+(+PhX({5Uwl z_p&QgVQ1H@Yr(BnQZO0)7A=iC;zDEz_y_X8@(*CJ18{N>z{o=CD=iCqGczrq5Hz-P z`j4994l({lK`1Ndk!$@s%FED*Nhr%%{VE|3(Pj7U9q*T|Esau443D>~Z)L+!EV0OU zb8uC1Rv{JiWbq*vQr@0+`ZVq5Zv{x_3>0WV324qqC-T^vDB!i_@yp`RUKcru#pXri>s}^QrEIodJ zNm#JBw7G`2iIu*HhrNRLh|m-a|DfhSYi9rPeAyg8QVHNb@652X1EV^S$jzdqr(>(9 zqYq?1dwoN&9e_-AEDUdFyX5zOXS?i3wCA4k#*FV|ROL1og-F!PWPM1`>Kk~y+gtc2 zcaxGBpQ7@IUSXV)ou5RE5?HMcdo70xX!lBIDkwe6{}lWwq4WC)I%*~x7gEMOL#Ool z8eNrzgqZA6j@7S1EVFR@crca;T5jsSymlD)K|M->fiADh9C|0TnM8tAwabEqOXd`h zG3<-I`M7e=N&R*@qhAUmf`HO-*nguJpr&SK<7EDA2{+LCKV*U| z`94U&S${$6BaKEbvtIVJOIEF_oIw9kM5N487W+Qg6f?QIR!7+(Ns*YmCqrEYOTqkz zzE?d;6hfNPqt`@T#m%C%@`2%!^7o^D91tpGV=;OjXZucfmxXb?8KqqL5f*N2M&^kN zQ`S7^Kf3=QDwavVT!ssE^!3oeE+S;H=4aZds#)4Oq_SjYxkiPZSp+=`x%_WQpLm5#Zo>$5uIbU>Q5&@ zqYg8Vheg*P(5OqiNOvw+C7@QJ&a-~wf_PK8!X%3da)44HM9iI`GN4>$CKllH7S8`a zC-u8TN6G$I64H|Uzk(!mf&8t|2m$`1x)i<`ll~Z{3=FsM-m$rjAOx=J0_V=Kl><_= z0QT~`Du)!G#uf&aT6&gd_U3?<{I_!aF0$C8AxY8Gc<0OBl45Sf>#V=N(v@!a~O!Y-^>X z2N0rmmX>C}{(aNrJEDe6fS!Ek?hE1WJl|tv=kiVdZWB|oE&9x7{;W*=^W_&OA?}}E zp=T$S3p2)JT&^ZEvMgUvZAMAWph#&(88T9dWt@3{j1frmhe*Hpv#cH`zN^-SEm7$} z)K=%%f*q8|M}eb#fAxVJOMrPkZy$=+Da9QhzgB&bQdbh(&>W(1G(51C5kN5IiT5Gn zQo{omxCT{hAK>)6<6YfiJD-v}9X11Z<_6rK_?_<#=y?ON>rcDJ z)=uBp5@LrpFUwb^4=6h^>JJ|#YSwn6FXq7z^$Q?J2?-*3&S0RTkHO$BHx-0ig<+g> za#wJ(#K2cR%3Knj7WF1QC0#u7Ai^j_3|Rlxh}9`4$>IlPuVIBp`4%emiElnD^iHx< zg6DTaV=n?D?2%Uq(N}u2iG&>W&0Hh%{1wQ!^y`I ziu}_{i;vHSBhbS2K2ZQog3C*n(t?xsr5RRs7aVN*14#i4>S`BsYVJ4>Jk1egfROI`0^zoziTE?0}2za3g` z|5RnoO*30iqXQdXfy!}(z|8nKTe+($M9 zwvV)I_%HMhn{?VJL5%j2BBGKUEVXAuja&*MIIRSW$mn6}5t(!ihIDG_z4BO%F}X^8 zhCw99Xs5KJS;>i8S&dilJ*nNX-{0N}@3e#?vrvE%M}Pwf@qgd=|6YH%4d^*R>>w_n zlf?=&tso%~s9KD5e)CelSk_;0@K>1s2UHglCJjs}VYlzT4Sa%Bjd-s?$=T|7`e8bb zgw#NwcA*TEeyA0`Ixl8trAN;$7vo%1qc7w)U6_ z)IJ||8RQ2RoSs(}oicJn20poq(kf^MA$UU2C)-VQrP-|BW8_vbWA?CJ``2``>;uZ) zrc0>}e}3SzP_fUbdY1qJzW#1+PWC5Fk6?UT=NsKBd>EtPsa-?skn zM8dh{^>nn88xH$o47&Kyo|ae`b=4Z>4p-(EoJO#=w+Hj!Qy(GffaS2$CX|S>YNKHT zL?w@(scD?-iXrv7u?KLU+(b`TZt-R=$k-2a0qsx%7Agh&eJ2YA1~HhpSUG@kPkRf9 zh3aelZNoQ%R&-bkFmr?YZr3GnIMNHYB_n50@lj=>5#<*V!befjIMq^n+eQ9F*l@N# z8&|F)vzyI7Kp%I=wNeDak_=@+QA(0fE83>F*kyliarA3^+U4<$%Zx^)0F$Wg2-}tA z>;o)QcfL4^rT{~AeGF1^H5q+`bWYB!e1IuFA)ae+WfDK2~TesiU(wxYQBVX?#hd@D5jRR*C4O9r4;-4I^ePM zAr_2&Zq#-)qAsR6uXi+x%i=oilhDDfr-b(DuIF;D^-aC}ZwM<9CS}`#v|ie2;rHZY zf2cwz>UwPtS0($HGNFR>Eo0(==PRKL53`Dc8ep;OsoUipY&(e%CZ?9xWU4eij|;Z= z@8at@bU`~`pP_B$e_?;&98Ikid{$Ofwf{i!uuqv8nmoq3w?xxlz{fUw_|p2t3zbv} zV{VQ3fbZFsL+k0bR!JTToIM88Ee)7GzEH(SRhxdTKb}8$QMGp6+5ji^Q30k)ecrHW zR{HxT^Mj|a3x_e4jNbI|?0Fdu-4pZ)2}tlR>UB0AA~HZ}-yLg<`nl~SDP!q!O7#$< zX>a5z#?k{$XwjLJ#F9BEg}6TZvxNmVUhO~x$YA-y^Nd(9*=tORmgC!_0r|)Gf9wFZ zh5}p=q$<5j^kf1tvjBuW=N}Mg3)ZpGGx}ZN{DUL8h4Wapk>L3_yzQW+R13@rP4*Y0 zPz{#9R28Qb(#=;rjx|#ThMTBun8}O6<90>}=O~afGb$*6df$Dyy!Q%E;E?loILn78cNL5u8cdi|5Qobr*b@jmQ~rr`ivmo-tRuLc0vIR z8y3p-ZZ|6NPGYNOQ1Av>{T`W6javu#TXOP`5+t8`-83kvq8;hEkgAsbqN8TtNQw@8 z?sXiwB5xWxyEO)cb5=1s1!z?p(C%HyL7Wn9v6Df!TA7Wnv> z2qf@eA+6v1(1J78<>%(D*>u@Wb=Ub9PMUFk=~P-LwJ&a`is#3L(Omb)re<`j6KA zF_Ti!#r%^uzUFGfcDP_0a8H-xldE6Lk`&2SKc7J$wih8j1<$mXU`qKq8zsWbD6p0b zI`bN*)Za8B`{|=;NWlxi(Q3fCQ>cfCJVKR%1Q!TgfQWZDu>il|U}0nV&A|QHi~)K{5JdE6IvE&0 z{MA3cd1ZHy>{ff}<`_ZxcDNxgG_SnLIv^DC5}d4zC4a$ zRd<~tXJNm$j3)D;@=4GFjCdN@_x7V})HT64d>;>=sw zI;99+KRlH3aRf0>3@>$KPWSN&N2z&+#*jP7Yvsz>AoA6>*tkqCp4&eMSl^Ru5N0Y( z$%jv-*APvBqjTuQ+2hlWHWGTOrx-jbj$t}cOO}UDM)D7rm!5q}Onf-eW}%A# ztfg3oW%A(4-776mpwD)4oFma9wAKrmt)4cOGrNhu#?hNHL@;~iNPNs7d0rH!#l?_g ziDCb0#5++cE98qkdGxcj0kQ9;wp{Kij&N6&hpDqY3^*co!|**H`^{TXznQaanoYo- zvu1}2K5(e5X{K;!$!kkPs2mgfxR@!%^u`i#`9ee6uE%l<{mpXfb4!K`LJ<@s$Yz^nma`G$Zi12w>Z&KP9L{_ZS&r^QJ&%a!Ii$>da%q3z3~&*pwnB0>wq=X_kn1@}|NU|y?kA7^h zvQX*6>B>5!OlRbLurA=AC=0=mt})vE6xKPLJ6D}dIJf=@%eN*wrl4-#8~=*uGS@n8 z_hdqH#9MjFNXfkX+hwaaTO%_WYm^9e)Qsk9bd-yQqv!?i>zuaqFsnFZFg<<|UhK1T zb8c7P+K;mJx+aLbwccAl&XZOceXw77)t`Pf6dM-(se@6gzRXVV7ScUPStWs(GXKtr z{qj~heWLqGx&hEBWC!iefRG8;Z2&f>n14r;KkV5b#sPTN;${R134_oJK$qu5M^+>` zGbf9ps$^sZRV6Zh%W&EQPX*nhZ3qcMvigzWeD6{ww*OnOx5g`)ob%>r=Er&Aq#5TTVB;tqE?CWSI`GeKc@(`^~YjG zgg>&l8mPKYIIOzm_d%SM${R_2D~|tuBFB@H>aFcrHYxZq-SVsD^?XEMp@D+tz_$#> zdszBW&*OdTH`2-y!E?toz9i1K@ZCEvl{HA%h4gan9(tJ}y$=>3Jp!?c1c+5S7RJuM zC!>w^N$f01ennl9f1cX;|2n;OnSRt>-=E(e(SKQg){p()x;gt$sE!vsZi1vp$Rf|L&DmdK{G)d%Xh5*lul-ro(u~qB9+kupkSRC5jvLD0oP``O zGaF=qR&_GQSQZ;ja_8gIo*u4>N9D|W_dfi5374>{`&R(=gP%GHq`v|^`|9JurLP8! zJUJpHxH%$tEVj=rJgC}`Vvc`Yo3A6MSdqQ>eDI+>cPs7_?&dr1KH(E{ zHhp9kxOP7$KlB6ASSD{fdZS04_cuBTAA)8@Xm7dl2unQ9k}9?02DvXGLGX&l6$O|FPX|}K7kiZP7oT>;$P9e zd+HqXU4D&gQ*^_&%Rh2Pjx4UYaOVBq%q*GX(7WBP>C85y#%D~?0yIu%(GBuo-@LnHA2^4mO_hlxxCeAsh)l3mIqk= z#J>p@5MyCw23B)_8OyH_{NFd;ioym^y!fzvfl#4y(&$lS!Q}-qUPpT8f5y_|lgD}T z(^CLXsM5-XEA}vXsG48WyIk=qSXL6eh9)U?FJE*aHxIA{BAW=AOb}=KUZQl1@$?LbNoy!msmBAii>mxL5AU-jp z?fs9_~!4Az*9h$E``_E7ujK3P3Ro|4uPnkRH^Z zJqmkc09e}D0m1Vh6O`Ng7C!7~Bbw+SX;M}HUFhF&h4cIp7-(s0TH?v()1&D8A*Mgf}wDyjpH4sh-S zS|KwtkVg3L${4bP0Ks^=z@qN2?LmOc`(L}N{~xA=e)qwx`})P;3*`QH|9H<%eh(Ok zvI^NlMMGASp<(D@grNzQ7MB>58h9lOjL?Q4m1U=V%o64Tv3$G*&(Y3(pgseJ&xs~JnWD{O} zk7!`hbY%u>Uf~-^j)~6HRLAmSVfzh|GRm`<>wL%-88&2#%*Hs+W7MCRoYkwk>x(ca zQdsIRU10e6$MXk|J6?Sy71^2Q=$hURTi`F&UOANadUPHO>h9Zoh@YGFW~8&sj0HUq z>qx+_^<{kAJ8XcX_zvFE^bCaebS+Ev)^eLY0*bYA5Eb%z3-VvsF zNPVw7ybWFz=QUnU@E7pnTUG{9=H;yy6mK1oBF|74`vwrzo}KaVl}sR^3PZbHb1L9^ z5FK}nDitD5l}!4p8&9_S6X1oOB$muxKX@CHO>S^$Zla5qd6v{-yY(Tp9Wh3?A~B1= z%Gh%I#eK4*7azC}JlH9*)C@0}RCLG|!BR?ih%0a7N46vpzjX-DBz9bycY-vEn7`SE zjH<NFW{O^W zHo`kQ17#$xl}tI!<(k!OAtTY4@aY-!SijW>BHFN1Y(aL0QEAW|)$uK4IhR$2j0;SI z@nZZQa=zoY2cjZ~>(4H_KG+Ow2Q+JdRfXR>`G5TYvP%0WP~P(U&q`{RGr%(KH)Wyv zDDm$K#+I}49)-O)0uNiq^YiH<338R64iXrG6HgWlZM@ZuP^XFBdHC7xe>`(t*#8{O z{?!A%)=^joWQ{t{#;Q!L_Kv;MAF3}Qjamj{vChML+$=W)A>k4=fOlMddGPbr$4h$p z?~;_bqnHi(?NN)IW8ZW;IXKCA%Shh zFFJDXfwVsh%IM;luEqPl=T&+8vRbOeZ}z{R3U5%9zK2&T?@+50zr63C#-7aRP*iUp z9PknJ?TrQION<|q=7&??YbHDL`VnWSMBW90x}so&2^uLd?iuUN6kF?Y3b$oI$zo!} z4^GvGe^MWO-(4{mp1OC$OcOF_PoR7LJ}bv`Fxa56p>?+kzQ2i|R`YLyBSTop4XqZLevx08ID3p^l=bS>l{FS~1qx#F( zXd*w}z)?XBvPuWJOr>09;dTZr|NOkcP6(;gMnKJ&c zxlwgk>>nu&Ov8uJ5AZTxxFA35I!0Y-WhZrg$wIF;9JtA#~eQYF_^Gbw;h>^(&q)iJq7pf9$XEN=ULs5UJH}o_6yGTRF^fMmbwG^}{U82&iJf=jlgcro2@62Dk6gl_K zaM$;pHo3Q;*O|bzBJB~0NtJTnJ?)h6wZ-<&_&5s@1CRSE9Hqc*z#okI^$L`i!Ia#u0768_t zI}dysLoUZ+zonl`JM!4SIlNb&t24R~gYQfhS@G%=hCVCItZHR-=ga)GWfNl42qs%> zjsW*t0py(KfG(=q{tpIp*c;Q>zJw1M%m~-tn<7fY&B04)aS2(wzYm^wtN2)E|AgBU z!7rYlnQ11!bKEH3#xrqLcPt;e=3c9kdBzb90?`H*7c(0EXVpT6L}KfaruKBl$q)_1 z`gnnRiqW~R>{|LbDcs}bG>@u?>YM{KS$oyf?Y?q_kXi2R?{nG9EW57>oG?8{GVT>~ zocgj`+;`DMp7sE@DL=X`PffqZ{-^YEj}AMLHE*a~N*Ku!hXccA#SS8x7)AuSH*P}9 zn=jXc(>$Dmw}@>@8T3>L4$2139e20bK|lirQf{;T>Gy%{bO6QcXxZER|Hw+Vk46Mv zrvFtzj;0g*E+GjS?0kjuHsU)dgyB9%uNdZ#QkVYckaV*yRy1Ux*bX>%tdk*9Lf(&H z1J8#%kcRxrWPGruB$?j8(y}EXkL5f3}jN0#WsF8Hkj?Xg?eKAF>?4u~wFlEvUa~Sjcl2 zZyj*e={|yaTGyu86Sq!yX@VCdWv1rSLNuCG+~l{FSIx&;9V?wW%$0u<&woD#~ZkaZFa6M-mZrO zWv0nq;l?UP8(b|j!auw#285W?KnL4J3$V_n;mBnV9VEIL=9nnTK^;E*Ddi0E-SrXp zW*^HW7c7E;yduuG{KLOZ%&u;z?tiRST`Yd#HIc5tlAo0dE3@0K{r+2F#B;059Vh>pR{Q(L9V_8iomY(dsfV*4eY2?a99Op1zf0#6xUW z$JwYIC@k@yS*5)NZeQ;>MOK!*aQcwf z37Svnv)f0$i3u^<#Lg`;E|}JhPYT%55TR@2mtv%=dl>Uf2#+vV8@C%i^ir79xbQTw z@}9Cv)M2q4{%qEC z^)HAqD2eVSD3=LcUXNSFkAD%2y@|B@VOaw|Gi$ zbG4%@V02%AV-B1rxsR7=?`kmOd{sP;-(D(mB+`o(xU@HE$%A1#z9KwYL zbf}?RnsiqwqqX=}wdFrZlD52KXD%h7?Tksv61YrWhWF2V$EcS-k8F6&-$_G9XTU zc(~cjrny8dN*CYTf4;DKtDA}Kqg)XSSmJk}6xX@48h~L3po{#c6t}W8GuHbn82@?} z?AK3>fwwMx?UUaWeO{ptvd~cfjkZwJh{AaZ;b|lu^&)47J;uX)QY=R8@K{XmT#Rd} z(MoD+VfrE1aM$VN`Tea>76MWNd@7&%(3VyOj8M22*>K0X?0k8+&oI29DflRD70`9M z_^YL1->3cfzN%74b{~w6Y(rg~-TmY#=LZ;_!Tl8zXUD1b_jIPUEDP}1KSW-RuG2l7 z=E-atxG;qAwh8}fo=!}hrs2Ul44O&O=AWQ`Y2P$0ps}E8-7iq|7_SS8=i)_1IIC{*q2 z%VFOaOcvsJ6tJ87B$=LPgY32TaG+VxmWOSt&{X=wGxeco;>0$`1D$+{~VcU7%gRnPMj@!IoJma>>8 zd{$Z7X}lfl8;JJFI$F93%&u2YN7iss6 zwES-+GK#+2cHf~Oa`h5PN)CPpo^6yM{8KVd?1WaNK_X90g82TXPDUo?Z@9&OkQD#pNBLh|-f!UPKMTccE&l($mK}VxB)yd^r97=75(1yNl6BYj->+fj6Z?QI1 z@Tbh(5Fh#N)0XX1P%%T<(ibukFxkOu{S1z^v9Gy#cs9Mh)sO?XuNUcNqyYFBNKTMgsWslo=*_ zKL7Gp>SJ2?{_~Xjhu83L>V*8`yx&#UcPaiqgcSbkU->^TzW?pc`>%eCH~t?DG=Z6} z63lhCK(%DLV9Ek#+DK(EF@?F0g3s5FVqn!Yx*SK+yy*!`Z^~KyT&khatW}$xpvHU0sH4gu_EaW>|{eN)OZ;jAz zUgdu(I{xosO#h+6_n$DRf2E!N071ax|k8fFGj(xydMcS~=QCfXuhf z)OT$;lSYXq1@r0a-bi6dA;?O3LO~(;KxwJvWh7~pYE>1eM(F7z>7{7tCKUeXz65Q zc)|N&x89t<@Ad^_q}cPw?UmRV270;kic9XG_i;KPc)q{!6AJx4Qm5Ets}R*$B?#aBy8c72T|e$~ie3ZB)Ca zTK*?(Mqftr7$z@2W4!(^xtxovbKVv1&78j;J{%kA@0@8gqI6W7sH*gRR8{KU;@umR zHHa6M>3656!Rpd8^Fa5u>33vZh;ueGUhc&vy<)Q|=k888#VY8QR)FExQCq$jQ6_n9Xw`P2Eh^EFmx& zkHH2RlRK&R#YpT|r{8nq#X6JB=Rak86P?S(DzY6<-_|YP^j+ewR&B{?Csefd%kMfS zx&yJozV1SKLF+RMsRI_!ll=9e(fbRw7`_+_08!GIJYJo<2Z!@&S+<<1ay2SzBig2p zpy%a_dIi|^8l^u!=4_yfla5ZV9FINC;xdKOx7@Cv)b83sB3B35!@ zR31)qOVSCv3hFFCtY|A1nV$#|QS*iK2#EE7j)-JWtZAvB@A+nT1_WvVfZWAF5Db1j z&k}L(jAIf7V)gxew(3T%P_{VZp{3=b@H31gRnjao8^rPw_UVwxxMMNT{K@je>}Tjg z*6*9;4J{ek8;n+1AP#ZP`x`2aS$XPse-bGVwKbP6hLE$RX3z}cE*9sEl3H&c{~y`C zk;N{UjR`qujvn=Y$AoZ%-_Q--LQIUHoTW*m{&vlBK99KJV zv${bK6(Js7uO{z;p%{M{1HC*Bi^kFOZmbhCUopB!$Whd=S%-8`LNND4zHr^_Z?g+% zvjVM0#*7roEGJNQD!4WGv{BEzLC9NFYKv_AVNW78Vpui$-1jgm;k;<9*bmE+LbGu^ zuR|=Rx06OPj5OhAMxdl=9UPdQfo}rp`UpA=D>~v9A#s3Is8za#DS!3g8kkF=DsLiiCao|ryf;k3%4{(eCqgaI#>%v023?8a zHYdzA#yJqsn7aCrSy3u0=51dE= zW09$*182z;y*2|6b+*}^Xnao5IG=5G_{uuoUb*do;+Lb zDLA-hg}YcU=koSIM%)Ixrls?dRQDs47S7H1;-m};hpF{1Vp8W486olhv^_}f5(s(} zlas4}v_C~Doj?D+=T4iqxK^i@BvK%n;jVvNPv)q>*vl?xIg!2xPhzPqUSO~U3cAl9 zDMRZ&Qibhh(0knFv!?0BB_#IZI4;zeVra^!WW~84m_-GiTv!xUi~`lj1>&ca=<2Fs z87}|>JnE~?67i*n0F9ef@EfgI%3fe95mutJxroP;rC5hl1^L&mm6&qrkAVfVXNz8} zg#tX%qU_$Z9}f%RTuCa5;d&2{M$0)+BO%^mL!=(oX-FV_BE{i&mA@K;Sc5vLaD!=o1(*p zD&_ffhuCSMEg88j_ej!uODUpB&$3c?p>PpIr%2_R+IJmSo;rUO`S5?DQX{hc1orIu z>(G6ivPYO;0?wy>J=i7?zX}9=HQ&}IT|imtM)b)QSve`dAjI1o)4e)SFJtTP71(YB^jwYo8W5f~dQt;T3{K7?`qIMVC z0bUN;|FP->7b|LXk${2jW)e+Md_bzQ6)0fuFpi;q@blg9^;7B?8h6+#3c)pd<24iGdHTWp}W?;~kPEwy>ejd3khj7Xlx7O60Lx5QOrDQ0( zg8E8TKldkWzYPcmPZruvgKeg6#IA0Nr3Sb`@LekfMciG?}Ks*+T;VKb(1E?i&Ub{=ewr+sIPijOfZ?g0o2} z?sr=yXN@4EOTnvc$rfOz9P6|kY&1A0;$}4XxSt~MTo6InlbBB4c^q{p60cK88`>K`QN0(VcpfnoA6VK`L1D$9i?nE64`t_)WKq z49JyC^7TB>pdnA76q+_93!KCUQ*SbIm!v-3FM-jm8TVggT`}N)*&t^05DRzQ_={7D znQ#XL&}`>Y5vQ5wgPOQ7pyy+AWxqSb$)<;-6B$W)5N4 z7gM?EQyIHurQ0V_XDxj1iQ6xi+5o+__O@+tak!~HEIiof^>pS8o{kEzeW};!VF)Mh ze)6k=3*&yI;*JOF$eCVoW`u-O3%U0+cwEE9Vb!j$MkVCDs~7!(FA%d#RJj@okO3YQ zcb|8~gs_@04G<}8+ZwbySo#Y9PYpT!jTpapB$`1=QGqxY8@1}3GVlrxz`c+XJ0!iY z3HbD{ROVeqZ8(6X`6M79vv~e`622IXA7h2u80=MPaa=2r#-vcYhKNL77i9<+jG=;5 z1`9jfZL1+H0r*_MuK1XDojG>;wSL_P^YVJ$-J0ClX`jEG6=3{QPLeA+V91j7O5*Dz z4&hH?9iBJ%`tne2eh;3cu*`W^2w@a#MD-pU~^(zwnoi8TQ>E#L45myzA}7uOze-J^mqgSbYl|9wE1Lj`=_%tp5)M*sVT06|Xx+G}q) zgs^o}FXQZ5{OL#*&utz864gXRE^xKG?kZUvnSpBbi!f9pn7l& z*2P0jV7#GR`DUcEu_0J%B znB!=Z#(m4nJ}|T-N7+i?h4%reUAU+HDb8OBPWzatS-;dKb`DK#j8Oy9*|--O688&! zN7#xVXC$u7bc?lb(gaa*Je&=)wf0k#B*6nr(W{t7&DFh=LMZoUHA*sQ833$IY4w@2 zU8+K?QndwpCc_MG)PO?Gd$7mC3QKPgU<|VSD4hBC7!fx03@U!{cO-KY$Kwu41a!&i5R|nLs zm#{_5u&?5A_W}=15x!A~{vH8OA#@uNCD$h@fuP1qkL#gw^H|pg#&5;^Ek(-$;rsKj z>g>(p^@b8FJe@8=RaN=~mFMH3U+qvpV9Iwp_3l?(bw?fQzG}ADr7gWJ^m&xOo&l}3 z#0;GC{>EDw{2g=_`wHN;9uuMtSBP32<14>jKI9zVy}$UM}W5UDSF` zJMgtutDMO)E4^C&9XhDMZ-TP~dX>_!3i@z{L#a>-2rVqCKeAU%un&tGm10U=Pm!#7 zA4=R}4N)G%rcH6Zc*cKJ{d(pw$PM1qxqgrg=dQy9oM4f5ya)5MqgEn((~Mx5QXEiV z`g0pE`(3VK0WvW+_&Te*6SrZ~IH_}gf0FRgiBPguF)F>PM1@qzI%^SB>V(~a-~Vzq zcs-^6UH?nGU6DyvtfXUSr9#V`$@PbHYoP$i15h-P@pN?KjFDFhi~;hNnJW)aXKD}2 zaTop{vHf4kr9EIY$0Su32s(R~=R?R}w&d6u>%|IKb^F-4Ph=lI6)qrAtHIcQDw%QX22I9FFGWT@09B!2V6+vexy@c~Dr2q^}>CMq;VDaV}SU8hKgtR`{8B{H3vXeC3o1J^Be zf**HiyRCC0$@0mxNc&QVVQwg?L^?$+Eg;#BZe2sVmh7iS$u_PTh!Jm^0?+zxc!RHZ z8VA;JfiB5eInsG4+G`E#6x5(|^S21Bsgc_(*?MK%;0l>Z`4ptjH6aORx9N0T*}CSi zFKgi%CB^Q$^;n%(LP5`pf(+r(uqVCM`DG0w?vWjTGX|nvq}wkmC+n%z_DjHP64G@G zGq;P_iiMTvmBMw}S;k4s1f)jj-N~Mah}L-8F`!|kb-M%NiX>bd;F~lhusk6{f|0vfUKtZo6_vHv+i{>$juw zSud$SVOHBgBCRlSz!`oNH=tsoh2yB{2kf;+;?ADn6U5XNvcl5;EIBb_UI#yo&7{9Svl^4lY5f+zutTaYmpqdODE zXJDw2Qp|69zG(Hzlw^FLvut%fDK0}EXmU)w`~apCCaiaUA;;M2rU61ZDx9?QE;P7= zFukajtizJjU_B~*-N7ZtAC?rRKe(2tTGTS;sWNG1tW-O!nK9w|JcYA)XL_jN#v?!J zLUo8YBcMOSi}=xCUriADlrtLh#?_EZdz2|#5ao>QH?2d(AyC%t{$=M)K zzv1sPx1 zp|-_6>^4{2YkXfBfXyo16seuId1*nR?k{QE8W>}*yG4san_PdQQFd;~k$7m*ersZ+*)ijL5*gR+U8Q?#yC1L6FJAv2|w$pBaMiO=R0D zoLr)_dA{u?h11{6`QyN(d_Lpi08vQm_-O5*)lfd^WblN=JvJkqKixuvQAtA~g{Y^j z>TfEbAtxD@g>Z=W@^`!IT~03^^p_n=9(6Ff5|rY0!aLAFFEh%OYJ{kw;;u-%bBi&i zlU0%P=(;A%nf}ty=AI%s{>>=$E0Yv+M(~dP^p&xuLA%SfKS`wWf%Vyi=F6e@HTMFNFVox&xSj-W?v@?G|^j=1Gzmryr9Sf5Jg4;RQ>@NqOu1 ziBB%K22O&(DmEjLR0KG$^=Of6-AY^h1=ptG{Y6H!6VB3(y%W+pRIq+=q9vrxY>B}nWl;&vvrtlD<6Tf8W|e_Sgv zob17W*c?pc=S56JE#zb{8EGEi-5#L!6RaLu#g+dLpk7T8ldliy!gde6<+YzXEYO}wR&{C?Wu z6Rk71+|$+f`jAJF&#cUl+e`(HkI^ZkX6H%Z`}1mv2c}!et=?uYC(X4H~+AG*bb|O-h2-+hiKj|{>QGVBp93X{Lc|O_j z^a|tEFX=+~OAV5{}V0HlW6*q>;m2W67mFFg`<6`55jQ(69wBzqS5eQTwMGqHRm~f z-NpqYhAk!R5zKb@;|F?Y-eMlRsfv6V4CPfBOM7E2ZPez5{ju#N%fLst939}ycEG7* z`dy^$X>N)Yj;Jd^#lo+uo6v`kDn)oNsL<6&oZ-uDxY z9*3c(&xe?x9=T#yNMjlZd6`9W!#lbg*B3iD9aOn&@tq91TgVMJJ>oJG2VDao1{%%c zezgKb|6q?|hdoU~-Ht!hhWS8K#Ea9$Z+givz>1dbFUjKt5p%#4)qZE1bmo-SFW_jW z=@j=1+;lUa!prtUL9-3VZ!LF+wqX~QeYducSuWuP2z~6agmw&Z9Am=k3*o;G*-;0y3V2X}fLy;N5ApuHS&EDEx90Qr z4(tWoHQP0gM_zqF3W}lZx+GV+s0|>zUu3e}!0BumX0;m5FflZW&1CiaNJXt;@xwlA z^2;A>9|^o+g(M02baJa{bOgxSLM!L62s=|zcZ&O z+DW3B^IKzsWB`z{QwPz`Vo1g8D#>FE)W1kC#jkXmw(mws15u|3q~2)sksD-kDGPT? z8AMY`{G5pNiLJSk^^XR#Puy|c^o}`gdI_{CW0>z!cR@ixzr}*l+SE-piN+H~t8)I_ zw}!r;Z+2xQygY$z;S(TQIEc8@BQxJpNtd}t(w2y(mF&0WFnH$jY;SCktj&5ekvd3` z$;GG4ry(6de^OopV!!ex(xe(YNoftx7`5N+3fu{3+xNLwFiBDeWT&u9u;BMGGo+%*qboPjYRS7pSr`{X*1+i^@X zHcYf=^+_?EL3oh(k8V%jCkG>*USIA~j%ublqm}?unJ0}u0YBUH6vdECMUr}d-J{>2 zry_Uz_$p4bDiB=5HQ1z^_>NdX!Fk$csi$CUo}8%z0Q&AJ>gE7Ib$LXVn|}x4^>oi% z;v9kKml@Urrrf@o`Ent#UjwHjD8I%`fpdecSz39-<*+hF-!5ffSovyeKz!#Jj&2k9HR|HMPC~Rc)tN zNy^}e7z2Mib}Arn=Ohw>Ju~TB?;<9^2)f=6Cya#z1vQu=?`lsJHqwy23Ni|bO3h}q zhpBTs@r)qLOoLaQ8TkovXxAKr%m_RKQx~<|2xV+IW{x65WJ=zf`v8Q*#?S~6?6%k6 zse~}!K7FP_mT(=p4er6LoSLbY-R&3bYim+oL^pIPs9!@yyZ>Eyp(gNarA$AToNTji z_U<2vM<|Y18h=m(*nJan-Be=n-kMOW;d#>7YAAs+C(_7+;i3 z(7OC1v?-{NsCHmW9N{pVgdqA-an`)8%;RBT-I0*i`cn7K_CxvJy}@D9e|xkpQ{(g(}Hd!hl%O+wI`h9O2p0tAF1;f(zuD$g#tfRr8|?MnnS&Z39knlX#8 zCnG%Fw0f?0_t>~dZ||t<1B%Q(HfdszmXL!U6)Ou21vDS|i09zz;C;&eJ9Mp72b~a? z8t|)7`xW5hI;}Q8WfQyJD{rA9C`JW-0GFAR66uI3IdoU~Gmg35V9C&*vgGislBivJ@?q z)tde`t86o;)qqzy-I|5m?(;Drw-+c5Kg&Cp#u3dEfje+rcIF)1G!{%;dpO6X-7q#& zkb{EkuD#lNy2N)d2A!ttkTNMq=TSS~OHh{P>d=(Sk}{%2&BhGQ+ibDg(Ge=jpn>Fu0b9LO)Q21Fg|*&k;{>HSPY`!7pn3<92nw`MMe-fG zE3RBS2garX5MSOPZu3|wqmU8^?<5!I3Yj(Z>LqNb26H1HDs--tAMh8LE)PF?rKi?4 zjE0p(~R=j@)5N`fzaWF}XGEMQ`6(`ZlveO8W!Q2!%LA|DC=CdjQ zRDhagjkh0=Ze$Bm%CYYdLx4(Z;RZFnHJ2Ow+dm1|)>WlX*z_{dd(^qWTw%ApiN_Y* zgiVD>1oSG>S`(C{gvgB?tk)C%fqy6u*_Q9ehA|IoKWJn$+IXfB4O&TMv_PYgN=1)c%eE1H8L*u$(`B@Yc6c(w zp6a$`T3(i;xTcbXvU8ZCd=zfq$dqeO)r&U67Hdb(MOrSL7VD3}d$>T|(O&92UwnTX zR9!PJ@`>8vm#A+8r{sS6Hy5`3(knQOfg3Tw6?#rGG`pD_Bl%sJ9U_;3EXF#~AZtSgbQlQXJL(QpMXD<<*&Z_T zEd-_Bji^CRD!9f$m<(hlHbHV8{R1-WXVG5j9ksnm%))#%Np|c0AM!~Gj72VawUsEY zAhJW}5Wn8L^>HeUBb*go{FWibd0b+ji-d}Lu8|rrc&J**dOZFXkV-*>LT-hr+;l?M zUyB<8WsbC%B?~9UthlUWW@F3a)43;^^?ZUW`1KNM1a|6dbTC}UgJ58 zjZ9$mKT?s&y{@0@tMkbv^$_tQnIl+*uiZ*G4K6S%aGlkJ&hWZDikuG|qiwc`$oc3Q z)PeG4XLmLQEXYVW&@(b%bn*-6yZ(pp;c=RD7nEG+Mzmg%N(K zuV2s(dNRe7ONO(C>9z`1#Temvla#LePiHVv^V$qv&}114?B66{iZX>nt|j!!Pn3P4 zcxy|a!Mv}>>IOgw7=zK^)VNv6pCB#p(gWF_@L`NQ`DeJIMw?D!7%f6)<$0qErCY2Y z-nMW@)*L>bL;3u#&cZo*STeB=+xI?86-Zj9tD~&(b7mNM0|KN_YNZIA?RgwuA5Z)d zKVcsz^Al=LRcR4aZ2|}V{JMrPu`<8s?26qM;>?~6d`xN4mcpVuIZA|C)6mw;k4zp2*zV|rA*YsKo zcP?Sun1wH=DHaCFPu297A6=u%$vBA%qZ~CGASAZkr&>$!`$827$YxD0q4G;;5Srfk z>N`A9FS&GDcPxvN&Wrqv^M!wvzb391QV2L5q%$@4o%7ln<~p|F8lAJ`;suV_AS^^c%VwFZ9bt^u=?a3C>B=YP3p69nL`nJ;e^fO&8~l zi_WKl&iPje(Mu$UxYqhw&z@*}{UKgd-G{Rz`4%>vd-Cu&#{){$CD(FjVr&A7>t7Hl zJ~XEwG(I-kwc0G;_HvyK@^rNnZ2-KMjKa}`DhfDAGqU>uoBGl5Om)ZVP$tqixQj-- zrzz-Sp_Q;&(Ixw0`#jXWc3OE%Bb#c29Wh%4u}xukN7jj=0e7xZlZB9I<7&SQrYRGD zOjH(Zm4{gzup(nx)8h~Ku?ZGUdM?&s^U75AVzI13Jf-qUx*1_}=w>tY?1~$$aX;Vk zrT8?Pb^JSqqk6A{)n;el#;mr|3t;#NICI*b{<>i;I2>NJ@#XfqL##_XeE!cU=JPI< zi{IBi@B>wO67E!R2O_;{s_Dgejd0i}hks;6}!n0=^HjEdt*{6v=U z8Bt)gv^4-On03C*iq-Z+_U2}Dq2Wa_5|7IMP!#?}L*au)pUM5Gn*zv8-=-Kipm{bn z%;}+X983#|*dNIA&E`G1&HQReY~Ur4uz`NDV>DXBm#m4&Tz`Q z1okkj8HefL&}@IFY~z%eWC_l*lUL*~*X|V*F#Nc_Ws^^C^PTuha8ro&35Q@^_@cO` zL%0E;`{=ZNQV`Q!x`6E5Qv00ljwg$=VNFruE_rz#@0Rb3svrJ+xcV%7VD`Dv}6tpRUId2 z#s=^`L1)z+NQsbFx2eqBFC${F)he>c!A&Cf$DiR4a2Z>NT@{4p)yQWubxAhZaQ%|^ zx^+>VlVTR%0jlot`z&e)85)hlva)_6u~IWVyxMSXEpZM$ef@<0f?l>AFex)^t9IoRv{(|%gz z&1YAuRP7osqNI;|vYPu6Xo_3$3D7COZm2D)0t7m({O%)0uj&oOZtZ@B^Ie+k{(dLf z#p(lfDO*2$FJTYS{j&-77v(~{aXF@m!0SaZzFv@KrjaY=`rz3~k$0o^s4tPabvNeQ{f-lHYUv~nM*2J?^dKM5KPS4YSt?FLix zNK(gI+;{@??cUz~Ac)WE?3nGPvaF}xi0Sy(Y@TLU)V4p2>~IesJ=(Vka!E5gV!>PH z_M^;NJ;FnNeWM==H7jaY_226vG!{}d>VaLb^0i)t+!QWlSyml_5$tJOpOQL6(1bU0eb&*&tqk83esA+9D=o{VlW&j^21(==Eh|N= zgkgu-l?=65Vcg%P3i)h8H)C7ZzOP}c6DjBBBwCk63VNxSfgT2{l*meB9GkZ?CP?9@ ziAkbqiT&n^y@t`uzRIPP-lq`58JwEX=`wQVh@D78rYV#7_mfH@nbL$jYOf*6n9WYe zR0oD&nv;{?P`cBmP&g;EMJ3n%3Zdlv0!R;6?itu`A%K!Vlp)ptxtAqS6Rd{V9R>KM zR+P3g#X4d>V~8?KLFyElDQ`UVR!C}#9f^zOV*KdJlX|G&Gr_@cwD51?*Zm$A@hZZE z@U3ZDk-f>SSRQKFKVnO??oyq~)RO!i)`K4xIg^iisP(NZgiQ$f)sWlRSxn7ZRn;LisYz_f z_x@P~dVyORZ*HM=c6_tVeCSd;$J&|h=VpDE<#0#SQ zMf4nG6EZSg_czb8mMwu|xL2|r1FxktB-j{HmSu5;#u-YOvn;nRO%WJb`bpLJIiz1R zFbzXz02@Md2vOU^e^C>uB|a2@{8kf*`u1$LfBWlwb68Amzqd8h%SZ}~$|;MQDofj~ zF~D}8sUvb$1!cgm*{}~PVjITVj7+uT<7wNna3uevJiEOTU-e+YoX$lg*a|9iIt#4p zf?N|?#`#_oywl}~eY}*W^`ytH;9+62wBq)aRR`$lfVWzM+<-hU2jRGoE#Sp{6XS-p zOf{72=*6nZF`LFLIWyQe8QZ%z=*m+7E6!4|Rei0w=`|kOVp*>R{Iqbk_MAxljHQdO zllZ}yQ84>lIg1ZUnRz_3KJ__c-vgU29{i(05MePsYg>j_)~sBVbRlQVd7*lapzxQ8 zM^cb*JH@JH=0YYfA0Jb>L@~^e5qD;$#~S3b3V+WMz5d@`A#QVj=^@7p_#rP6(&jE# zPq#pJ^}Hamz{L|q->#bVj95aA(pMDNRgYh_9SUii_jU?9W}zr=emT5VGi$3V0k~_g z5}(ve&{ZeXFC`8RgKfr-3<)glrigb)#fdE@9w)YF9CpNz_-2{3?p?Rw7M(1Fs%FJv!6Ka>;IsrSAFSA0@4(y zuFr9#{C>iNcbg-1jyPu7NcX@D&fTKbqg~&l`Hnba1w1EFE293UBxCm=0v|HLCSNO( z$uSJ@u?$_Jz6nKYobZPNC@7(VH6u~!?IqHi5RET|SDB7^?ZuauplYLi+SwO^ zA$q8ID;oqVa5eU7ts7;qOE*Q(P(2psZ@_NO;iqles1$RZt3xnd>Z9S#l2f)ZX`c;4 z+rd*(H9c@>zktGo3JGd1l0T6=i5&il=yhpOJ=FzOMmP;M)W(i5^i!c4>7|ZSXBJLe z7jO~Y{pF)YlE=CxFqwv1__^%q7hJTUMWz@OH=NcQb^Uy|>7^S(;9abBl%a0gwb-k$ zX2*n7e~oY!kbX9SOT`LbsP&9Kmt83Fvj`@p0B`_GA>7wm$C~MdhoI)F-02zyW>(J^ zR;_wtCc`_t=>lX1isqbko*m`yr13#lz1NlsTs9>#nkANtQM`-0jY}3#R9-kz_-qQs z>j|UXLOG}aUBb?*+)OLy7)3~k75k-XRHlppY3LR0Zr2{EvVn+6<74c`dPNhl`~ejT z3L((Qo^vXDWNv6=jCyi}ND`{?qqXjVSp&Ewq&jJ$L ztkwLku6`k5az`jguy-PdQ!I4AO*QaD^4^*aFh4xBFTyOKG06i+8HPSFIaa+3GF^K} zb!)$?sVx#d<@Aow&CAJm_FV)R)bK4!>6S~}0Onl$TlXRf+GGhzjV2^FtK^o6=IvOR*w9bi< zOoNYuk+?=!*?aW@1NSqiSf~fF$KV+<7@#%}rk8}~3auZ=9#TZAwp}xsxRSkmN1M51 z%59EF)YQIdz*0n;Mun)siwWx*DvxIeCKSW-N>Mp5{v~r%6e9vbIYBEt7cXkB-Xw(B zLN>bW(XniS-BbfzhU7*)mVpHHZpWdQb&Sp7;L>Es9f)#(3|{w@*-XsCVf8a;Ggtz@ z(hNXIRvmb#_s46(S<~ep3P;c03I(i`{dja`M{dkDbQT5Y5eg1oOW+G?hWGiHkH4ci zD|j?R3S}m%ehxEg!|_u$omlnqyuc&|Qh0QG$x9pj6q`h42!vx=0wyamQj7wXa5-u; zNqGT)$-Bm)5@AQ=Jb?0&E~+7wGGt$PF&AN|+0hj?nMH!UTUiTX=A>$7avz;uiUu7F z{9Fnw_S2+85vs_}RG%=*F9VmjW;2)R5#BmUl%8$4!PdIWF)c{19NMjGDGnU_vTjFY zw@<66zD`yC0962p#vS)dyiVBNciN}Nlc2n6V{d~jMwBfMmeZOIZnjRP^Jm|}?l1Ig zLR4V&Fh~@QCre5# z4Xsc`qmeL**lvLiFe#`BQ>u=>!dXAb&uv!5^Tt?XoT;kP(=AuC5KW`zCKr;JZTzOm zGS!^vXfIkPHkd|X=aCN9<$m@$4k?Z^T)QrL$6i8^f#xUrd!+aW2c} z>S{J^W6EM{M?wsBZcXv0%vz+1#+Sy4M{T#uvl5Rst-?jdOv7ar3j4r(u$b8|+l4KA z@ey{}RM%OqVYSQk>2^*#DkFOxEuCt(vexhh@n7}Od!fvs*`)!w#}FI=MEm>&yFAp~ z)Is-db$-+h?2bWQWk+OWAMP2<4JB`KK)(3xb#?UnPu;aQWlt34IAI85=1En?vhO3+bxj0o1AV)8UTR=Btfj`MK= z=tU_LkM*r`v-}8}jk)hcQ);Y$hr8H1`h#Ar-$2||r3@rqZR?0@6+P^C8+c}pTqLl9 zh=-7c^OAu~ZakPpRO7ox06U2(h2?-{f(V`K+gG0ZvET8rwc5lKetLkVO6Qwb?Iv?m zW46U75s_&|AtRsT_xjs{l27+NO-A>@kW^P;v8}3p$_v#5%f6|#96A|&Gc#~c*`!(3 z5deed?sH%{s96L^&CSCevD^-cJhJTX=#Lj<*k>KBjw1oyKOFM-)q}}JtL^mYVY@8H zA)^)oR=mr#@e!>SVBjP$aC6ViBwxwhqMC=Ub-1f2Ghl&rFe0*}B1JcssorK|aI)o1 zEVycS$Z5So*)aYhm=-?v$BtrJJ#9#w^f!0=7m3vTP3KD^=!X7n#eKiocRX?5Gfa?3 z#Vz|C6VV4P-)cuG@YO)P2-mRqTml*V#VlZn`FOBYFgK6gvfK9xI_Vh9{Q6vFmto2> zx*iN%WnByvPV)4AC_bU%(Wly!tVM^7ji2A)*)#vn){G|s~8k@==9RT?{b|_Y+bJwpSc~jImZq*olht0Cs2M#pw50=IGm?Vlt`z*Wu z3(-HLxvp!nc%bUaVPnGd=z^<>^8=?M)igJhF*`pq7KM?tuO&T{x{qgXwi#L445tiHr6lIJQ$(GlYTUmV_ z-9)!=3BqpqMH{rB9BvHYO_6t^Xs)sGW6wkioQ{8^H0BmYFG8}BPzEtrGz|sas>{=zEg5Ku|1KYgz4T6yRcQ6E<0vDC)Lc6VfAfSt0pu<@11 z8hN8BtuZusVTCI^^yfZ#v1rkV|M93Qpn=k$Ho&7)6u804PQE|lz#&Tlx6h8d2K8A# zsE71ikZ8Ya`4*H|Oi0u_V|S>yQ=S`u$e>*FXDt_Pf`5*Sl8j=W(dqRrA2C=g?VwQ;98y zajR!e&Dyj1$L-8=g?xi6O~dAAW{=*LN}r|TvUAUEmSI+IW2T*1=0nR&M%?nVyZ4q2 zvhs@CxHoOshR0fSsuzz6jhnU;zV=lOPj{@*cX*KAE3PylHQ->rXOR}-omp=joo%(nN2)%prilY+~&i1U)Bnq9dFR=UP)c{HCGO`ITYY zy_z!5JreG{w>j$`ZArI$)kRx2%^#Dix8L<*L~!4~;%AifRyE(9oB8ZgZmh-at1FXN zt!c7XTNp0bvC(37k-uxvqU&aN{rt>Wk;+LUX3$5DtWVxDKYVg(=e{3OUmnwWxbRT7 zy2{1M3;I0V9>V4%1ikn>_~MkD%EXGbHKJ0TE&DzVifQl;xjxAJlJ2qStTXBw0aZzJ zuZ5;YExr1;#oXx6RewC$u>8@ONS|4Um}Q;Qu6Q}6syiv`cnouFcQ~s5Rc`#Ejk{hl z-`n2%5Trv>8`^6_Q{|nim^6#DP3{J!9G-ny>EYk)IyepFUH;TGEN6?$iOb=y2Hu)m zu*q2~Vw&#U43(`~RBgvo)+X6~uj+rg|7l=W59X(+^uNweS-FC@lsn0H!jff;gHkeb zd8&P)hnJ|0i+CKi`?TsG>6@Y_#YS3Ax7Zxq&9zed{qcKuZ?nVVD(gyosxRg6>?Z8- zo&8r;Wau5ISKf2|UWS~0_2oY8w{dCpBj)sVG7X-?89hO*9qnC*x&aB}8xm?}1)bBs z82maW+P1izt9nL{z|-|pmhkd2`uOAtV%8QM>}uK{Ob+L?AKp86M{2*mgOyE^2X#>@ zAE_JhroO1#z{NQeOBWxc_IfOF&VQYMI5p*3&%zXsiVoAKWZb_qe5R(YPV(EZ=&aa1 z5!`hFmAr0;OwYRCWc5$9nzwc9v_HFW6CR(goqE#eXimKSAhWcGY<-IolkiR?(Zs;M-oPu)DB$@K#NqQ+%5Z>W@wbnD_>5t>JYZ?x^;8Zn69C z$0nbSVW>Rjk7lko!uaQ}Ww-BLDBn>V8!(0M(x}PF{aq_?>7xaOGn|t{*#-|zKJD%n z$ba}{=ZU-{ON;CGZLh6UNzjd-HqXG`EJ@vEjate6hOzK{;Q5f=>3KoD7V1Y7jK3qi zaOQYDNAKE5hs6{2XC$O$l&jzRyVpmRJyqF5RVGHX4;x}}=w*Y^Ki1V2?K|sy&=Gis zz0j&?R82RFt@c{L+J44?v*S=k?a_*fgR6b~Hkol8uUOr^b;muhbI^uc-FJmW<|Mdz zJqzelU6iu>!j`h0xA$ilyLF5#yRXd;8+)X*k~%<(o7c-fwy#skmG)b9mOQX|Q{hlJ z&JIZ&emLT zqrjxrNp)?-=~LQ!wyp5jn`o)s{z`@0J$rMVA>DeIDL49#-b>l}^%;~`d~Ws7Z8;U$ zd#AIzG(WUi6SsPY`<&jVW-c4i`MRj7bfWR*61`J%*oMynJsw@1Y_xLZ*d((DZ!dSr z*&Z6u?>`2e-WA^YuyAIKQf9Gh!1}S1G&Y{P%V~_iK7(y;m}OX0qGP{O&n4;D;%UoO zCii7_Gq*VqXm0j)oo_j};fdk-!!?!RHxA!q>Fob|RMm?fsjKZ%(;6@BY;XuK8k_&< z_3EYh43D6Q#<$}S2gs~y-{^gIp-v=YUTUp^Wmc|o^N(;-mY^!+977LYvK{dQH6QEZ3LJ1+j^WB zbj-J8CAUD=R#3a!|Qp0DaoZCANDlEc~U!Nwf1UocGLy>P^49@q>O$* z8zTNG`5~gv^_vndY!Ib!_hz%<-+}SF>F%*y+X#Qui0pvhZQ^(D25uT|AW`>?yKT1a zbmSoj!cAi;DOn*p(k7zT0OfCTYtfVu8qjHU7L5UJqDFKu8v`wRA=ks5kBQ*o55PKE zHpUqqAPyeDPKF0p5tY`p4Uoo2*IO*rBoT<02$22T%?5|3jqq?m4w`)E90m9d-HGAB z!2!hhB})99Rqo*Q>*d20B0C+KffOm7CG+W_Wk3Tt&LE9QSrbtnAf!t80zlu0X+mSM z`P?;ZX8{OSt?+dEmiGuA3(p(5T7H0-1|7vi)Dk(Tf@fxz2pp)n=04oBv(8t-C zE0pMyyMlxk=#{qwN2cWeO)Ec~+b(-e`bQw-1>+DIe94-KI^Tnk8xt%gfSXi4{cC#! zlHlyWG|>-+fqdY(zm6$Bi<5b5rE z@-azOoZf__&9{SK*wW#TP>K_|51gM928adbN#IBP)kOZDbQ}&}p}o@j3c#O%^kpjx z4O~SO%Dsf}Mj&L!kWSM`heX0h69~;3cLA3NE?uoNd07BC7vR@GM>5m0CZb-og^3BM`AMbkkbohY zx1K)UpzJA&6!-qqlTsorW9C~Ia<-S#D~Z6 zF-3NWLo;wt9Gs&;zm13k3q<)q9a}m#T8x@W250jK)t#|`t_A2w1W?vQRH$Yf(Itrk z9HTwFm$Sh_15&(L>Q%hdAyVHIA;-g^o%v{zHIUg1a}}u*%9@DUKd23aV3m_vCn01| z;+5o8c|v9L+;~5jE{O%2mS{N6g07g>-L;9Sn1ID67=4Pfme8By?IR9{ID!7#xeO0R z?DOMQIe^Z9ya^wKwKNf>JD8Y(DYABDvXI#r+X>8ZL7S58>M2l6<)|9^YH;fdy*zL- zTZg)SK%sDju9(N%hmev1eQqY5#bipt(K1UV!io|bP0$~7`vH@E3;WD|2`rjmoG30Q zw4uaIki8Zi!h%-27WGwlIX|uoUoLPuxB#FN|5cWT4kLx9Lt}uNW{VY*r_&11!zF=N z&-d5%gs1QNucx;iP7Dv}g2|+T&ZSs_mhS_80m!cv^z(m|GtU-n+Pn(j*FtxJE^8u6 ze*`f+Oirk_B=Y47xf10pX(;4#^#{XY^Zi41NkC&J1Vpj^8=_0d$O!f(Xlyc|@DVzL z=eSdU$#Yx%`%~>I%g=dxg-G-}>rmHKc6b9AJS~ zR>kaPH=3LkSa=xGCDT-%KV|DI`K+nu2`o|8a)Pdyy@re-Bx8(HIfO~F(4ln&{TCnz zHhcK+AULEe+_;pij#luZ%sWdQs(_HViKq{4VZxx%Xp(9NA@eyru5cB{8HzD)Pfri= z@2$Tbz2ScEvH&;$s30#JS%b~>IMRS1BR+5wmif%NxP!Hv^QzWywKoqqh18?D(3i4(OL53dkB))IgANl01Kdr;dlaZ}oI&DUAg;ruQ^Zx^$s=Xt-&L(& zGO*Ew2-(0@)bx;q1qUk0Rv4ZaI)@eTX{5-adj3euU7GyQ{2;s8ewG z)iT~%*%rWw6T{F5GOFa3kNZk>YezF8a@g#|KvxQeJhS#6+T3PXe`*}BM_ z<)y_sD0+7MBs%OGgu5s5hFctg`yv&1<9!^njzx-Y3o6m*Isxt6J}-=Y}6IcExat+ zz{LdW2PSg(W`n~w>`{?ZEx~xeE$!uzh@aFJ^dA`9<1YYreV4=jGnp8+4Ic0~K5nEg z$Jd%divP2=>NpqN16|IgmXu3d9noBRVLu(QVT zad&7r{sJgwiB^eU@ruUj;1I^tEBYgH>^zg+w;dZ5RPL+_c$vBaim*Tj%S*IMA4Yd>jF7mefIqOcG3ht;Wr(y*Q zDIt}g^w-3>;J%k~E@R+$nUKpbxM1RZaGOOrpP6uEL&)bRO&4)4xB;S^i#cq62)U5j zA>!b;?VlX{%0g1`|C0$IP7Sx>lT*8~h?Lqd81vzDa1S^+opLCQ$zIb>Il|#YaOX8S zk$Tv%5)t_cA2yr??uI63@f`kkMarV>-e@=v+{H}JBOSI0ggpK)o@O{R++ar?(Y(~H3kSemx#R$!z6YS- z(S_d+SEiKTZ<_yi2+MRT@mt|v+skitKH}TA`u?}~_`UJZvE}z}j3T|a_~UH+#`uTS w@*BtgNqS@avuYdz|Dv=7qusk`QVhxWrMA``VPB4R=C(=>@HZgKB=p<=0B%7=PXGV_ literal 0 HcmV?d00001 diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..e1fcc0c603ff7a6841ffa918cbba387cb3b32ba6 GIT binary patch literal 24232 zcmX7uRajeHw}o+cD3s!~xO;IZTD-;G-Q5Z9Qrz82Dek3^Qrz7oxD}0Jq$9%__iz*Hc4es@Y065rsSUNa38oRhT`}i7L`gpjRy0dYz^Rn~svU`}i!JT#; zd#w&NaOQw>Ri~p;S=E;blrLzPC%otzrI*)|=^Q^746{0O;7A7(nGu-4O|^2{2L%BW z@AIzR4nC`BdLtg9yd(L_OE)`iVL@={uen%L{i9OFMzykOYwf;!4f~X%*crvY#;R@k zNKBN1{C2YT3DkaGiCp;NlnKKuyaAsEmJ$Y9PbN(Cgj}M|JoTc8#ZmJH9>Q=T2M2$z zr3T#*w-Y}593y?W7}&C&Z9|6%c*?etEfI4ea^F>`pSxd_Xg%`?=BW%1xl65L7iW+X zKyb)Mafbf3I8ae8m*Jc{fX;+NUYb={2ixR|4+c!?(=xe$v<=!m{Khj=H-Qa?+_6n;B}%<$KQ9xUf}~mW zKU;7UA9W(`Ot(@;N(_R&n=GIHCRxCr3zk$9qCZIH^vj#mTD2L#h63;Zo;^o9LsB zrOr50`^d@3)w`{?f9-~gaha855r5@f5*s(T(p1721a?A zE$=s@u*9F2Xzw-=Zv0J&#YFz&CN~=qVIFOynxEb4|L32_z9jZLLnKZ_@&op4-wd7s zkfTt|8SfZM}iko~IO|E;>&@3<()seSN%MFa^bFqO8H+z_=5M-k$*~lFXNC7PlZ> zvfo`FDQ>owH%|_4)-N|t_EZX$ zr&BQz7ed*6Hmi3qHDZY!*D#%22)Vh$zw<@|#D7ud6>jk+PE6KvY#p$VU0n&)K3#AA z04w5fplY{MGh`k(d)v3?sqN(PBiv3r{d{9!M=OaGyLs-d{n9S!wk9qT(OO&2t-{5; z_EQw?rI7dn7=>+Jt_HbV1scL2E6y$IQ1I#*xeW{)Gg;jKopg$H0C9lQ+<}NDXLqKR z_`Oej62bw53o#$QjfUKJXpKCOmqGqpB5^mak|oB?ntP@92e^U}`$=*W%Oy>ohs-5X zXNx3HG~RJUH#CWC74I0ZOUMZ9>9|h7%0)j{`1&y2z&>e z^PQEwG|{A6E=zfk8hgOHyDdSbuI3)t2?yqKl_c==sGzHkJJi@f?ezQwq;D|0*yl40 z9lQw0+AW-w7x-I5nJ+~(IH-@`B1^9NpSzUd<>@ZmRP&4NqL}4g?JkConZiHVVcSiE37_&ac zQFn4wrkF`5PpDaGQ_8^TP{I3qyE47epY;4o&BRTBIQb^>RIBWE!&+f= zHirj(-|J9^=ZMZ!J;R(?7pAKkSs6?PErWn-z_<)@l){FV%fL{Lhb{;%}UjB z%|H-De2l@cq6P`fE`Gwf3lg6cq8dT<;xB2wneS$lLclQ+GV!-vsoLqh8cnC(a zs|flZ%O~G?z6tr+YX@f_G5`Lj-8Hdv4Ab`_hmX+AsZC9^c47bC0)eHQJm@^pJ6ja5 ztdozv!0Cl&Xmp1_Zgf9c(2H^ zj03nr&w9Y>khOCX`IiOX1nCMdBq6}F&_KOO;e`lq^XBICV*W3}4>`rT&tXnGPuPL{ z9zTUT&VVq|*olq&0&X$Q!ffNnpYr+Sh2Y+OcJ^nvuB0 z@66J_KcTUcoJM4G@d)zr4D>})tdI`ff|g?Qaj>I0<8pv#KibCuxB3BUOR&ZdEpRKM7uf~d@Gw}~`*kL>4 zBW-qg-Xp+y;-@O&Qb+i~IT}WPzTF9LVON#z5}z4}j#uzZIO_>hMlZ&aR`p56RIK_Y z=qVwjmBF$@S48wDC(G7kQ1NlM4N@5W?O^>kfam&-ciqxgskKfzCOWBP01;SZ}^l2iD3Ha;HANBls6+Ih9)_@MP0_ zW0YT5c!&F@sf0eM<+SMe{2SeoV*50ojG2$@3M8!JpJM zYii;@XL!q{|;s5BLe&fX8Tc-2=-Op+ zH8Y-4hoeQ$kGJq5-X}^w(lqet-oPu8mxZ=Cu38>SVKuJnVBBGJ$ zn^uOXJBf}#j3@rXdzIFA$#A=fG$U=?4fk_qZ|!p*t$0^j-O4_RD`MSY^!WT!4)yt+ zH?g<+2|m8}!l{6V!u<^E=&&qvm^i3W2o`Q+^Jy{q6#b2sMl|ulz14NS-y=Rz;Yyl^ zMr{~_(9Ma69BE8@nSDBm!V~yw8*|F&%&l^^nEQrmfHT)53%4FpF)Dd<{%D{Z>*^#+ ztlFSd$M5_01UI=9vm)lt!EVL+7U6UxEyyni>eg)naj!x!um8d!Xvfr;?tdTS%QEc>^fO>uBt>tFWc8FGG z;`0|F1zkTD;hrgSn(}G~iNx50qnb6!WM|4+yJ(}uka5vO%iVON-s!Qeziv~d_2f49 z{EbOY%HAb8b)UTA;&7^&arYSu$D%YcM;F^f-Z^H*r3l)wrOEO zaN5+n%S3ZIpFjbwYjK|jT>&a&6(S}uQLC3k&sDzBJ5u8eOVs2%ZRzc=t*g6OY91&$ zjiu6DBoh>kfpi)1kscDs5W^pZ(glf@dfx$=GrH+ive3iMPgKeLU)E%Py0doL2(D4< zbrG#k)>`4P5c|kw>|JqD>Mfu%!>_j+@P~!tVeX;>PLH~`NaIZUE`|*JbXMoeAenjq_%E^ zovnsZ6r`m1>c-4itTPRDG{-F}v;$T21bAn&G`?44siEzyA%U@F1r~*IdkAs_lEMiB zXYthYME1r%+WWwsI?s!(AC!Bw_>T^h^|-x3gm}sdPud z(J?!Iz~Gq;@qipwJ~ltzIFsyW3bfiT!xoR;Gn|PhKJwhDUxHBfksc5qOLlPtjLEkxH@5Z<|rSxDu^H`;>`$FyprOZnzPZNDxy#={Q6#Jkihaz=XBzR=J_l|je3Ci6{s^k9pC~oGXMcWSmyW&@N{NIoC^eXz`+2>#`v40iI2Z&Uz)s~KkuM#WL>BO!|cA?A9o;`@^||k zQ{TVsjJxA&UnxbnPcp}Pl)NXD%l1@G9DEx=l`Z*}BCXcW5VQB_%8F0yHV?`ElAE)0 z#k&%yycfR$688bJEnwaHcpIQ7b-el2^+H3R4f}{iT$)UQBsJR-X|6`&z%q@M&G#oy zsoCdZ{2Fkp&%6Zgf`og4&Kv;5@&vMTCn=0AXL0u+<)~PRyjcGSphmU=;zWQmnf*3k z7{(o%D2d&}Gt0*ngm>;cqZAbP2tuyUya4qs3CuhKxOew_0OUK&IIApE@)jvLM|r;b z2F@+Nfv{N8h~_B;8Ae=jS&)6*v7>^)-6mcHR`Ro5AkogV6>RygiyDuW^^R0R!%Lj$ zRCFiyYlFA4TMINZ8CcPMtgGAuj}L)PTZTcW3p_hTi=;CaJgmd_YVv1Q$bU}L0EHf- ze}L=M$P(yISP$m^;jU0gC8W6>dQPb( z1vmPFZWqZ1J(2RHkPy^gBnn{;!T<-n zzb@2U;24-*S%iXK{gN3}AP4?OKZs<%Z{ZSRNz$l+RT<-_GZ8{B0BE_QT1tawd9k4YwO(`e`Xv-5`R55R?(v>Mpna<14`4QWn=>yR>PKG zLaZ;-L%_4{Be;AI7#{*A7wCa=GTFLb#1N^}XRletO)@r^a~;$e+nQnDaatSGLH$)LDB7O-5y$k2A&SwByt(kn~C>+X{7yHm$22hptut>A_uN zK&j_9LExy_0|rq(0u90m(8H4(7Z_`#-tnR?KtX{}@cIlCD1HF7b;`nkpVl7P?}G*N z{R+S6-Vqm5V3iV!p_2q~g#~AzbO*wW5$<3UjVDAov9R7?0+WZHaTBn0lNF_~X)#{7QZBfAMP%!yxnSD;C3u2vxza%zvJKdp^R`Fit@8jyZDv>YPOxkoNGy}cc90r=2e*{y-0&R;GuZL>j zS|yVwlo5Z9!Jo(v#u+T$igUFl3p2*K<9_|*@IOEZ=57uJyTR%T8P)0-mG(qTXxrW2 zi@O=-%pjL97`R(07O2kXkK)L7^T-8^aqu=t&2menD$?iPjGO8XK8l4>vaMoMxHFIc zU~SW}T28MOvBlF+qN{xYw*dEk51=~FV7VRqx))>s}5L z72V`tGhyT}{=@2aVfKxTQo!BLSCV>I)|+OYaq>YE=Jq@VpCuR@XN%hSFE$B&{W zZ9^fPH;;hSB{0#G^C+%} zU<^{n09g~DPi*S)p2#V@Cn1~?KJn9%7o8~#UQ$m4&*9}#)G&==(7Tzs7n=QgRC9ZUHqa3$VId0Q(YTo&W`O@3SviwUD?tqv`eK zzxLu-r)J~8E%E>V*>NsS?*cl@AAyPo(C{PBAR^HFoww`tDYw+w(Fri$}b zUiw<7g0i+tJnM1E+MSzn1UxKEKk)%P4I53-AMrphG#Yu;;1Z4!%S? z&?zER?wmale@HG$TP|wwWh5?nnR;0!I?iEzmq-7j@MHVXg0fiVkOc{i`VE;L7F$dp z_^GfPIDo$ex2yxR_M-L_!D}0)iGSwK!sv~2#x4ITH}`le-h#i@XAXc!b)V0>7XZpQ znD!>XrsvH^_$pzV&(0j9z-kWQJOEqOB?CZC9tlQm0tQHT7X#T?6awUlqQ>3Vs7LX{ z$gMH|$*2ebVz3A1FCj@0K&6{?Blq-{Ei*3NzlWk_s+416_NP}rd%Jn!*G8}XrL5$q z1{(*Nl(ChOBxw35plo@PP^w7NN`E< z6tw;l4~5L!ft81WJ>Lp*%pxR+N6As2$`anJ@S($a4siS|PQ?Zouf7zDmSV;it)G68 z<M#-Ys+jJo45u4VzY4>Ke%%Y zB`2S8Y_sNO(sofRmK6`N-anlcs5j@sYidz`rIwI_-{!-jkHC8ql_Fa@Lh?sbFEPIO zY-6+mDx^UcW=Fvrw{Ryu4uH`gL6AewPPQ;Z1a1$K*(1DJib!uEOep^-P;SC84wzZ& zf_5Yd0a4;d@Dz|_l&*Z95QjA2=~FR%1J9^hx8ha{nWNn#J)U8+hXKKlkQE_d^gm@- zxdOKql1}=q<0vB0(!mY-)GPVa-yyCLz+U^5it-}Qm2BCdt#s9I)B~UOu zo(HtNf(#Kb_zbDuE%dXWbaD`H_uMz|pBVj=P89D#Ga}Ti(aq8TE-BM~TrW={?{{|R zChICWFYQHZ@JQ*4wq{+)A(zboUU`Ea<}700=0CYI*%UXi4=%=EQGaiEWxO|JN7tr$acQ23ZxGVuN&5&C^0dikVfuDqUXis&L;Y00z zo6W2XEbXUQFM6{nkoKj!Ne0@&1UI`yvGTJ;rM>rduD#oJUW4n>p7~8EYxgm|H0oMf zB{#dq0y=|V0O-6R$~8n~Cu#=5kia4{AiIxhveGnAn4Imq@jYNbQofaySZoZk+(WUyIi!0Bs34 zP-QxVg=v%331IZkE6Po8Ozwlu>|kITm~k?N;+DD){!K=J2TULza1@0CaCPu-mKW6B zXgEB4SaKU4T<*91;MEa;CHPfm^ap_1!z1WqQY3Hy$tQ!bGJLT7?DY6@de5LoJOu@Z z0#URu_2d6k|1^y63cv|I_Z>sK_`{Z!`+f>RYSJ9lCFjwZ=09dzegUtC{e(eycmW6= zAonaToL>=zoz=I zy_Lk?*SBX)cB`WUeazCdZ*JpecjOO9J>DM2lmrgBMsu~{@+O8>kHki)%}YsbVEZ~+ zQSe&?*zP7-EtB(BJ5BzbHdVW4uQFGB;BdwQb5H#t=@?eUK}))Mxw>YqKp$my{Y|pa zJusSDwY??7b%y(E3O&TYtvGeO`z*pJ#EJx$1LG0Ez^r-I`a{q&TOjON?WwSV9jHUJ z`LVv@-1Z<@wC5wQ(fqe<@5%U85qAI_ZkTaR?E=vK=DF@jRZD(f@XCs3w&jW-a^;L6 zT~tsX@3HA~9S)hwYkZOvm9oH046_rG>Aykk*zs9o!lN8~tosXemOcWq51`6Uei$%j zbB}}Ng`bu;M@11@2Bp?xUZVW3Zm+H(Mt3ldeBgQd0<;g@60U=_J)ZY3B=(dkpU`xU z0lFuUsw=Ge8gP^cjJ2TvPyCL(w@{6#hd{&K4GibuKl8Bx%#ZW|WRZ8k3K77O1b~aa z)p)t142J$*OlN zV15`F;`M8FL_a5~_1J(_3{kI;bQ#x(IwujG8*^I72Vn6Lph^VlSOCC%NB3B_a@Jq0Ilx&A zYyZwBddsMN^pT$;Z0QGG@aZ=2=8w?;AiN&N3q%37{|$O(#Ss;smv0O=rFt;u0`G@7 zHbS$+>Y-%L__U*PLn#*viCK$vv}qu^KQ|Y9b#@cCEasugZXfe!+=_y8sS%^GgSc|8Y?t#-gTuLN0f18|Eyr=AL%TT4s&odz}=MqqBz-b8kjl=zGK zdIklph>OH2Uz)#8Cv*ozJ@zX#)xIj_?lkpRTH&#)i?Qd7JhGHMjn{0q*V7h`9eID7 z+;{B*dGi7wMi|6M9VUQy^U)k^?XfzCA8&**x?`y$#z}!up8y48EGfPKcf!lppg>^K zd3i&3H;S>T$r=0wpKyvEVA}`iOpHT-ww9Ow zxk_H4ZTE8bF*F29|~k zcy}X&q;~ykSPs5i1D<;T*Ob-s&(*CM2`<8Xtw%8Ge}%L-xK@B| zjUvq6JHGZZo&f>FVU}v0J3?(VWR>;52B8GACp~q5$(;g^KD%HRjCuQ;^KW|+_-nqT zRDIl!+~(!K@xFQ|u?kB2C8S;H+jV735+@(gb^Tf%!zbP3YouPD{`;EqP^!Ur8n2AC zM?tyCK)mNLhyt-c_!SdGx#imKTy{Im<;W1)^k6Kg{@GA#5A`SG=gt2Jg#PjTA2d3Y zSNvH!rCbz3s@Dg*Z|FITre{ zMxSxsi1{mHU)Vh&8z7nnUDVA41PYK!n7$8^hv>}MFdYjYo)m4`*}7Ekf-y`uS^x@* z7x1qY;Q4Y0w2XBzr=iJ4!@7EN-wbV#?T#FGp6^}P=qU569n)xzJsxGyUs6Pf`sO>+W$fJVT zd);HNYqI(PVpsLg$#%FE;hhupyQ~btHnJ(v+OZp_HR>Rb^tsp`OAEBBrkI6LU733v|EVvM`Oomk!_U z7QM!zOMkZH{kPGzKUHaok|MXAam!R4O->~F{-Y&yl-YcQ{Q##EontA15va|nKTp{n zwi7)6at!K1tN}!`VAem?Uqh8nR4F`QopUM{a$gF(3G12z#B%|P5&M_(=|fNyQDbAL zc!ZbJ4DOxR=Soo3+0aeRc2@B1GweX{9%2kuyZKhFjaFTwLBE}DEcQ{tK?wefmKGuW z23|;21QQ!jV+@;pgixx$^z~r^y*^GJ@m{$UNbaMg;E+{U20gqrt;af|6UzhO`Sd3M zd9^_eOpoVF_xnm-AG<}so3y=~;Wq$R`!C{|P9wD?w~p|h<%aJkf7GWl9`Mm_kGI!5 zJrmi6DY{56N50!BsV8xna}!1(5NH~>*^XxY5*I7}X^_5p?$n4O40_yn0N(-r5-_kB z0QOq7xfeR+r$kCk`;r+>SsbNe@lV+U?GOLuf7#|Ua9Vi{I`zK&pH(M?R2=YDQd*TL zJj0k&A6jB0jseAgz!MA{H47|y0#x#@vYpRA2iZe=uD4`Np2xO~3y~e!fI7H4?$;+R z08~lguWcU-s5=A}y=%PTmhEamT$DC#;-`0Q5l*0OE1k^2!a1vuro!ge1ot}CkD%4A z1*g9&FexaG;^}K(?fxjkt@iFf=0C?bxMCf3dwKUW>vi+a93)FAos|0d)3k%mb#Tbd zodpZPSv7EU4SD@xdiHa`uT#V@L!^lS*_*v{NCZq9;g0+HT+;t_gS8<*3)gd|+}b9tYNLDkCFG*cGfdj^yOgcwu>Us_X}b zIt`15A)&TH?HIA50ZR`k5%2apUEzpllIAt23lNwD3U^FRybHNQBM6O6N)o`U2$Px0&{gVrT-oX?~V-^C@avqd`tMOJ7KJJ=2x>}#$3-#(a-2Qc1f%* zWgr~uSP8X$TcfpVgNwMyz~laQt>eA!7)Jtd{FW2|3SQe&8?^F9eTpV{1=jTUF{wBj zaJGM*lsyq&Z+Qg0oeUuqIe$BbXA(wfK6^vmGv$$rTI@K{?i?(Jw1V*;bDF_$XL@@i z@1SxJpp6M;++)&PgtI1lXP_WC zIbxt-eTpnbQ#GnE|HA9gUm|jbooVJa!nI}!>9qf+#sZZ4x2_JM@R**wR$HN(3=pbpn|C12);zFs&8*@Z|~dlWE|jXkLy0NOl@P(jZrZc(+NFx`Cn<; z|6r6KGbp5ja-yK{tc6E3-lvLpjc&KvYqQH6)n`2OjAxyY#?qMnneu^Tw5EG1$q2*m z?>^ZH*dgY$0uTt`4lX|ha4i9Wg>;~8-HCMFsctbACp0^}))A`j^uLJdMKyAIBoFXz zegQ6*fI2auS=)cGO1Q$kl?OJ4EYbC@d&`1*Bn6`6c)@z4q9#D)lHWP1B!D8Z&ndKXYO; zoN8mhLGvx&%SzUYdI}PU8bg2^7bv)k9|l4Ev3^`12X}bM53{Td)`MFz`mg?8hXsNC zFG)|ZbLU7vk(0x{bY{)q8c>4D-9anM&p|7A0OS>>AOe0}FJ8jG&}_~G*=5t@KGVn+ zUk{_7dz=hrTmr9Hb^!z`-6ih>&#zfjEWmOO>h{+7=&n{Zb%$}Es^Ro(;Id{)QMMjj zrjlXzypm^di6xL4+P_I=Gu-OvK0tP$qAo+hy6Dxo6oHxV*r}v4R_6H`GZx|Gs?6F>hIQ={oo0nzgcdQIPjxn3woL=!`N zF>x(NNqN%z>kh826fRw8!RK;K-_wWBo@~xALiH|P_@y0MzSf+Ao}lN#B7m@js!^#F zwj}S*c@p-p%ZC`e3ea>b*s&2v5A8aE*3C}5PHa}6S77(@6F_(~2Wb)|=^At8q1^zQ z3~luM(~R)b>JBgv^cpnd{?Bj=;4BOP*_e#;+$W@rR8nIuA@2rB#xL5k{(nw)e$V{~ zx$9&b2L$>6nQA1ch->!}yb-d<9W?tW(P0~@2LblW;^VItIjX1yGcg$>M@$KZoe0IS zw)&V{VYALZCn2h-4W!`~7ASWGJ1DJWA76Kn{^!KpwHW)pB;oNUNL+Y}&Bd2sLkRu` z6>)_Q&V z^9G@Bwy1w-O~51<1fPH_7ZbA#&b3qeOpv{uyU0`@vsfzEbzpBKNt{B z{FYd@e1*1gn4VqVIj;a%1Ml*a=~Z?&U1#2^PskMmZNfsZ=hxAfw<8p?G#Iz(|15%_ zGl9%p7Fe!8`z=x@ZE>cjZ44OgG5P^`-!(mf^sfQA62Q6%9K^>X5u_*Yh`nU=@>&JY z!BCL$J#e;@?kGEWcony~Px&1!xsGETNu@COgE+bH-Q0`_NdMv0bB~?@$2W~Iu;^>z zO97*DJZ)rcJtjt2SUn%0&$4+m?kH{HF!lXWgsA|xt09@kxVwk3U6+a^a)glRk4cEm zQjVX)ssy=Gwthz~`XftA_xtWD&v~k0!^V(1TNJkGXcIiTiKZ2zg2ZoW0-uFL)3-UL ztJ$ZK+zVNk*YjKqZ+)wv${I8^Xhy^nI-*?O^M6;v378IN*M@IvrB@ z-^B61=yFwf=m|tB1RR)80~HIR^XlL)0HLMH3!wE5*uZ-Yu3%AK)r)>k&BCGuNYQ64I!%;~aX)yk zA+G}^_>=Mj@ba^`Elm4$G4Y?Bdm`(;;rt7#Sbn~Jg_ze3X1Nl-{U3LNTE$?_9-Ys! zam=CH*tta*GcTW2kz0`JJ^u$jk5o-CvConisbw$H$S zX~ttAX6rAl9zLM%8=&h7tJ?t!^a8@D$2$NDJ>uDSq8>&@c{IdXhyP9o>5}3t;H3Jh zsy>gPju+!xcwPXE3oPCNQ9FQrGa&9PdMs*BmWCfd@CkP*(1{m0%nMi*``=Tc95qIF z|0#0)E9aacXU4pbU5B{kUo;lWeT6NTQ(QJm>HIk6yBRFkgWzn|k^T+J622(a1FA-N zt=X%Mfp@xw{7t}4UGMA7yd|uzW!_BS^^)(>`4!ChUMO+`Xirv6+sgjk=fa#rfxhHs zcx^cj%s-32U0>m861>khvC_%}-0#8DE9V|xp^_DSXRUeD32lfNE$a?~FKga}eUkobm8SjzRSNG*OXpU! zimN(}Vcd^Pm8!!mBRRwBZs@e|f}U15@o?9Fve_@8pA#azQldMWX6A4{F$|D)nkQFJ z4F-6F!sEt2tQ(Z4n>|X8Tz^ycZ!HqaT)H{rEbI5^AZW=78Fp zqH-35C>lpzFT2FPFQfL^Q+jY0KM?GEFLC$`Mx1=%B5iFk!OFvh$))^`m44$E|K<0@ z*nr*VUFJK&E}AQbg0ADh$l+A*_?WL0mft$Q-mi|G!1qr5Th*h@6ZRC5zu(iKdCA#m zoXG!1ql^6MV~!DV{)~$>WsVXk=pM|kfnl@oNB=ri`LBL15`BCk8wqdh^xqHKC`x>? zm?XT!>rdaBhd3s`Bw%5jH~d9ri`@TUd$N-E{auM=?1}`FpVg#OQJucEiZpI<+okye zV>J<;v(mkKwi1tTbw?@fP@c%L+!er~la)zpPz2FS*Ju%G;{CHT3# zzwb>{RKpm;2M^*p9%omU%(shKM*10@11FA+-P2uGbjPEyRa|-vW8wpvvK7wdaDR-l z{=0?K8*|RK5ARHL=WfTf0a{w7y%;JQ;v|I|0zu{enbi`smoZkpj@g4hsE>|(xY-)R zQ6{=WLyqC>cyb0W>Nc^)Am{rbkv)n0c@A4jQR9-)%YD7^I!B3NX)7M<^g9={QtI-t z?IoKKX(kO+hP;64$9DlHc$jaLZPYF?|AOHaA@cbC?#MjUR3{Z>Jcal-^|`rC&=Os@ zXlvkVgyDxSldjn&&y>rR2>*p^iz^l%aKPH+E*2RxLnHMI=ZZ7bxxj1k z5iXKtm-za=sy~UW6e&H|pHLw~RnhiKOe9RcQJP<-SmAS)?dzT0Z^Jgt{u;PZj9{EV z=Mh(l&jl=ME)f~orPk~+3vZN))l|n9HLgA9Kb1G@utl`lgtW35CxFm_ZY=oilX*mlS?rVAfk6&QQg6&tm zx)@BR%(4Gmm9ron=St)xsQnn$?O$5AlN1KZSWC^+s<`vy_OQ77n|}}yWb1tD`k?TA z+v=*aUggtynJP}*C`-z>MTg|C^g8tz3W@SpK4w?L2Z{fMh#o9}Hoi>$HuHS^?UynFEY=Zq zdUqs@m}raSvYdxhW3Km`(qZL9B(4Iqh&M93bOnnJ1wI*bJL}T868CD!U-|^9w#Ni7FNj@Z5}xwZ;n z{ECz_R{cazuh60cj{ApEQJdWiA-@u6@8s@JBu>yXndglC?*rDD(je$Z-GdvlmNeQn zIegIA`ONkMPD2q-+TXy(*Ee?AxcUxVygTClfPE4)+TwM_- zMv0vKRqIbvvf!_l6V0%rcg{qcvboWo%i$yi;(+0(V-%Vf1-{GE>la`Ag&!oAj zkd7u~DIvRv3)1ASf8($>aK!@)d#)ul&Zi2#)vG)Eh}7rxK&xeUK*3V&gqr;(fVl=OK0l(t^gP7B9|jEQwwaRh>k|W%G;37QX0ku{yV>%=sLjK; z4Q>b(ONJiq4nr@B`G?Upjg7xS{+-|p)&4bGu%+U?ZjlVM*(p4Iih`Juf6ZBsz2;jU zHZ6FYXUx!~(C=y1N*_oR^~Qs8__Ft#Uoz?^-P^73_VF;KXADT9o(FA;0R3fnI6A$v z$e#e_D!R>BY}vIfYR?bU5i088iIU9Ah!~ey+QkeF2;LcENWEB~_vD+#z2KzP>ZI@`zq&6~uN% z?^;dQGD7;DF7s_V_iGL7T2g%)yQ7^Q3e6oxLPMMx9llZkS92g$v{7RbXKeM9+iy|4 z64~}nbaNgwOJ{zb_dN;{K{o?GWNRJnQyt0}Ly67W59F>8)_MP0nKt15%imokWIzin zE@{^O(hvo{(d!E}k1esl=$L=&>o-o$;pjFlPk)t;86hkm8SLiHkZ-u~my`n6{po!b z>cdA`sg#Wm;qXZ&Q^cF{$$XS6Tk-hQ6S8u|YHA(d98?^sbkgigb54@Ua!pb%5cLf| zjc&n{=(6Y(q!3b!q#&?Gky&GmmSO95c>Rpucrs6RV;;flcqi9DK>ya%U9M0_Pv6jg z)aHA`x@oGPp+*P-9pl_Fr3SK+@Fmm-6xN&5G7`_wlVtp5S>xBDOZ;`#0^*c$d$CQ* zn<}_+9&60G^C|nZcc0(elTQ&91vFDQ;Yi6P(D+m9_JKY~%YcKrTO?U`qr#VLT#nf) zh-a7`Y*ihEDB06IBpo36>l3_iiO+koUxrQqs+0*kC}^WqPHbFjmvnW< z6Jr%1owd$l)cQsrOJv|sRF5piLL!Zym$Nick(WpLw0Ya(c`)?PZC?D8$}qL`+S__D zLHzt%>gaV_Sx@0Y_O6;UF%ljBF-<2y;=<3bVNhrVXnAn8`$sb+mt^a{cV!bI12*Sn5X*q@`>} z@0^WnGC$DYCt!0L)t125dKx6eGUD|QuqPq7D*u5qk18>2i9JFyyfTjdgpM*?oxsS#6&M$LWfa6S@WL1~6BKlZuxG*=)pb2W29grBQbm=*FhqZI z%lKlMR^BaxOO&;IzaJcWCGB}nIDwY##IQU_H^JPX_I1CS7SqfunEg$+#UQ?uLPB}I z4XJ7Q_xJl7OXQ|pdPQ2}6{HPlvMJMWY;PwIhoN9%>RA)P)+2K<3cm;H6f5%Doxug~ z1tL+U5{gF3zKVuc!CR9*hc}-KzDHh_#i)68^?e^V+$a*)G86a*hbkQ2DrHb^j{orC zb70a%5tC3wItKMLo}ipUWt>(#@4xtHa4`mjCAmhmLkbT^+3&yP5tt?m;(}r?_vd$p zsW%~Of8x20P)#-o75WPQaI28;kO&0U!VJ~z9Xi4}kod)w4G?L2q-iqXK3HF?vHxHf z3=NHcp(~%K<1$yuioK6XPK;<^vuZDB-o)$YM<^Rt!kF&*J0oJT9ht^vDi$5;mC|6` zZ=TvW)J~)M*)tal@%kr9!>$pesw%$}Sx}NOK1}jvEB=oe*~F(4lib9vdZ%rzqhj9( zt?YEUF^mfh2Y78;`1}x~p|9^60(c)!UEf9LO?=hd(?3GteYvuAccqp?*Jl0x$yX(@ zwyB#)qxyIH-1=Ipu4y*XE^(p_wxtI$qAyTP+3{GGF0jR`THZm&E27|6vhe&MuU3&` zUi(?C9Whz?uc|vO3mBONrRAW2C9Jl{$0Sa{!p|n9PI{kBD41Fd#ofKnP*gfdl7wDm zxX2-dZgbm7B7h=RmQ3*{SNRjQXd<}rDz zF%t>dOg_@An%HXo=1+L_ckrIt2-$er`{+|P%*(?^g|JkEVsXORY_Dm9(NK{R9sOLg zDjhl+42B(Ab@hiq4UIx7m42igd!|Vk2H}7PY8Vp2j}!&l1_FTxdS7!@N%wf1OgOPI zmW;?uX-@a@e;B)(R$8e9Q)pZ)w)@!xP(nyifKU0VPPO%@c3g&hzk(XpBnl@}Ju|m+ zxr;)xpQfU%HWSw=rHZ&X zZ)EbP=RBsKR*Qz-vNG0@aa7C-Z+8e39O=V9?q)TM>D-YjvhlYmODEu-(r=e#4|iWCj^1fPf?Dw%3skeJsc-> z&$w~Y!$W-nYp4DH0%#qV;wEvDF*x%3#7se{>~`pB0qRDSbZJ>n2gNa5o&^!nGIbhK zAUZRG;RCrL!5t{BGu4rcIpQoeX*fRHDQFmq6rSY7&9X1*1z}AwNbmA>*yV>z)JTo+ zlw3GcRiR-}3n(Oj@y9Y_V6kzpf(?e53g*C`!`u<5Ov%ij zgj6&q7nG(pIoOHT&~fJrsh6@^kz;~fGr!hSmUaZ1;N4R)phSR3D&?0R15ToYi`aa@ zWEI{T1%z7VU6B-zMfI9mL}a9pV{B~uQsCHbE^6(@7$uw8uxa^;$Wh~myl?p-9Y^X> zPWKs$Uo?~~IgxaS708OHN!*XT6x)Y2biFG5XkG`;!8=#5Bj1mxZ@sS$d9Ffv0=R97J^G+X}t$V%~Pt|lv* ztbD^{B|`2E)0C*&2O}udGPD9c!Sl}hk&^-yV5B4V>_-tT~3Alkr*(O)@ujVE;$8arAi9iAQ`aMc@yi-i*dP6+%1H{l%*h zhSMC^Rr>E|mY*e>?vnt&1a{=7fhb|Sib%1$mx*%~|C$N8|D~*b; z(}7(EV6lc{s3wNKCHH+?!&uRgcE1Z72pu)8s1m_SfDuSGusT1tjU$|{c>Z)~;2h}N z;Q+_RlCB6hozCZRiY;ttYVz2%Xh>q_JTV-aMl3>UGIandv9l{3lVd%>oDB%Kv56k& z1SaF(;mi`RSPOH!OH-jo#u(lJ3pu=jJ&8nF2%)=TuSoQYa;qrSkNNL~Zx4BGO!_k! z1?Fc3eALAaP@q?(?lPy5&AwHdzLJ6Ze7@5tyM| zQ8PFsc;Ul{eHTNp@93il7ck%|7IG9jqI^Tp8uNue_X1n+Zn7PksOkw_w2#X>Pvrg)WDlD%YYr1Z7RUrn67)%eFWv5h;q<0&_!C2cYOR91o})*bR#I4O9J zg}c!hry+nNPI6-bQNg_y0|RH6rq9vCYNY-CaFob)R_95E^C2DX2J0D9k}KQvfn#0M z+{)Q460p~(R~7FQ)(&k(%;Sv@nO0Vc4`ShENQ%_S+P4f}rK|H)5_7s#={2}I7lPh% zCS1q+QdnC|K!u-D*4s6Xyo`Gf5G9tNZ4`x;0uHOO0$A210&xHyp^c~l2x++rK#WsV zMPOtPz7`-+uODDI7;C+J9`6Qm;i73QoOsAAtjGD9Q@Hx*OqEoIk9u#$Fq8zeV0w4a zhXQfI-|0Z7e4DQ21oNqR2gE&`+PQZDh%X#B2>rzZcVKPm*z*KT~g#vh# zZ7wZT(wYe>(_(jmMK5vG&yzA)jWn!ijhG}mAPV_nz5^-E8O9GFG62Cww3zeRcyTqv z;_Utq*EG5Q6_X7}mkgxR=m{l#d**$za}(|Xnp%z+kC7L{Pba})5LXH!d}9kfLv`;Mo7=fb_An~<$)BIl=}o5I4I1LGg6bNa9BU*c#Pnuj zo=OzM2%&r1r;%kei)_Va{%+spw2}!jFKC5E`%@ETx|K(adw^6IeG6_83*ujk8^91e6^@?JGeasU;?`toRoI*8 zH+u_IXxy2OWBN z18Q`G#@fAz+k6l2fW+mes%#m>-pKq;Pvq0vVYRQj5@A=fU&a$E~|KSc=U#q zPo_~58E3XI0O=_RA5WW9?S1ObXGt$RBG8jA74YQORxy~ua!MdYse&&_biEZz$hk{y z?l5ByF3!bWcngx^XCDTGp{T9|-WZ_wffkHmhU(MQ(6+P-6oD@2Ly3z_Zx}p!3rvJmv-KHDQED z(cu%*8Y#uVZ5gGG1$`7x{-*X9=f(lTaJLgz=EpL^sIqGV(->YHUcad))#ARAng}1L zi{G2jQ4j`=R1slSY3f``e41z)YK-geC^-29_}W6p=w9(#s-543fe4tEDwSOzmb{mx zZj68_&L}ndG{mY5A3RsP0q{F~X(jl~m1%hbqn z{1heP)F}QQktK@oeB3iZuldC*wWEs${R%kyR@GjWh+(? zDqB&mKv6@&B02$1BpP4N@aYt&Y)j>#(6{Zl%5FR?%$(*2LkgCm_wnU52qNnKLA$mq z_0(2$BR8pTisDuRRm-yj_esdmYd1=&rhu9%dfdzqT}9mpw^7|`B##BU;yNMkBwgz%sVD1I$Dges7MF!4HqE5qQ&vcz z;_0rMvyXJ6Q+sJZQF@o8Fgy|X_|j0vGl|e1A6_56I=t!+-(8-{G8J-Tv7S=mpr_Yz zE3{%@hibthQN-BHktZ$YlFEc|&DV<*`UA`;qnIUg5B<9|c{iHL)vF0UUH#7~a|pY+ zqn_d>wJxIcaK|q>1-3U@_|!lRypBH06~pSh2PUATk+<1wrq(BZMVS^%DaI)ggPOs= zrm_^2=<-c};Wjcv;=PiZDHWexWl`QasK)BomNOpbz1!PE1#*wBVSbs=+XMW{c`u@1 zZVT*8K+(d7>?2^1-XeVI&26_KFFNRvpX-WMpw0_yhh^TOX#|_=w6gC25nm=aDZdvG zl;LEjN#2_NM_G(R*JyfFg~yPNJEE+?x9MGEC@DnPSb~b!uo@4A4zeiu;HU-D^roNe ze9YlP&nv(wZ6&nW0FXHe#6IHZELpC&)Yq)DKJ^E;lvG!#O3@j=jR70dEP>Dv*<;Ek z-$w}hTY4ENC|aP~+}h?Rv?rKWvz{7!Df!M0{s?auvo6>#npPSy%wBF5<2tL!f@qX@ z`-1#t|Go=-Sbm=( zm=QtTPZ7jCB04hXAx1|)ojFv?7RaNbJH?p7COeFvzBYu3_)5LLR)DMFE5K8LpTyo$ zuz8%&a02o2KBgH+-mL|CUi>{aA0vUJwWE6?(0dPEcF@xx;by|M&a`whVYwHnS3z_e z_SyM0^1m5i08oVl+`CaETD;nc0_9j#uVGO-m=o!XJz8LfS|I_i(D$J~31p_1T8Zn- z^e9{4)=-WRNqd)+%BGs6t4PHOwqQ=c?m|ef$u48t#KS<6&yK*rB?M1k7x@YRrprXi zuOCMX&jyd-G%rZE-D}~g86Hr0K{)Yk)TJCZSd7Q$8-fl!zViu0?1`OfZ?*`iFyei7 z`8Dr&T9%?teqET7WxF(V2VI<~4WVLTb;FtdGi`oIpfFDSDCqS!*%f<{HVOJ5l~9-9 z1~0NJV*;klZ4O?aonQ8kK!H}6xl~f4B#o8qNyh^9%R|fJ*WHsYNqQonU2$rf-j#!- z5Z{%Jsq@h$b$;oTIYk5V9XGhvcc3fN&DoV0)MSsxHjVD2v9YW!4Rne(b>M;4^aOUl zIi@Q7j8gtY?|B@%R$9I#wygl(f^C~x)$}glYv24cDJf_znBOEzzst_tKRF(C1kDtV zl1Eo)7bGmeLlZ;&D-6N|d_!tX-)CMQnP^e9d~Y7J325`i9+?-n@ql-cJ4`Ji!)Q!U8=V_<8g9~)NYc?MRbrKZ8%tf`P5_ne&~Y;jowAqqX_d( zT8V5esHer|V^QM~h=XdbkH(xTSa@NXxCL4kI;mb(Wd4^8)TvjXX+{n}P3p=AI<`N? zpN%Jf`sGivKUsfz^QX6ex_Ywndt&6L7QtFp|IS!qqd`}(Wv6To{pgY*g?#~5yMB?D z0qVGqVek!GvGd9-68xw-BUkL98RCE4g&W;25nC#(n4%YSL-o%j_W5h#}4l6 zNp=?@tX~L*&HB!IY{i0Um~JE9wPY%8)_25hJ`&~rWQ2G>3-=1oFgS6KMQ#u~ON|R# z$EJ7>YPP2~Xr@|1Hm!~Pq6xL0w+sE>RTKgy_V3#awPvvHrpp&6rAl9La z*vkzZ##lFb=O}#S1*d$@H1934Txs=p3QqA9>UD%-j{H{Y|KI=fe@JRG#HWaYy{i5-NgG4)&C@j@vM-mgiSoZv z>`&9Rj8;@clu}1Wg^nl1$4q%AH95?3@9 z@o-LVHI1x~QphSkY_4?RTy~05BJPxlGiBmXCDK(wzE%>9QkY@cH;YM(8n6SaW5$Yf zDAM}Skj8CCEFYU>-zWSkzt}2Nr<)s|MpQ}d`d39eWEx^Z{x~)X1TZ~wi5M!|EIe)r zfauJJ(Nyp@Ku%2T*knVEMkTggegZ0a2rQ`oIL|wkGPQWVAkoYh!EM~qI}BG8kFQgRskX2fq=9pX>ObGwbUrL>BRwtA z<(AKD87nRY1{GZs{#wm^D^9ekha&k;QhzGxy>*GmM?UZ9jE(n^HM%Jg+~eW%lHc{{ zv(?CPNefnUprl;|&CxABPII<$J_GA2Vs+8hREpfPb+E2>_*M2SI|!&;;Du%zSf^^D6oG(mQYBr};o>!P^s|Cn}T4rFn zPI$*vYiijGU{TSSHRE{^Q)aSi81Dt)Uczd`PO>VcWF1!S=(4kfLb-=Ko{RcgMt*)N z`;n~%W2)v{~eXRXLF9Uq)J2EsZ=V5P$^smtFK3D3BLYl^WD`R!=P70 zD%seiKrB9OtGP~mD5KlegSytR1k95*O;TSd7K5=uzD-28)HN>64@n?FhRVymR%&h; z>9v~iW(@|N6^%SA8h&z&LB;O0%k1LnwLerZT2%p9MH#r4Vz7dea1Vvy@esnq;Z)dz zAfF>Kqsi&-O9kvH2pz>kreb4H5*rK4;}Ls3apGYH7l*rF(Xg&cO_j(sORad~gQ^(L zZc)@olMJNzbPTi$?t;*s?UFgWh9SW44Oi`;@vG9APaS!EVszjbDZl{WfL@5T+1moFKs-)l&oac2;Z(d1h5oHQn zuELZJEU2z9{jhu_K!r9VWjd`v!>oB?tLo^pit5bN=eJVtFG!?KS5={_F~!uW;Y(6& zcq^;UYW2NRHI(&9ec^s|z0YpF!rW^Wr2&R^wThy^|NDQ7qCo28C#`BraS}1A2I+h^ z`Q2SG-o>n5!Z8jzzF~0}&|E)la9YdyaCJkLYN*^`e{n+QfLFfxSGCxb+VD4ENXTxH z%2jtNmt09G&6s`Cn5Yy<>uVmq%4HY3qwCGApvx{p=~-;q`78|r$8%Ft`f3K@oh2kMc{Nu(zxbvEu~j)_Mu5AQXttt#P^NMximpzqRp`#-c{uvOUY5=y4XM7r3fO2?W54Nz>CXJqo%I= znElcj-&K|j%s`Yl6n2B^s?bV0P9*`WRvmah&Iu?J~9V=%Std4pw4Z{k19S+2KBjNZ=L+*0|Ng5lI2qcd0sHZFgr3fS}ShO=Oj zYhoGK(H)T<7U01_<2R}K-{am0{@khO<=B)5^;goW<=ax~)g6C1JTOY0uT#WyC=;Ra z3{7&(?ysXMoSMJIbN+n|a2lgm<5Xb{h}{rH*T-ysn8{lmdi`zO|1iIUr^uU|p*NeZ z-beBM4=tm+-;?iuFnR~AuD<_a6QB2Zw_tPb;lBA$uf>{%&jI+kKgv2GYhk2i!*92{Auu^cI(aTIp)`Zve6k?ay(YAy(xL` z+jB2)=qa2~Vgf99anl4&`IHYTs#LFCE?vX&MhiS3j6B8aIfr@m+HvKQ@>*CnW-$Ok z%dufiSUP^J{@0hC=Ch`;`(NE=YnS|A@auW=04v`toT&DHK?fdn3=c z0TQfWbayp0$$P}t&6yc^F8na($%s(807Ut>?S4>U6@N5jI27d{smt`}1uw;&CF;%Q zQ$9pooS+$g316u2_~lEXMyg3=fWb@hTQbJ~1rGM%JGuYo;_t8b24^Sd_wA#E{-wVMacKQpSC*ZYz(> zUv6Z_i43-P#BP1qyS|&lc0qTzr+7An_m$LYxjS^e%Fe7Ev7}5>7$tRp!Ye}URLVn4 z_9Ap9Q50z6+Ka9g+9m$NeNn6Z`s=SZW^h}ZJM-w3nhrt1+W93Ztnr@K_&|P0E-zAY zczVGPk57chM3j_hM>x<0KVVCIXVPPimS}GeBh9!WB}(H)o6$EC>W}{D&*S<0{|-Fo I?EvTl0735q9smFU literal 0 HcmV?d00001 From 79b24c8f259df982c9fd0d59ea20181e602670a0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 14:11:55 -0800 Subject: [PATCH 108/117] remove stack from charts --- .../src/components/activity_metrics.test.tsx | 155 ++++++++++++++++++ .../src/components/activity_metrics.tsx | 1 - 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index 02738a46666..308b4b395f7 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -34,8 +34,25 @@ vi.mock("antd", () => { ); PanelComponent.displayName = "Collapse.Panel"; CollapseComponent.Panel = PanelComponent; + const TableComponent = ({ dataSource, columns }: { dataSource?: unknown[]; columns?: { title: string }[] }) => ( + + + + {columns?.map((col, i) => ( + + ))} + + + + {dataSource?.map((_, i) => ( + + ))} + +
{col.title}
+ ); return { Collapse: CollapseComponent, + Table: TableComponent, }; }); @@ -95,6 +112,7 @@ const MOCK_TEAMS: Team[] = [ created_at: "2025-01-01", keys: [], members_with_roles: [], + spend: 0, }, { team_id: "team2", @@ -108,6 +126,7 @@ const MOCK_TEAMS: Team[] = [ created_at: "2025-01-01", keys: [], members_with_roles: [], + spend: 0, }, ]; @@ -141,6 +160,7 @@ const createMockModelActivityData = (label: string, overrides: Partial { expect(tokenElements.length).toBeGreaterThan(0); }); + it("should not display Top Virtual Keys section when model has no top_api_keys", () => { + render(); + expect(screen.queryByText("Top Virtual Keys by Spend")).not.toBeInTheDocument(); + }); + it("should display top API keys section when present", () => { const modelWithTopKeys: Record = { "gpt-4": { @@ -329,6 +355,58 @@ describe("ActivityMetrics", () => { expect(screen.getByText(/Team: team1/)).toBeInTheDocument(); }); + it("should display Model Usage when model has top_models", () => { + const modelWithTopModels: Record = { + "gpt-4": { + ...mockModelMetrics["gpt-4"], + top_models: [ + { + model: "gpt-4", + spend: 100.5, + requests: 100, + successful_requests: 95, + failed_requests: 5, + tokens: 50000, + }, + ], + }, + }; + + render(); + expect(screen.getByRole("heading", { name: "Model Usage" })).toBeInTheDocument(); + }); + + it("should display Spend per day in model section", () => { + render(); + expect(screen.getByText("Spend per day")).toBeInTheDocument(); + }); + + it("should display Requests per day in model section", () => { + render(); + expect(screen.getByText("Requests per day")).toBeInTheDocument(); + }); + + it("should display Success vs Failed Requests in model section", () => { + render(); + expect(screen.getByText("Success vs Failed Requests")).toBeInTheDocument(); + }); + + it("should sort empty string model key last in collapse order", () => { + const modelsWithEmptyKey: Record = { + "gpt-4": { ...mockModelMetrics["gpt-4"] }, + "": { + ...createMockModelActivityData(""), + label: "Unknown", + }, + }; + + render(); + const headings = screen.getAllByRole("heading", { level: 2 }); + const gpt4Index = headings.findIndex((h) => h.textContent?.includes("GPT-4")); + const unknownIndex = headings.findIndex((h) => h.textContent?.includes("Unknown")); + expect(gpt4Index).toBeLessThan(unknownIndex); + }); + it("should display average tokens per successful request", () => { render(); const avgTokensElements = screen.getAllByText(/avg per successful request/); @@ -962,6 +1040,73 @@ describe("processActivityData", () => { expect(result["gpt-4"].top_api_keys[4].spend).toBe(16.0); }); + it("should return empty object when results array is empty", () => { + const result = processActivityData({ results: [] }, "models"); + expect(result).toEqual({}); + }); + + it("should populate top_models for api_keys when models breakdown contains api_key_breakdown for that key", () => { + const dailyActivityWithModelsForKey: { results: DailyData[] } = { + results: [ + { + date: "2025-01-01", + metrics: EMPTY_SPEND_METRICS, + breakdown: { + models: { + "gpt-4": { + metrics: EMPTY_SPEND_METRICS, + metadata: {}, + api_key_breakdown: { + "api-key-hash-1": { + metrics: { + spend: 60.0, + prompt_tokens: 18000, + completion_tokens: 12000, + total_tokens: 30000, + api_requests: 60, + successful_requests: 57, + failed_requests: 3, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: { key_alias: "key-alias-1", team_id: "team1" }, + }, + }, + }, + }, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: { + "api-key-hash-1": { + metrics: { + spend: 60.0, + prompt_tokens: 18000, + completion_tokens: 12000, + total_tokens: 30000, + api_requests: 60, + successful_requests: 57, + failed_requests: 3, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: { key_alias: "key-alias-1", team_id: "team1" }, + }, + }, + entities: {}, + }, + }, + ], + }; + + const result = processActivityData(dailyActivityWithModelsForKey, "api_keys", MOCK_TEAMS); + + expect(result["api-key-hash-1"].top_models).toHaveLength(1); + expect(result["api-key-hash-1"].top_models[0].model).toBe("gpt-4"); + expect(result["api-key-hash-1"].top_models[0].spend).toBe(60.0); + expect(result["api-key-hash-1"].top_models[0].requests).toBe(60); + }); + it("should not process api_key_breakdown when key is api_keys", () => { const dailyActivityWithBreakdown: { results: DailyData[] } = { results: [ @@ -1130,4 +1275,14 @@ describe("formatKeyLabel", () => { const result = formatKeyLabel(modelData, "actual-key", MOCK_TEAMS); expect(result).toBe("key-hash-actual-key (team: Test Team 1)"); }); + + it("should return key_alias with team_id when teams array is empty", () => { + const modelData = createMockKeyMetricWithMetadata({ + key_alias: "my-key", + team_id: "team1", + }); + + const result = formatKeyLabel(modelData, "my-key", []); + expect(result).toBe("my-key (team_id: team1)"); + }); }); diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 61006fa6147..2337d41d8df 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -148,7 +148,6 @@ const ModelSection = ({ categories={["metrics.successful_requests", "metrics.failed_requests"]} colors={["green", "red"]} valueFormatter={valueFormatter} - stack customTooltip={CustomTooltip} showLegend={false} /> From a6f90586ac4a3c1efe97690ebafaa51a87c521f4 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 10 Feb 2026 16:46:40 -0600 Subject: [PATCH 109/117] feat(model-db): add azure_ai/kimi-k2.5 pricing entry (#20896) --- litellm/model_prices_and_context_window_backup.json | 13 +++++++++++++ model_prices_and_context_window.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4812a2d8a10..d794aa50d2e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5848,6 +5848,19 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "azure_ai/kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4812a2d8a10..d794aa50d2e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5848,6 +5848,19 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "azure_ai/kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", From fc0563fab3fdb6e7998675f318158e80051fcf3b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 14:55:37 -0800 Subject: [PATCH 110/117] get pass through include config defined pass through --- litellm/proxy/_types.py | 4 + .../pass_through_endpoints.py | 69 ++++++++++++-- .../test_pass_through_endpoints.py | 94 +++++++++++++++++++ 3 files changed, 160 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fedfa011fdd..06bedfc6c09 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1910,6 +1910,10 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): default=None, description="Guardrails configuration for this passthrough endpoint. Dict keys are guardrail names, values are optional settings for field targeting. When set, all org/team/key level guardrails will also execute. Defaults to None (no guardrails execute).", ) + is_from_config: bool = Field( + default=False, + description="True if this endpoint is defined in the config file, False if from DB. Config-defined endpoints cannot be edited via the UI.", + ) class PassThroughEndpointResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 51a7c37717c..b87ff58db1d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2201,6 +2201,31 @@ async def initialize_pass_through_endpoints( InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_key) +def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint]: + """ + Get pass-through endpoints defined in the config file. + These are read-only and cannot be edited via the UI. + """ + from litellm.proxy.proxy_server import config_passthrough_endpoints + + if config_passthrough_endpoints is None or len(config_passthrough_endpoints) == 0: + return [] + + returned_endpoints: List[PassThroughGenericEndpoint] = [] + for endpoint in config_passthrough_endpoints: + if isinstance(endpoint, dict): + endpoint_dict = dict(endpoint) + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + elif isinstance(endpoint, PassThroughGenericEndpoint): + # Create a copy with is_from_config=True + endpoint_dict = endpoint.model_dump() + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + + return returned_endpoints + + async def _get_pass_through_endpoints_from_db( endpoint_id: Optional[str] = None, user_api_key_dict: Optional[UserAPIKeyAuth] = None, @@ -2223,17 +2248,27 @@ async def _get_pass_through_endpoints_from_db( returned_endpoints: List[PassThroughGenericEndpoint] = [] if endpoint_id is None: - # Return all endpoints + # Return all endpoints from DB, mark as not from config for endpoint in pass_through_endpoint_data: if isinstance(endpoint, dict): - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint)) + endpoint_dict = dict(endpoint) + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) elif isinstance(endpoint, PassThroughGenericEndpoint): - returned_endpoints.append(endpoint) + endpoint_dict = endpoint.model_dump() + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) else: # Find specific endpoint by ID found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) if found_endpoint is not None: - returned_endpoints.append(found_endpoint) + endpoint_dict = ( + found_endpoint.model_dump() + if isinstance(found_endpoint, PassThroughGenericEndpoint) + else dict(found_endpoint) + ) + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) return returned_endpoints @@ -2312,10 +2347,25 @@ async def get_pass_through_endpoints( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - pass_through_endpoints = await _get_pass_through_endpoints_from_db( + # Get endpoints from DB (editable via UI) + db_endpoints = await _get_pass_through_endpoints_from_db( endpoint_id=endpoint_id, user_api_key_dict=user_api_key_dict ) + # Get endpoints from config file (read-only, not editable via UI) + config_endpoints = _get_pass_through_endpoints_from_config() + + # Merge: config endpoints not in DB + all DB endpoints (DB overrides config for same path) + db_paths = {ep.path for ep in db_endpoints} + config_only_endpoints = [ + ep for ep in config_endpoints if ep.path not in db_paths + ] + if endpoint_id is not None: + # When filtering by endpoint_id, only return if found in DB (config endpoints use generated IDs) + pass_through_endpoints = db_endpoints + else: + pass_through_endpoints = config_only_endpoints + db_endpoints + if team_id is not None: pass_through_endpoints = await _filter_endpoints_by_team_allowed_routes( team_id=team_id, @@ -2392,7 +2442,8 @@ async def update_pass_through_endpoints( ) # Get the update data as dict, excluding None values for partial updates - update_data = data.model_dump(exclude_none=True) + # Exclude is_from_config as it's a response-only field (computed at read time) + update_data = data.model_dump(exclude_none=True, exclude={"is_from_config"}) # Start with existing endpoint data endpoint_dict = found_endpoint.model_dump() @@ -2404,6 +2455,9 @@ async def update_pass_through_endpoints( if "id" not in update_data and found_endpoint.id is not None: endpoint_dict["id"] = found_endpoint.id + # Remove is_from_config before saving - it's a response-only field (computed at read time) + endpoint_dict.pop("is_from_config", None) + # Create updated endpoint object updated_endpoint = PassThroughGenericEndpoint(**endpoint_dict) @@ -2490,7 +2544,8 @@ async def create_pass_through_endpoints( ) ## Auto-generate ID if not provided - data_dict = data.model_dump() + # Exclude is_from_config as it's a response-only field (computed at read time) + data_dict = data.model_dump(exclude={"is_from_config"}) if data_dict.get("id") is None: data_dict["id"] = str(uuid.uuid4()) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index daae6d465a7..a64fef1c4b6 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1316,6 +1316,100 @@ async def test_delete_pass_through_endpoint_not_found(): assert "not found" in str(exc_info.value.detail).lower() +@pytest.mark.asyncio +async def test_get_pass_through_endpoints_includes_config_and_db(): + """ + Test that get_pass_through_endpoints returns both config-defined and DB endpoints, + with correct is_from_config flag. Config-only endpoints have is_from_config=True, + DB endpoints have is_from_config=False. When same path exists in both, DB overrides. + """ + from litellm.proxy._types import ( + PassThroughEndpointResponse, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + get_pass_through_endpoints, + ) + + # Config-defined endpoints (from config file) + config_endpoints = [ + { + "path": "/v1/rerank", + "target": "https://api.cohere.com/v1/rerank", + "headers": {"content-type": "application/json"}, + }, + { + "path": "/v1/config-only", + "target": "https://config.example.com/api", + "headers": {}, + }, + ] + + # DB endpoints (one overlaps with config path, one is DB-only) + db_endpoints = [ + { + "id": "db-endpoint-1", + "path": "/v1/rerank", # Same as config - DB should override + "target": "https://db-override.com/v1/rerank", + "headers": {}, + "include_subpath": False, + }, + { + "id": "db-endpoint-2", + "path": "/db/only", + "target": "https://db-only.example.com/api", + "headers": {}, + "include_subpath": False, + }, + ] + + with patch( + "litellm.proxy.proxy_server.prisma_client", + MagicMock(), + ): + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._get_pass_through_endpoints_from_db", + new_callable=AsyncMock, + ) as mock_get_db: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._get_pass_through_endpoints_from_config" + ) as mock_get_config: + db_objects = [ + PassThroughGenericEndpoint(**ep, is_from_config=False) + for ep in db_endpoints + ] + config_objects = [ + PassThroughGenericEndpoint(**ep, is_from_config=True) + for ep in config_endpoints + ] + mock_get_db.return_value = db_objects + mock_get_config.return_value = config_objects + + mock_user = MagicMock(spec=UserAPIKeyAuth) + + result = await get_pass_through_endpoints( + endpoint_id=None, + user_api_key_dict=mock_user, + team_id=None, + ) + + assert isinstance(result, PassThroughEndpointResponse) + # config_only: /v1/config-only (not in db_paths) + # db: /v1/rerank (overrides config), /db/only + # So we should have: /v1/config-only (from config) + /v1/rerank + /db/only (from db) + assert len(result.endpoints) == 3 + + # Check is_from_config values + by_path = {ep.path: ep for ep in result.endpoints} + assert by_path["/v1/config-only"].is_from_config is True + assert by_path["/v1/rerank"].is_from_config is False # DB overrides + assert by_path["/db/only"].is_from_config is False + + # Verify DB override: /v1/rerank should have DB target + assert by_path["/v1/rerank"].target == "https://db-override.com/v1/rerank" + + @pytest.mark.asyncio async def test_delete_pass_through_endpoint_empty_list(): """ From 10d891a36579b012a027902f2d3482e0ba00a0ce Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 10 Feb 2026 15:13:54 -0800 Subject: [PATCH 111/117] Guardrails - add logging to all unified_guardrails + link to custom code guardrail templates (#20900) * feat(guardrail_hooks/): add guardrail logging to all unified guardrails ensures unified guardrails use the 'log_guardrail_information' decorator for logging * fix(custom_guardrail.py): don't log inputs on guardrail response - just emit state * refactor: don't double log bedrock guardrail information * feat: add in-product nudges for contributing + trying community custom code guardrails allows users to contribute / share custom code guardrails --- .circleci/config.yml | 1 + litellm/integrations/custom_guardrail.py | 53 +++++++- .../out/{404.html => 404/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{budgets.html => budgets/index.html} | 0 .../{caching.html => caching/index.html} | 0 .../index.html} | 0 .../{old-usage.html => old-usage/index.html} | 0 .../{prompts.html => prompts/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../out/{login.html => login/index.html} | 0 .../out/{logs.html => logs/index.html} | 0 .../{callback.html => callback/index.html} | 0 .../{model-hub.html => model-hub/index.html} | 0 .../{model_hub.html => model_hub/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{policies.html => policies/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../{ui-theme.html => ui-theme/index.html} | 0 .../out/{teams.html => teams/index.html} | 0 .../{test-key.html => test-key/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../out/{usage.html => usage/index.html} | 0 .../out/{users.html => users/index.html} | 0 .../index.html} | 0 .../guardrail_hooks/bedrock_guardrails.py | 12 +- .../custom_code/custom_code_guardrail.py | 6 +- .../guardrail_hooks/enkryptai/enkryptai.py | 6 +- .../generic_guardrail_api.py | 6 +- .../guardrail_hooks/grayswan/grayswan.py | 43 +++--- .../hiddenlayer/hiddenlayer.py | 6 +- .../litellm_content_filter/content_filter.py | 13 +- .../guardrails/guardrail_hooks/onyx/onyx.py | 12 +- .../guardrail_hooks/openai/moderations.py | 100 +++++++------- .../guardrails/guardrail_hooks/presidio.py | 26 ++-- .../prompt_security/prompt_security.py | 6 +- .../guardrail_hooks/qualifire/qualifire.py | 12 +- .../zscaler_ai_guard/zscaler_ai_guard.py | 16 ++- .../check_guardrail_apply_decorator.py | 126 ++++++++++++++++++ ui/litellm-dashboard/package-lock.json | 15 +++ ui/litellm-dashboard/package.json | 2 +- .../custom_code/CustomCodeModal.tsx | 71 +++++++++- ui/litellm-dashboard/tsconfig.json | 2 +- 53 files changed, 418 insertions(+), 116 deletions(-) rename litellm/proxy/_experimental/out/{404.html => 404/index.html} (100%) rename litellm/proxy/_experimental/out/{_not-found.html => _not-found/index.html} (100%) rename litellm/proxy/_experimental/out/{api-reference.html => api-reference/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{api-playground.html => api-playground/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{budgets.html => budgets/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{caching.html => caching/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{claude-code-plugins.html => claude-code-plugins/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{old-usage.html => old-usage/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{prompts.html => prompts/index.html} (100%) rename litellm/proxy/_experimental/out/experimental/{tag-management.html => tag-management/index.html} (100%) rename litellm/proxy/_experimental/out/{guardrails.html => guardrails/index.html} (100%) rename litellm/proxy/_experimental/out/{login.html => login/index.html} (100%) rename litellm/proxy/_experimental/out/{logs.html => logs/index.html} (100%) rename litellm/proxy/_experimental/out/mcp/oauth/{callback.html => callback/index.html} (100%) rename litellm/proxy/_experimental/out/{model-hub.html => model-hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub.html => model_hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) rename litellm/proxy/_experimental/out/{models-and-endpoints.html => models-and-endpoints/index.html} (100%) rename litellm/proxy/_experimental/out/{onboarding.html => onboarding/index.html} (100%) rename litellm/proxy/_experimental/out/{organizations.html => organizations/index.html} (100%) rename litellm/proxy/_experimental/out/{playground.html => playground/index.html} (100%) rename litellm/proxy/_experimental/out/{policies.html => policies/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{admin-settings.html => admin-settings/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{logging-and-alerts.html => logging-and-alerts/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{router-settings.html => router-settings/index.html} (100%) rename litellm/proxy/_experimental/out/settings/{ui-theme.html => ui-theme/index.html} (100%) rename litellm/proxy/_experimental/out/{teams.html => teams/index.html} (100%) rename litellm/proxy/_experimental/out/{test-key.html => test-key/index.html} (100%) rename litellm/proxy/_experimental/out/tools/{mcp-servers.html => mcp-servers/index.html} (100%) rename litellm/proxy/_experimental/out/tools/{vector-stores.html => vector-stores/index.html} (100%) rename litellm/proxy/_experimental/out/{usage.html => usage/index.html} (100%) rename litellm/proxy/_experimental/out/{users.html => users/index.html} (100%) rename litellm/proxy/_experimental/out/{virtual-keys.html => virtual-keys/index.html} (100%) create mode 100644 tests/code_coverage_tests/check_guardrail_apply_decorator.py diff --git a/.circleci/config.yml b/.circleci/config.yml index e171759f1c4..39182e4c6f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2277,6 +2277,7 @@ jobs: - run: python ./tests/code_coverage_tests/router_code_coverage.py - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py - run: python ./tests/code_coverage_tests/info_log_check.py + - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py - run: python ./tests/code_coverage_tests/code_qa_check_tests.py - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index bbd55a59bce..407bc581f71 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -616,6 +616,7 @@ class CustomGuardrail(CustomLogger): end_time: Optional[float] = None, duration: Optional[float] = None, event_type: Optional[GuardrailEventHooks] = None, + original_inputs: Optional[Dict] = None, ): """ Add StandardLoggingGuardrailInformation to the request data @@ -625,6 +626,17 @@ class CustomGuardrail(CustomLogger): # Convert None to empty dict to satisfy type requirements guardrail_response = {} if response is None else response + # For apply_guardrail functions in custom_code_guardrail scenario, + # simplify the logged response to "allow", "deny", or "mask" + if original_inputs is not None and isinstance(response, dict): + # Check if inputs were modified by comparing them + if self._inputs_were_modified(original_inputs, response): + guardrail_response = "mask" + else: + guardrail_response = "allow" + + verbose_logger.debug(f"Guardrail response: {response}") + self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=guardrail_response, request_data=request_data, @@ -650,8 +662,14 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ + # For custom_code_guardrail scenario, log as "deny" instead of full exception + # Check if this is from custom_code_guardrail by checking the class name + guardrail_response: Union[Exception, str] = e + if "CustomCodeGuardrail" in self.__class__.__name__: + guardrail_response = "deny" + self.add_standard_logging_guardrail_information_to_request_data( - guardrail_json_response=e, + guardrail_json_response=guardrail_response, request_data=request_data, guardrail_status="guardrail_failed_to_respond", duration=duration, @@ -661,6 +679,25 @@ class CustomGuardrail(CustomLogger): ) raise e + def _inputs_were_modified(self, original_inputs: Dict, response: Dict) -> bool: + """ + Compare original inputs with response to determine if content was modified. + + Returns True if the inputs were modified (mask scenario), False otherwise (allow scenario). + """ + # Get all keys from both dictionaries + all_keys = set(original_inputs.keys()) | set(response.keys()) + + # Compare each key's value + for key in all_keys: + original_value = original_inputs.get(key) + response_value = response.get(key) + if original_value != response_value: + return True + + # No modifications detected + return False + def mask_content_in_string( self, content_string: str, @@ -768,6 +805,12 @@ def log_guardrail_information(func): self: CustomGuardrail = args[0] request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {} event_type = _infer_event_type_from_function_name(func.__name__) + + # Store original inputs for comparison (for apply_guardrail functions) + original_inputs = None + if func.__name__ == "apply_guardrail" and "inputs" in kwargs: + original_inputs = kwargs.get("inputs") + try: response = await func(*args, **kwargs) return self._process_response( @@ -777,6 +820,7 @@ def log_guardrail_information(func): end_time=datetime.now().timestamp(), duration=(datetime.now() - start_time).total_seconds(), event_type=event_type, + original_inputs=original_inputs, ) except Exception as e: return self._process_error( @@ -794,6 +838,12 @@ def log_guardrail_information(func): self: CustomGuardrail = args[0] request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {} event_type = _infer_event_type_from_function_name(func.__name__) + + # Store original inputs for comparison (for apply_guardrail functions) + original_inputs = None + if func.__name__ == "apply_guardrail" and "inputs" in kwargs: + original_inputs = kwargs.get("inputs") + try: response = func(*args, **kwargs) return self._process_response( @@ -801,6 +851,7 @@ def log_guardrail_information(func): request_data=request_data, duration=(datetime.now() - start_time).total_seconds(), event_type=event_type, + original_inputs=original_inputs, ) except Exception as e: return self._process_error( diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404/index.html similarity index 100% rename from litellm/proxy/_experimental/out/404.html rename to litellm/proxy/_experimental/out/404/index.html diff --git a/litellm/proxy/_experimental/out/_not-found.html b/litellm/proxy/_experimental/out/_not-found/index.html similarity index 100% rename from litellm/proxy/_experimental/out/_not-found.html rename to litellm/proxy/_experimental/out/_not-found/index.html diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/experimental/api-playground.html b/litellm/proxy/_experimental/out/experimental/api-playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/api-playground.html rename to litellm/proxy/_experimental/out/experimental/api-playground/index.html diff --git a/litellm/proxy/_experimental/out/experimental/budgets.html b/litellm/proxy/_experimental/out/experimental/budgets/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/budgets.html rename to litellm/proxy/_experimental/out/experimental/budgets/index.html diff --git a/litellm/proxy/_experimental/out/experimental/caching.html b/litellm/proxy/_experimental/out/experimental/caching/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/caching.html rename to litellm/proxy/_experimental/out/experimental/caching/index.html diff --git a/litellm/proxy/_experimental/out/experimental/claude-code-plugins.html b/litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/claude-code-plugins.html rename to litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html diff --git a/litellm/proxy/_experimental/out/experimental/old-usage.html b/litellm/proxy/_experimental/out/experimental/old-usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/old-usage.html rename to litellm/proxy/_experimental/out/experimental/old-usage/index.html diff --git a/litellm/proxy/_experimental/out/experimental/prompts.html b/litellm/proxy/_experimental/out/experimental/prompts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/prompts.html rename to litellm/proxy/_experimental/out/experimental/prompts/index.html diff --git a/litellm/proxy/_experimental/out/experimental/tag-management.html b/litellm/proxy/_experimental/out/experimental/tag-management/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/tag-management.html rename to litellm/proxy/_experimental/out/experimental/tag-management/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails/index.html similarity index 100% rename from litellm/proxy/_experimental/out/guardrails.html rename to litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/login.html b/litellm/proxy/_experimental/out/login/index.html similarity index 100% rename from litellm/proxy/_experimental/out/login.html rename to litellm/proxy/_experimental/out/login/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/mcp/oauth/callback.html b/litellm/proxy/_experimental/out/mcp/oauth/callback/index.html similarity index 100% rename from litellm/proxy/_experimental/out/mcp/oauth/callback.html rename to litellm/proxy/_experimental/out/mcp/oauth/callback/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub.html rename to litellm/proxy/_experimental/out/model_hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding/index.html similarity index 100% rename from litellm/proxy/_experimental/out/onboarding.html rename to litellm/proxy/_experimental/out/onboarding/index.html diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/playground.html b/litellm/proxy/_experimental/out/playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/playground.html rename to litellm/proxy/_experimental/out/playground/index.html diff --git a/litellm/proxy/_experimental/out/policies.html b/litellm/proxy/_experimental/out/policies/index.html similarity index 100% rename from litellm/proxy/_experimental/out/policies.html rename to litellm/proxy/_experimental/out/policies/index.html diff --git a/litellm/proxy/_experimental/out/settings/admin-settings.html b/litellm/proxy/_experimental/out/settings/admin-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/admin-settings.html rename to litellm/proxy/_experimental/out/settings/admin-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/logging-and-alerts.html b/litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/logging-and-alerts.html rename to litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html diff --git a/litellm/proxy/_experimental/out/settings/router-settings.html b/litellm/proxy/_experimental/out/settings/router-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/router-settings.html rename to litellm/proxy/_experimental/out/settings/router-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/ui-theme.html b/litellm/proxy/_experimental/out/settings/ui-theme/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/ui-theme.html rename to litellm/proxy/_experimental/out/settings/ui-theme/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/tools/mcp-servers.html b/litellm/proxy/_experimental/out/tools/mcp-servers/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/mcp-servers.html rename to litellm/proxy/_experimental/out/tools/mcp-servers/index.html diff --git a/litellm/proxy/_experimental/out/tools/vector-stores.html b/litellm/proxy/_experimental/out/tools/vector-stores/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/vector-stores.html rename to litellm/proxy/_experimental/out/tools/vector-stores/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index f8fba5f5984..6800dff55ac 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -795,9 +795,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 1. Make the Bedrock API request ########## ######################################################### - bedrock_guardrail_response: Optional[ - Union[BedrockGuardrailResponse, str] - ] = None + bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = ( + None + ) try: bedrock_guardrail_response = await self.make_bedrock_api_request( source="INPUT", messages=filtered_messages, request_data=data @@ -867,9 +867,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 1. Make the Bedrock API request ########## ######################################################### - bedrock_guardrail_response: Optional[ - Union[BedrockGuardrailResponse, str] - ] = None + bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = ( + None + ) try: bedrock_guardrail_response = await self.make_bedrock_api_request( source="INPUT", messages=filtered_messages, request_data=data diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index 68f9dfd7abc..66b80c10f18 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -35,7 +35,10 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Type, cast from fastapi import HTTPException from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel from litellm.types.utils import GenericGuardrailAPIInputs @@ -179,6 +182,7 @@ class CustomCodeGuardrail(CustomGuardrail): self._compile_error = f"Failed to compile custom code: {e}" raise CustomCodeCompilationError(self._compile_error) from e + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py index 8e992297e5d..63541a1e2f9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py @@ -23,7 +23,10 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -483,6 +486,7 @@ class EnkryptAIGuardrails(CustomGuardrail): request_data=data, guardrail_name=self.guardrail_name ) + @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index b37074e25e7..9018675d7a5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -10,7 +10,10 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -150,6 +153,7 @@ class GenericGuardrailAPI(CustomGuardrail): return result_metadata + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index 90f689ed23c..8955bffc125 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -9,7 +9,8 @@ from fastapi import HTTPException from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, - ModifyResponseException + ModifyResponseException, + log_guardrail_information, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -108,7 +109,9 @@ class GraySwanGuardrail(CustomGuardrail): self.categories = categories self.policy_id = policy_id self.fail_open = True if fail_open is None else bool(fail_open) - self.guardrail_timeout = 30.0 if guardrail_timeout is None else float(guardrail_timeout) + self.guardrail_timeout = ( + 30.0 if guardrail_timeout is None else float(guardrail_timeout) + ) # Streaming configuration self.streaming_end_of_stream_only = streaming_end_of_stream_only @@ -155,6 +158,7 @@ class GraySwanGuardrail(CustomGuardrail): # Unified Guardrail Interface (works with ALL endpoints automatically) # ------------------------------------------------------------------ + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, @@ -208,7 +212,9 @@ class GraySwanGuardrail(CustomGuardrail): messages = [{"role": role, "content": text} for text in texts] # Get dynamic params from request metadata - dynamic_body = self.get_guardrail_dynamic_request_body_params(request_data) or {} + dynamic_body = ( + self.get_guardrail_dynamic_request_body_params(request_data) or {} + ) if dynamic_body: verbose_proxy_logger.debug( "Gray Swan Guardrail: dynamic extra_body=%s", safe_dumps(dynamic_body) @@ -271,12 +277,12 @@ class GraySwanGuardrail(CustomGuardrail): async def run_grayswan_guardrail(self, payload: dict) -> Dict[str, Any]: """ Run the GraySwan guardrail on a payload. - + This is a legacy method for testing purposes. - + Args: payload: The payload to scan - + Returns: Dict containing the GraySwan API response """ @@ -293,11 +299,11 @@ class GraySwanGuardrail(CustomGuardrail): ) -> None: """ Legacy method for processing GraySwan API responses. - + This method is maintained for backward compatibility with existing tests. It handles the test scenarios where responses need to be processed with knowledge of the request context (pre/during/post call hooks). - + Args: response_json: Response from GraySwan API data: Optional request data (for passthrough exceptions) @@ -365,7 +371,10 @@ class GraySwanGuardrail(CustomGuardrail): ) # If hook_type is provided and in pre/during call, raise exception - if hook_type in [GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call]: + if hook_type in [ + GuardrailEventHooks.pre_call, + GuardrailEventHooks.during_call, + ]: # Raise ModifyResponseException to short-circuit LLM call if data is None: data = {} @@ -540,7 +549,9 @@ class GraySwanGuardrail(CustomGuardrail): if isinstance(litellm_metadata, dict) and litellm_metadata: cleaned_litellm_metadata = dict(litellm_metadata) # cleaned_litellm_metadata.pop("user_api_key_auth", None) - sanitized = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + sanitized = safe_json_loads( + safe_dumps(cleaned_litellm_metadata), default={} + ) if isinstance(sanitized, dict) and sanitized: payload["litellm_metadata"] = sanitized @@ -566,7 +577,9 @@ class GraySwanGuardrail(CustomGuardrail): detection_info = detection_info[0] # Extract fields from detection_info dict - detection_dict: dict = detection_info if isinstance(detection_info, dict) else {} + detection_dict: dict = ( + detection_info if isinstance(detection_info, dict) else {} + ) violation_score = detection_dict.get("violation_score", 0.0) violated_rules = detection_dict.get("violated_rules", []) mutation = detection_dict.get("mutation", False) @@ -582,7 +595,9 @@ class GraySwanGuardrail(CustomGuardrail): if violated_rules: formatted_rules = self._format_violated_rules(violated_rules) if formatted_rules: - message_parts.append(f"It was violating the rule(s): {formatted_rules}.") + message_parts.append( + f"It was violating the rule(s): {formatted_rules}." + ) if mutation: message_parts.append( @@ -590,9 +605,7 @@ class GraySwanGuardrail(CustomGuardrail): ) if ipi: - message_parts.append( - "Indirect Prompt Injection was DETECTED." - ) + message_parts.append("Indirect Prompt Injection was DETECTED.") return "\n".join(message_parts) diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index e2c20604880..b907fbbcbda 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -10,7 +10,10 @@ from httpx import HTTPStatusError from requests.auth import HTTPBasicAuth from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -110,6 +113,7 @@ class HiddenlayerGuardrail(CustomGuardrail): ) super().__init__(**kwargs) + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 083a407e9cf..263b6eee768 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -28,7 +28,10 @@ from fastapi import HTTPException from litellm import Router from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -50,6 +53,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter impor ContentFilterDetection, PatternDetection, ) + from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS = 1 @@ -168,9 +172,9 @@ class ContentFilterGuardrail(CustomGuardrail): self.image_model = image_model # Store loaded categories self.loaded_categories: Dict[str, CategoryConfig] = {} - self.category_keywords: Dict[ - str, Tuple[str, str, ContentFilterAction] - ] = {} # keyword -> (category, severity, action) + self.category_keywords: Dict[str, Tuple[str, str, ContentFilterAction]] = ( + {} + ) # keyword -> (category, severity, action) # Load categories if provided if categories: @@ -994,6 +998,7 @@ class ContentFilterGuardrail(CustomGuardrail): masked_entity_count=masked_entity_count, ) + @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 3598dbe741e..1cfc805dbf9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -12,7 +12,10 @@ import httpx from fastapi import HTTPException from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -26,7 +29,11 @@ if TYPE_CHECKING: class OnyxGuardrail(CustomGuardrail): def __init__( - self, api_base: Optional[str] = None, api_key: Optional[str] = None, timeout: Optional[float] = 10.0, **kwargs + self, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + timeout: Optional[float] = 10.0, + **kwargs, ): timeout = timeout or int(os.getenv("ONYX_TIMEOUT", 10.0)) self.async_handler = get_async_httpx_client( @@ -79,6 +86,7 @@ class OnyxGuardrail(CustomGuardrail): ) return result + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 030b6036815..a196937ef6c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -58,7 +58,9 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): guardrail_name: str, api_key: Optional[str] = None, api_base: Optional[str] = None, - model: Optional[Literal["omni-moderation-latest", "text-moderation-latest"]] = None, + model: Optional[ + Literal["omni-moderation-latest", "text-moderation-latest"] + ] = None, **kwargs, ): """Initialize OpenAI Moderation guardrail handler.""" @@ -75,7 +77,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): supported_event_hooks=supported_event_hooks, **kwargs, ) - + self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback ) @@ -83,10 +85,14 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): # Store configuration self.api_key = api_key or self._get_api_key() self.api_base = api_base or "https://api.openai.com/v1" - self.model: Literal["omni-moderation-latest", "text-moderation-latest"] = model or "omni-moderation-latest" + self.model: Literal["omni-moderation-latest", "text-moderation-latest"] = ( + model or "omni-moderation-latest" + ) if not self.api_key: - raise ValueError("OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration.") + raise ValueError( + "OpenAI Moderation: api_key is required. Set OPENAI_API_KEY environment variable or pass it in configuration." + ) verbose_proxy_logger.debug( f"Initialized OpenAI Moderation Guardrail: {guardrail_name} with model: {self.model}" @@ -98,7 +104,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): import litellm from litellm.secret_managers.main import get_secret_str - + return ( os.environ.get("OPENAI_API_KEY") or litellm.api_key @@ -106,21 +112,14 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): or get_secret_str("OPENAI_API_KEY") ) - async def async_make_request( - self, input_text: str - ) -> "OpenAIModerationResponse": + async def async_make_request(self, input_text: str) -> "OpenAIModerationResponse": """ Make a request to the OpenAI Moderation API. """ - request_body = { - "model": self.model, - "input": input_text - } - - verbose_proxy_logger.debug( - "OpenAI Moderation guard request: %s", request_body - ) - + request_body = {"model": self.model, "input": input_text} + + verbose_proxy_logger.debug("OpenAI Moderation guard request: %s", request_body) + response = await self.async_handler.post( url=f"{self.api_base}/moderations", headers={ @@ -133,7 +132,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): verbose_proxy_logger.debug( "OpenAI Moderation guard response: %s", response.json() ) - + if response.status_code != 200: raise HTTPException( status_code=response.status_code, @@ -144,9 +143,12 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) from litellm.types.llms.openai import OpenAIModerationResponse + return OpenAIModerationResponse(**response.json()) - def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None: + def _check_moderation_result( + self, moderation_response: "OpenAIModerationResponse" + ) -> None: """ Check if the moderation response indicates harmful content and raise exception if needed. """ @@ -168,10 +170,10 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): } verbose_proxy_logger.warning( - "OpenAI Moderation: Content flagged for violations: %s", - violation_details + "OpenAI Moderation: Content flagged for violations: %s", + violation_details, ) - + raise HTTPException( status_code=400, detail={ @@ -180,6 +182,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): }, ) + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, @@ -189,51 +192,50 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) -> GenericGuardrailAPIInputs: """ Apply OpenAI moderation guardrail using the unified guardrail interface. - + This method is called by the UnifiedLLMGuardrails system for all endpoint types (chat completions, embeddings, responses API, etc.). - + Args: inputs: GenericGuardrailAPIInputs containing texts and/or structured_messages request_data: The original request data input_type: Whether this is a "request" (pre-call) or "response" (post-call) logging_obj: Optional logging object - + Returns: The inputs unchanged (moderation doesn't modify content, only blocks) - + Raises: HTTPException: If content violates moderation policy """ # Extract text to moderate from inputs text_to_moderate: Optional[str] = None - + # Prefer structured_messages if available (has role context) if structured_messages := inputs.get("structured_messages"): text_to_moderate = self.get_user_prompt(structured_messages) - + # Fall back to texts if not text_to_moderate: if texts := inputs.get("texts"): # Join all texts for moderation text_to_moderate = "\n".join(texts) - + if not text_to_moderate: verbose_proxy_logger.debug( "OpenAI Moderation: No text content to moderate in inputs" ) return inputs - + # Make moderation request moderation_response = await self.async_make_request(input_text=text_to_moderate) - + # Check if content is flagged and raise exception if needed self._check_moderation_result(moderation_response) - + # Moderation doesn't modify content, just blocks - return inputs unchanged return inputs - @log_guardrail_information async def async_post_call_streaming_iterator_hook( self, @@ -252,9 +254,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): from litellm.main import stream_chunk_builder from litellm.types.utils import TextCompletionResponse - verbose_proxy_logger.debug( - "OpenAI Moderation: Running streaming response scan" - ) + verbose_proxy_logger.debug("OpenAI Moderation: Running streaming response scan") # Collect all chunks to process them together all_chunks: List["ModelResponseStream"] = [] @@ -269,7 +269,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): ) if isinstance(assembled_model_response, (type(None), TextCompletionResponse)): - # If we can't assemble a ModelResponse or it's a text completion, + # If we can't assemble a ModelResponse or it's a text completion, # just yield the original chunks without moderation verbose_proxy_logger.warning( "OpenAI Moderation: Could not assemble ModelResponse from chunks, skipping moderation" @@ -284,19 +284,17 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): verbose_proxy_logger.debug( f"OpenAI Moderation: Streaming response text: {response_text[:100]}..." # Log first 100 chars ) - + # Make moderation request - this will raise HTTPException if content is flagged moderation_response = await self.async_make_request( input_text=response_text, ) - + # Check if content is flagged and raise exception if needed self._check_moderation_result(moderation_response) # If we reach here, content passed moderation - yield the original chunks - mock_response = MockResponseIterator( - model_response=assembled_model_response - ) + mock_response = MockResponseIterator(model_response=assembled_model_response) # Return the reconstructed stream async for chunk in mock_response: @@ -306,34 +304,34 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): """ Extract text content from the model response for moderation. """ - if not hasattr(response, 'choices') or not response.choices: + if not hasattr(response, "choices") or not response.choices: return None response_texts = [] for choice in response.choices: try: # Try to get content from message (chat completion) - message = getattr(choice, 'message', None) + message = getattr(choice, "message", None) if message: - content = getattr(message, 'content', None) + content = getattr(message, "content", None) if content and isinstance(content, str): response_texts.append(content) continue - + # Try to get text (text completion) - text = getattr(choice, 'text', None) + text = getattr(choice, "text", None) if text and isinstance(text, str): response_texts.append(text) continue - + # Try to get content from delta (streaming) - delta = getattr(choice, 'delta', None) + delta = getattr(choice, "delta", None) if delta: - content = getattr(delta, 'content', None) + content = getattr(delta, "content", None) if content and isinstance(content, str): response_texts.append(content) continue - + except (AttributeError, TypeError): # Skip choices that don't have expected attributes continue diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 71ad9819146..d71b8449f94 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -9,10 +9,10 @@ import asyncio -import threading import json -from datetime import datetime +import threading from contextlib import asynccontextmanager +from datetime import datetime from typing import ( TYPE_CHECKING, Any, @@ -39,7 +39,10 @@ if TYPE_CHECKING: from litellm._uuid import uuid from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import ( GuardrailEventHooks, @@ -568,9 +571,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if messages is None: return data tasks = [] - task_mappings: List[ - Tuple[int, Optional[int]] - ] = [] # Track (message_index, content_index) for each task + task_mappings: List[Tuple[int, Optional[int]]] = ( + [] + ) # Track (message_index, content_index) for each task for msg_idx, m in enumerate(messages): content = m.get("content", None) @@ -671,9 +674,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ): # /chat/completions requests messages: Optional[List] = kwargs.get("messages", None) tasks = [] - task_mappings: List[ - Tuple[int, Optional[int]] - ] = [] # Track (message_index, content_index) for each task + task_mappings: List[Tuple[int, Optional[int]]] = ( + [] + ) # Track (message_index, content_index) for each task if messages is None: return kwargs, result @@ -792,11 +795,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): # Type narrowing: StreamingChoices doesn't have .message attribute if not hasattr(choice, "message"): continue - content = getattr(choice.message, "content", None) + content = getattr(choice.message, "content", None) # type: ignore if content is None: continue if isinstance(content, str): - choice.message.content = await self.check_pii( + choice.message.content = await self.check_pii( # type: ignore text=content, output_parse_pii=False, presidio_config=presidio_config, @@ -989,6 +992,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): except Exception: pass + @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 5ebc7b96eb8..b3e761869b0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -6,7 +6,10 @@ from typing import TYPE_CHECKING, Any, List, Literal, Optional, Type from fastapi import HTTPException from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -67,6 +70,7 @@ class PromptSecurityGuardrail(CustomGuardrail): super().__init__(**kwargs) + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index 87da11efad0..6486da7f714 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -12,10 +12,11 @@ from typing import Any, Dict, List, Literal, Optional, Type from fastapi import HTTPException from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.litellm_logging import ( - Logging as LiteLLMLoggingObj, +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, ) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -343,9 +344,7 @@ class QualifireGuardrail(CustomGuardrail): ) url = f"{self.qualifire_api_base}/api/evaluation/evaluate" - verbose_proxy_logger.debug( - f"Qualifire Guardrail: Making request to {url}" - ) + verbose_proxy_logger.debug(f"Qualifire Guardrail: Making request to {url}") # Make the API request response = await self.async_handler.post( @@ -393,6 +392,7 @@ class QualifireGuardrail(CustomGuardrail): verbose_proxy_logger.exception(f"Qualifire Guardrail error: {e}") raise + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index c60752d7952..ff00cd73ca5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -9,7 +9,10 @@ from typing import TYPE_CHECKING, Literal, Optional from fastapi import HTTPException from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -70,6 +73,7 @@ class ZscalerAIGuard(CustomGuardrail): return str(value).strip() return "N/A" + @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", @@ -92,7 +96,7 @@ class ZscalerAIGuard(CustomGuardrail): Raises: Exception: If content is blocked by Zscaler AI Guard """ - + texts = inputs.get("texts", []) try: verbose_proxy_logger.debug(f"ZscalerAIGuard: Checking {len(texts)} text(s)") @@ -102,8 +106,8 @@ class ZscalerAIGuard(CustomGuardrail): team_metadata = metadata.get("team_metadata", {}) or {} # Precedence for policy_id: - # 1. metadata.zguard_policy_id # request level - # 2. user_api_key_metadata.zguard_policy_id # Key level + # 1. metadata.zguard_policy_id # request level + # 2. user_api_key_metadata.zguard_policy_id # Key level # 3. team_metadata.zguard_policy_id # Team level # 4. self.policy_id (from environment) # Global policy_id = ( @@ -154,9 +158,7 @@ class ZscalerAIGuard(CustomGuardrail): zscaler_ai_guard_result and zscaler_ai_guard_result.get("action") == "BLOCK" ): - blocking_info = zscaler_ai_guard_result.get( - "zscaler_ai_guard_response" - ) + blocking_info = zscaler_ai_guard_result.get("zscaler_ai_guard_response") error_message = f"Content blocked by Zscaler AI Guard: {self.extract_blocking_info(blocking_info)}" raise Exception(error_message) except Exception as e: diff --git a/tests/code_coverage_tests/check_guardrail_apply_decorator.py b/tests/code_coverage_tests/check_guardrail_apply_decorator.py new file mode 100644 index 00000000000..18a86277aa9 --- /dev/null +++ b/tests/code_coverage_tests/check_guardrail_apply_decorator.py @@ -0,0 +1,126 @@ +""" +Test that all guardrail hooks with async def apply_guardrail use @log_guardrail_information decorator. + +This ensures consistent logging and observability across all guardrail implementations. +""" + +import ast +from pathlib import Path +from typing import List, Tuple + + +def find_apply_guardrail_methods(file_path: Path) -> List[Tuple[str, int, bool]]: + """ + Find all apply_guardrail methods and check if they have the decorator. + + Returns: + List of tuples: (class_name, line_number, has_decorator) + """ + with open(file_path, "r") as f: + content = f.read() + + try: + tree = ast.parse(content) + except SyntaxError: + return [] + + results = [] + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + class_name = node.name + + # Check if this class has apply_guardrail method + for item in node.body: + if ( + isinstance(item, ast.AsyncFunctionDef) + and item.name == "apply_guardrail" + ): + # Check if it has the log_guardrail_information decorator + has_decorator = False + for decorator in item.decorator_list: + if ( + isinstance(decorator, ast.Name) + and decorator.id == "log_guardrail_information" + ): + has_decorator = True + break + + results.append((class_name, item.lineno, has_decorator)) + + return results + + +def test_guardrail_apply_decorator(): + """Test that all guardrail hooks with apply_guardrail have the decorator.""" + # Path to the guardrail hooks directory + guardrail_hooks_dir = ( + Path(__file__).parent.parent.parent + / "litellm" + / "proxy" + / "guardrails" + / "guardrail_hooks" + ) + + # Find all Python files in the guardrail hooks directory + python_files = list(guardrail_hooks_dir.rglob("*.py")) + + # Track violations + violations = [] + + for python_file in python_files: + # Skip __init__.py files and test files + if python_file.name == "__init__.py" or python_file.name.startswith("test_"): + continue + + # Skip base files and primitives + if python_file.name in ["base.py", "primitives.py", "patterns.py"]: + continue + + # Skip bedrock_guardrails.py - it implements logging differently via + # add_standard_logging_guardrail_information_to_request_data calls + # in make_bedrock_api_request method instead of using the decorator + if python_file.name == "bedrock_guardrails.py": + continue + + results = find_apply_guardrail_methods(python_file) + + for class_name, line_num, has_decorator in results: + if not has_decorator: + relative_path = python_file.relative_to( + Path(__file__).parent.parent.parent + ) + violations.append((relative_path, class_name, line_num)) + + # Assert no violations found + if violations: + print( + f"\nFound {len(violations)} guardrail hook(s) without @log_guardrail_information decorator:" + ) + print( + "\nAll guardrail hooks must use @log_guardrail_information decorator on their apply_guardrail method." + ) + print( + "This ensures consistent logging and observability across all guardrails.\n" + ) + + for file_path, class_name, line_num in violations: + print(f" - {file_path}:{line_num} ({class_name}.apply_guardrail)") + + print("\nTo fix, add the decorator:") + print( + " from litellm.integrations.custom_guardrail import log_guardrail_information" + ) + print(" ") + print(" @log_guardrail_information") + print(" async def apply_guardrail(self, ...):") + print(" ...") + + raise AssertionError( + f"Found {len(violations)} guardrail hook(s) without @log_guardrail_information decorator" + ) + + +if __name__ == "__main__": + test_guardrail_apply_decorator() + print("✓ All guardrail hooks have @log_guardrail_information decorator") diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 4205657ca8c..3a21813fbf4 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -13159,6 +13159,21 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", + "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } } } } diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 76ac97f008c..74caf14a594 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -3,7 +3,7 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev --webpack", + "dev": "next dev", "build": "next build", "start": "next start", "lint": "next lint", diff --git a/ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeModal.tsx b/ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeModal.tsx index fb1b967334c..866d24df50b 100644 --- a/ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeModal.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeModal.tsx @@ -1,5 +1,5 @@ import React, { useState, useRef, useEffect } from "react"; -import { Modal, Select, Switch, Collapse, Input } from "antd"; +import { Modal, Select, Switch, Collapse, Input, Divider } from "antd"; import { Button, TextInput } from "@tremor/react"; import { CodeOutlined, @@ -8,6 +8,8 @@ import { CloseCircleOutlined, CaretRightOutlined, SaveOutlined, + UsergroupAddOutlined, + ExportOutlined, } from "@ant-design/icons"; import { createGuardrailCall, updateGuardrailCall, testCustomCodeGuardrail } from "../../networking"; import NotificationsManager from "../../molecules/notifications_manager"; @@ -91,6 +93,7 @@ const CODE_TEMPLATES = { }, }; + // Available primitives organized by category const PRIMITIVES = { "Return Values": [ @@ -241,6 +244,8 @@ const CustomCodeModal: React.FC = ({ // Handle template change const handleTemplateChange = (templateKey: string) => { setSelectedTemplate(templateKey); + + // Check if it's a standard template setCode(CODE_TEMPLATES[templateKey as keyof typeof CODE_TEMPLATES].code); }; @@ -486,12 +491,45 @@ const CustomCodeModal: React.FC = ({ onChange={handleTemplateChange} className="w-full" size="middle" + dropdownRender={(menu) => ( + <> + {menu} + +
{ + e.preventDefault(); + window.open('https://models.litellm.ai/guardrails', '_blank'); + }} + onMouseEnter={(e) => { + e.currentTarget.style.backgroundColor = '#f0f0f0'; + }} + onMouseLeave={(e) => { + e.currentTarget.style.backgroundColor = 'transparent'; + }} + > + + Browse Community templates + +
+ + )} > - {Object.entries(CODE_TEMPLATES).map(([key, template]) => ( - - {template.name} - - ))} + + {Object.entries(CODE_TEMPLATES).map(([key, template]) => ( + + {template.name} + + ))} +
@@ -632,6 +670,27 @@ const CustomCodeModal: React.FC = ({
+ {/* Contribution CTA Banner */} +
+
+
+ +
+
+
Built a useful guardrail?
+
Share it with the community and help others build faster
+
+
+ +
+
{/* Primitives Panel */} diff --git a/ui/litellm-dashboard/tsconfig.json b/ui/litellm-dashboard/tsconfig.json index d24bdd340f7..5b0352feb98 100644 --- a/ui/litellm-dashboard/tsconfig.json +++ b/ui/litellm-dashboard/tsconfig.json @@ -14,7 +14,7 @@ "moduleResolution": "bundler", "resolveJsonModule": true, "isolatedModules": true, - "jsx": "react-jsx", + "jsx": "preserve", "incremental": true, "plugins": [ { From e002d6afe87b5fa62b851ec39bbc3ad3733d80a5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 15:16:18 -0800 Subject: [PATCH 112/117] addressing comments --- .../pass_through_endpoints.py | 28 +++++++++++----- .../test_pass_through_endpoints.py | 33 +++++++++++++++++++ 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b87ff58db1d..a7b60c8b185 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2205,7 +2205,10 @@ def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint """ Get pass-through endpoints defined in the config file. These are read-only and cannot be edited via the UI. + Malformed endpoints are logged and skipped; they do not crash the function. """ + from pydantic import ValidationError + from litellm.proxy.proxy_server import config_passthrough_endpoints if config_passthrough_endpoints is None or len(config_passthrough_endpoints) == 0: @@ -2213,15 +2216,22 @@ def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint returned_endpoints: List[PassThroughGenericEndpoint] = [] for endpoint in config_passthrough_endpoints: - if isinstance(endpoint, dict): - endpoint_dict = dict(endpoint) - endpoint_dict["is_from_config"] = True - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - elif isinstance(endpoint, PassThroughGenericEndpoint): - # Create a copy with is_from_config=True - endpoint_dict = endpoint.model_dump() - endpoint_dict["is_from_config"] = True - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + try: + if isinstance(endpoint, dict): + endpoint_dict = dict(endpoint) + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + elif isinstance(endpoint, PassThroughGenericEndpoint): + # Create a copy with is_from_config=True + endpoint_dict = endpoint.model_dump() + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + except ValidationError as e: + verbose_proxy_logger.warning( + "Skipping malformed pass-through endpoint from config: %s", + e, + exc_info=False, + ) return returned_endpoints diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a64fef1c4b6..e50e10352e2 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1410,6 +1410,39 @@ async def test_get_pass_through_endpoints_includes_config_and_db(): assert by_path["/v1/rerank"].target == "https://db-override.com/v1/rerank" +def test_get_pass_through_endpoints_from_config_skips_malformed(): + """ + Test that _get_pass_through_endpoints_from_config skips malformed endpoints + and returns only valid ones, without raising. + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + _get_pass_through_endpoints_from_config, + ) + + # Mix of valid and malformed config endpoints + config_passthrough_endpoints = [ + {"path": "/valid/1", "target": "https://valid1.example.com"}, + {}, # Missing required path and target + {"path": "/missing-target"}, # Missing required target + {"target": "https://example.com"}, # Missing required path + {"path": "/valid/2", "target": "https://valid2.example.com", "headers": {}}, + ] + + with patch( + "litellm.proxy.proxy_server.config_passthrough_endpoints", + config_passthrough_endpoints, + ): + result = _get_pass_through_endpoints_from_config() + + # Only the 2 valid endpoints should be returned + assert len(result) == 2 + paths = {ep.path for ep in result} + assert "/valid/1" in paths + assert "/valid/2" in paths + for ep in result: + assert ep.is_from_config is True + + @pytest.mark.asyncio async def test_delete_pass_through_endpoint_empty_list(): """ From f8619e2000e66dba5c98a09ce0ef517a97f2ff1b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 10 Feb 2026 15:17:01 -0800 Subject: [PATCH 113/117] [Stability] Investigate + fix issue where model cost map became poorly formatted (#20895) * init: GetModelCostMap * fix * docs * docs fix * docs fixes * docs fix * test model cost map resilience * MODEL_COST_MAP_MIN_MODEL_COUNT * validate_model_cost_map * test_should_have_minimum_models_in_backup * docs fix * docs fix * fix * dos fix * docs fix * docs fix * docs fix * docs fix * validate_model_cost_map * fix * cleanup --- .../blog/model_cost_map_incident/index.md | 95 ++++++ docs/my-website/sidebars.js | 11 + litellm/constants.py | 8 + .../litellm_core_utils/get_model_cost_map.py | 201 ++++++++++-- .../test_model_cost_map_resilience.py | 291 ++++++++++++++++++ 5 files changed, 579 insertions(+), 27 deletions(-) create mode 100644 docs/my-website/blog/model_cost_map_incident/index.md create mode 100644 tests/llm_translation/test_model_cost_map_resilience.py diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md new file mode 100644 index 00000000000..7f1324e5fab --- /dev/null +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -0,0 +1,95 @@ +--- +slug: model-cost-map-incident +title: "Incident Report: Invalid model cost map on main" +date: 2026-02-10T10:00:00 +authors: + - name: Ishaan Jaffer + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/ishaanjaffer/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [incident-report, stability] +hide_table_of_contents: false +--- + +**Date:** January 27, 2026 +**Duration:** ~20 minutes +**Severity:** Low +**Status:** Resolved + +## Summary + +A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked. + +- **LLM calls and proxy routing:** No impact. +- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted. + +{/* truncate */} + +--- + +## Background + +The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call. + +```mermaid +flowchart TD + A["1. litellm.completion() receives request + litellm/main.py"] --> B["2. Route to provider + litellm/litellm_core_utils/get_llm_provider_logic.py"] + B --> C["3. LLM returns response + litellm/main.py"] + C --> D["4. Post-call: look up model in cost map + litellm/cost_calculator.py"] + D -->|"found"| E["5a. Attach cost to response"] + D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"] + E --> G["6. Return response to caller"] + F --> G + + style D fill:#fff3cd,stroke:#ffc107 + style F fill:#fff3cd,stroke:#ffc107 + style E fill:#d4edda,stroke:#28a745 + style G fill:#d4edda,stroke:#28a745 +``` + +Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request. + +--- + +## Root cause + +LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged. + +A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models. + +**Timeline:** + +1. Malformed JSON merged to `main` +2. LiteLLM installations fall back to local backup on next import +3. Users report `"This model isn't mapped yet"` for newer models +4. Bad commit identified and reverted (~20 minutes) + +--- + +## Remediation + +| # | Action | Status | Code | +|---|---|---|---| +| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [PR #20605](https://github.com/BerriAI/litellm/pull/20605) | +| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | +| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | +| 4 | Resilience test suite (bad hosted map, bad backup, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | +| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | + +Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely. + +--- + +## Other dependencies on external resources + +| Dependency | Impact if unavailable | Fallback | +|---|---|---| +| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) | +| JWT public keys (IDP/SSO) | Auth fails | None | +| OIDC UserInfo (IDP/SSO) | Auth fails | None | +| HuggingFace model API | HF provider calls fail | None | +| Ollama tags (localhost) | Ollama model list stale | Static list | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 2c3dfb2b863..28e1724ef82 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1085,6 +1085,17 @@ const sidebars = { "troubleshoot/max_callbacks", ], }, + { + type: "category", + label: "Blog", + items: [ + { + type: "link", + label: "Incident: Broken Model Cost Map", + href: "/blog/model-cost-map-incident", + }, + ], + }, ], }; diff --git a/litellm/constants.py b/litellm/constants.py index 9c25cf77906..180315ace0e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -48,6 +48,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int( os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) ) DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) + +# Model cost map validation constants +MODEL_COST_MAP_MIN_MODEL_COUNT = int( + os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50) +) # Minimum number of models a fetched cost map must contain to be considered valid +MODEL_COST_MAP_MAX_SHRINK_RATIO = float( + os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5) +) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup) DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) # Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 9b86f4ca2f0..e622a317454 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -8,40 +8,187 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +import json import os +from importlib.resources import files import httpx +from litellm import verbose_logger +from litellm.constants import ( + MODEL_COST_MAP_MAX_SHRINK_RATIO, + MODEL_COST_MAP_MIN_MODEL_COUNT, +) + + +class GetModelCostMap: + """ + Handles fetching, validating, and loading the model cost map. + + Only the backup model *count* is cached (a single int). The full + backup dict is never held in memory — it is only parsed when it + needs to be *returned* as a fallback. + """ + + _backup_model_count: int = -1 # -1 = not yet loaded + + @staticmethod + def load_local_model_cost_map() -> dict: + """Load the local backup model cost map bundled with the package.""" + content = json.loads( + files("litellm") + .joinpath("model_prices_and_context_window_backup.json") + .read_text(encoding="utf-8") + ) + return content + + @classmethod + def _get_backup_model_count(cls) -> int: + """Return the number of models in the local backup (cached int).""" + if cls._backup_model_count < 0: + backup = cls.load_local_model_cost_map() + cls._backup_model_count = len(backup) + return cls._backup_model_count + + @staticmethod + def _check_is_valid_dict(fetched_map: dict) -> bool: + """Check 1: fetched map is a non-empty dict.""" + if not isinstance(fetched_map, dict): + verbose_logger.warning( + "LiteLLM: Fetched model cost map is not a dict (type=%s). " + "Falling back to local backup.", + type(fetched_map).__name__, + ) + return False + + if len(fetched_map) == 0: + verbose_logger.warning( + "LiteLLM: Fetched model cost map is empty. " + "Falling back to local backup.", + ) + return False + + return True + + @classmethod + def _check_model_count_not_reduced( + cls, + fetched_map: dict, + backup_model_count: int, + min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, + max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, + ) -> bool: + """Check 2: model count has not reduced significantly vs backup.""" + fetched_count = len(fetched_map) + + if fetched_count < min_model_count: + verbose_logger.warning( + "LiteLLM: Fetched model cost map has only %d models (minimum=%d). " + "This may indicate a corrupted upstream file. " + "Falling back to local backup.", + fetched_count, + min_model_count, + ) + return False + + if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio: + verbose_logger.warning( + "LiteLLM: Fetched model cost map shrank significantly " + "(fetched=%d, backup=%d, threshold=%.0f%%). " + "This may indicate a corrupted upstream file. " + "Falling back to local backup.", + fetched_count, + backup_model_count, + max_shrink_ratio * 100, + ) + return False + + return True + + @classmethod + def validate_model_cost_map( + cls, + fetched_map: dict, + backup_model_count: int, + min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, + max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, + ) -> bool: + """ + Validate the integrity of a fetched model cost map. + + Runs each check in order and returns False on the first failure. + + Checks: + 1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict. + 2. ``_check_model_count_not_reduced`` -- model count meets minimum + and has not shrunk >``max_shrink_ratio`` vs backup. + + Returns True if all checks pass, False otherwise. + """ + if not cls._check_is_valid_dict(fetched_map): + return False + + if not cls._check_model_count_not_reduced( + fetched_map=fetched_map, + backup_model_count=backup_model_count, + min_model_count=min_model_count, + max_shrink_ratio=max_shrink_ratio, + ): + return False + + return True + + @staticmethod + def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict: + """ + Fetch the model cost map from a remote URL. + + Returns the parsed JSON dict. Raises on network/parse errors + (caller is expected to handle). + """ + response = httpx.get(url, timeout=timeout) + response.raise_for_status() + return response.json() + def get_model_cost_map(url: str) -> dict: - if ( - os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) - or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True" - ): - from importlib.resources import files - import json + """ + Public entry point — returns the model cost map dict. - content = json.loads( - files("litellm") - .joinpath("model_prices_and_context_window_backup.json") - .read_text(encoding="utf-8") - ) - return content + 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. + 2. Otherwise fetches from ``url``, validates integrity, and falls back + to the local backup on any failure. + + Only the backup model count is cached (a single int) for validation. + The full backup dict is only parsed when it must be *returned* as a + fallback — it is never held in memory long-term. + """ + # Note: can't use get_secret_bool here — this runs during litellm.__init__ + # before litellm._key_management_settings is set. + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + return GetModelCostMap.load_local_model_cost_map() try: - response = httpx.get( - url, timeout=5 - ) # set a 5 second timeout for the get request - response.raise_for_status() # Raise an exception if the request is unsuccessful - content = response.json() - return content - except Exception: - from importlib.resources import files - import json - - content = json.loads( - files("litellm") - .joinpath("model_prices_and_context_window_backup.json") - .read_text(encoding="utf-8") + content = GetModelCostMap.fetch_remote_model_cost_map(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: Failed to fetch remote model cost map from %s: %s. " + "Falling back to local backup.", + url, + str(e), ) - return content + return GetModelCostMap.load_local_model_cost_map() + + # Validate using cached count (cheap int comparison, no file I/O) + if not GetModelCostMap.validate_model_cost_map( + fetched_map=content, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. " + "Using local backup instead. url=%s", + url, + ) + return GetModelCostMap.load_local_model_cost_map() + + return content diff --git a/tests/llm_translation/test_model_cost_map_resilience.py b/tests/llm_translation/test_model_cost_map_resilience.py new file mode 100644 index 00000000000..61e375eabeb --- /dev/null +++ b/tests/llm_translation/test_model_cost_map_resilience.py @@ -0,0 +1,291 @@ +""" +Tests for model cost map resilience. + +Simulates: +- A bad (invalid JSON) model cost map upstream +- A bad (empty/missing) backup model cost map +- Verifies litellm.completion() still works even with a broken cost map +- Verifies litellm.get_model_info() raises the expected error for unmapped models +- Verifies the integrity validation helper catches corrupted maps +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +) + +import litellm +from litellm.litellm_core_utils.get_model_cost_map import ( + GetModelCostMap, + get_model_cost_map, +) + + +class TestCheckIsValidDict: + """Unit tests for _check_is_valid_dict.""" + + def test_should_reject_non_dict(self): + """Non-dict should fail.""" + assert GetModelCostMap._check_is_valid_dict("not a dict") is False + + def test_should_reject_empty_dict(self): + """Empty dict should fail.""" + assert GetModelCostMap._check_is_valid_dict({}) is False + + def test_should_reject_list(self): + """List should fail.""" + assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False + + def test_should_reject_none(self): + """None should fail.""" + assert GetModelCostMap._check_is_valid_dict(None) is False + + def test_should_accept_non_empty_dict(self): + """Non-empty dict should pass.""" + assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True + + +class TestCheckModelCountNotReduced: + """Unit tests for _check_model_count_not_reduced.""" + + def test_should_reject_too_few_models(self): + """Fetched map with fewer models than min_model_count should fail.""" + small_map = {f"model-{i}": {} for i in range(5)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=small_map, backup_model_count=0, min_model_count=10 + ) + is False + ) + + def test_should_reject_significant_shrinkage(self): + """Fetched map that shrunk >50% vs backup should fail.""" + fetched = {f"model-{i}": {} for i in range(40)} # 40% of 100 + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is False + ) + + def test_should_accept_when_above_threshold(self): + """Fetched map at 60% of backup (above 50% threshold) should pass.""" + fetched = {f"model-{i}": {} for i in range(60)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_growth(self): + """Fetched map larger than backup should pass.""" + fetched = {f"model-{i}": {} for i in range(120)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_with_empty_backup(self): + """When backup is empty, only min_model_count matters.""" + fetched = {f"model-{i}": {} for i in range(15)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=0, min_model_count=10 + ) + is True + ) + + +class TestValidateModelCostMap: + """Unit tests for validate_model_cost_map (combines both checks).""" + + def test_should_reject_non_dict(self): + """Non-dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_model_count=0) is False + + def test_should_reject_empty_map(self): + """Empty dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_model_count=0) is False + + def test_should_reject_significant_shrinkage(self): + """Should fail at check 2 (shrinkage).""" + fetched = {f"model-{i}": {} for i in range(40)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is False + ) + + def test_should_accept_valid_map(self): + """Should pass both checks.""" + fetched = {f"model-{i}": {} for i in range(120)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_equal_size_map(self): + """Equal size should pass both checks.""" + fetched = {f"model-{i}": {} for i in range(100)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + +class TestGetModelCostMapFallback: + """Tests for get_model_cost_map fallback behavior with bad upstream.""" + + def test_should_fallback_to_backup_on_invalid_json(self): + """When upstream returns invalid JSON, should fall back to local backup.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Should have fallen back to backup — backup always has models + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_to_backup_on_network_error(self): + """When upstream is unreachable, should fall back to local backup.""" + with patch("httpx.get", side_effect=Exception("Connection refused")): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_when_fetched_map_is_empty(self): + """When upstream returns valid JSON but empty dict, should fall back.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = {} # empty map + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Should have fallen back to backup since empty map fails validation + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_when_fetched_map_shrinks_dramatically(self): + """When upstream returns far fewer models than backup, should fall back.""" + tiny_map = {f"model-{i}": {"litellm_provider": "test"} for i in range(11)} + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = tiny_map + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Backup has thousands of models; 11 is a massive shrinkage → fallback + assert len(result) > 11 + + def test_should_use_local_map_when_env_var_set(self): + """LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely.""" + with patch.dict(os.environ, {"LITELLM_LOCAL_MODEL_COST_MAP": "True"}): + with patch("httpx.get") as mock_get: + result = get_model_cost_map( + "https://fake-url.com/model_prices.json" + ) + mock_get.assert_not_called() + + assert isinstance(result, dict) + assert len(result) > 0 + + +class TestBackupModelCostMapExists: + """Validates the local backup file is always present and valid.""" + + def test_should_have_backup_file(self): + """The backup model cost map must exist and be loadable.""" + backup = GetModelCostMap.load_local_model_cost_map() + assert isinstance(backup, dict) + assert len(backup) > 0, "Backup model cost map is empty" + + def test_should_have_minimum_models_in_backup(self): + """The backup must contain a reasonable number of models.""" + backup = GetModelCostMap.load_local_model_cost_map() + assert len(backup) > 100, ( + f"Backup has only {len(backup)} models, expected > 100" + ) + + +class TestBadHostedModelCostMap: + """ + Simulates the hosted model cost map being bad (invalid JSON / corrupted). + + When the hosted map is bad, get_model_cost_map() falls back to the local + backup. These tests verify that after fallback: + - get_model_info() still works for models in the backup + - litellm.completion() still works + """ + + def test_should_model_info_pass_after_bad_hosted_map(self): + """ + If the hosted map is bad, get_model_cost_map falls back to the local + backup. get_model_info should still work for models in the backup. + """ + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + + original = litellm.model_cost + litellm.model_cost = fallback_map + try: + # gpt-4o is in every backup — should work fine + info = litellm.get_model_info("gpt-4o") + assert info is not None + assert info["input_cost_per_token"] > 0 + finally: + litellm.model_cost = original + + def test_should_completion_pass_after_bad_hosted_map(self): + """ + If the hosted map is bad, litellm.completion() should still work. + + Uses litellm's built-in mock_response param so the real completion + path is exercised (routing, cost calculator, logging) without + needing API credentials. + """ + # Simulate bad hosted map → fallback to backup + mock_http = MagicMock() + mock_http.raise_for_status = MagicMock() + mock_http.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_http): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + + original = litellm.model_cost + litellm.model_cost = fallback_map + try: + # mock_response goes through the real completion path — + # routing, cost calculator, logging — but skips the HTTP call + response = litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "say hi"}], + mock_response="hello from mock", + ) + assert response is not None + assert response.choices[0].message.content == "hello from mock" + finally: + litellm.model_cost = original From f311fba194fc42eeb9a7aa2738853d3232ec7a46 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 10 Feb 2026 15:24:46 -0800 Subject: [PATCH 114/117] fix --- docs/my-website/blog/model_cost_map_incident/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md index 7f1324e5fab..b9ff20e4128 100644 --- a/docs/my-website/blog/model_cost_map_incident/index.md +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -74,11 +74,11 @@ A contributor PR introduced an extra `{` bracket, producing invalid JSON. The re | # | Action | Status | Code | |---|---|---|---| -| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [PR #20605](https://github.com/BerriAI/litellm/pull/20605) | -| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | -| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | -| 4 | Resilience test suite (bad hosted map, bad backup, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | -| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | +| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) | +| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) | +| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) | +| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) | +| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) | Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely. From 39bf5b780b7b198333eef97d413cf47d2cafb1ee Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Feb 2026 15:29:07 -0800 Subject: [PATCH 115/117] addressing comments --- .../proxy/guardrails/guardrail_endpoints.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index c0c87842427..07702252a10 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry from litellm.types.guardrails import ( + BaseLitellmParams, PII_ENTITY_CATEGORIES_MAP, ApplyGuardrailRequest, ApplyGuardrailResponse, @@ -178,11 +179,16 @@ async def list_guardrails_v2(): unmasked_length=4, number_of_asterisks=4, ) + masked_litellm_params = ( + BaseLitellmParams(**masked_litellm_params_dict) + if masked_litellm_params_dict + else None + ) guardrail_configs.append( GuardrailInfoResponse( guardrail_id=guardrail.get("guardrail_id"), guardrail_name=guardrail.get("guardrail_name"), - litellm_params=masked_litellm_params_dict, + litellm_params=masked_litellm_params, guardrail_info=guardrail.get("guardrail_info"), created_at=guardrail.get("created_at"), updated_at=guardrail.get("updated_at"), @@ -196,19 +202,27 @@ async def list_guardrails_v2(): for guardrail in in_memory_guardrails: # only add guardrails that are not in DB guardrail list already if guardrail.get("guardrail_id") not in seen_guardrail_ids: - in_memory_litellm_params = dict( - guardrail.get("litellm_params") or {} - ) + in_memory_litellm_params_raw = guardrail.get("litellm_params") + in_memory_litellm_params_dict = ( + in_memory_litellm_params_raw.model_dump(exclude_none=True) + if isinstance(in_memory_litellm_params_raw, LitellmParams) + else in_memory_litellm_params_raw + ) or {} masked_in_memory_litellm_params = _get_masked_values( - in_memory_litellm_params, + in_memory_litellm_params_dict, unmasked_length=4, number_of_asterisks=4, ) + masked_in_memory_litellm_params_typed = ( + BaseLitellmParams(**masked_in_memory_litellm_params) + if masked_in_memory_litellm_params + else None + ) guardrail_configs.append( GuardrailInfoResponse( guardrail_id=guardrail.get("guardrail_id"), guardrail_name=guardrail.get("guardrail_name"), - litellm_params=masked_in_memory_litellm_params, + litellm_params=masked_in_memory_litellm_params_typed, guardrail_info=dict(guardrail.get("guardrail_info") or {}), guardrail_definition_location="config", ) @@ -688,11 +702,16 @@ async def get_guardrail_info(guardrail_id: str): unmasked_length=4, number_of_asterisks=4, ) + masked_litellm_params = ( + BaseLitellmParams(**masked_litellm_params_dict) + if masked_litellm_params_dict + else None + ) return GuardrailInfoResponse( guardrail_id=result.get("guardrail_id"), guardrail_name=result.get("guardrail_name"), - litellm_params=masked_litellm_params_dict, + litellm_params=masked_litellm_params, guardrail_info=dict(result.get("guardrail_info") or {}), created_at=result.get("created_at"), updated_at=result.get("updated_at"), From 8507df483c523c837e69b993ed97e6c87f087675 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 11 Feb 2026 00:52:52 +0100 Subject: [PATCH 116/117] fix(router): propagate model-level tags from config to SpendLogs (#20769) --- litellm/router.py | 11 ++++ tests/test_litellm/test_router.py | 91 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 42058c79c17..d9de7e7fc5a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1924,6 +1924,17 @@ class Router: "deployment_model_name": deployment_model_name, } ) + + ## DEPLOYMENT-LEVEL TAGS + deployment_tags = deployment.get("litellm_params", {}).get("tags") + if deployment_tags: + existing_tags = kwargs[metadata_variable_name].get("tags") or [] + merged_tags = list(existing_tags) + for tag in deployment_tags: + if tag not in merged_tags: + merged_tags.append(tag) + kwargs[metadata_variable_name]["tags"] = merged_tags + kwargs["model_info"] = model_info kwargs["timeout"] = self._get_timeout( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 75ec806ee17..9dcb16b545e 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1990,3 +1990,94 @@ async def test_anthropic_messages_call_type_is_cached(): # This assertion will FAIL if anthropic_messages is filtered out assert cached_result is not None, "Model ID should be cached for anthropic_messages call type" assert cached_result["model_id"] == test_model_id, f"Expected {test_model_id}, got {cached_result['model_id']}" + + +def test_update_kwargs_with_deployment_propagates_model_tags(): + """ + Test that deployment-level tags from litellm_params are merged into + kwargs metadata when _update_kwargs_with_deployment is called. + + This ensures model-level tags defined in config.yaml appear in SpendLogs. + See: https://github.com/BerriAI/litellm/issues/XXXX + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "tags": ["openai-account", "production"], + }, + }, + ], + ) + + kwargs: dict = {"metadata": {}} + deployment = router.get_deployment_by_model_group_name( + model_group_name="gpt-4o-mini" + ) + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + + # Deployment tags should be propagated to kwargs metadata + assert "tags" in kwargs["metadata"] + assert "openai-account" in kwargs["metadata"]["tags"] + assert "production" in kwargs["metadata"]["tags"] + + +def test_update_kwargs_with_deployment_merges_tags_without_duplicates(): + """ + Test that when both request-level and deployment-level tags exist, + they are merged without duplicates. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "tags": ["openai-account", "shared-tag"], + }, + }, + ], + ) + + # Simulate request that already has tags (from request body or key/team level) + kwargs: dict = {"metadata": {"tags": ["user-tag", "shared-tag"]}} + deployment = router.get_deployment_by_model_group_name( + model_group_name="gpt-4o-mini" + ) + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + + # Both sources should be merged, no duplicates + assert "user-tag" in kwargs["metadata"]["tags"] + assert "openai-account" in kwargs["metadata"]["tags"] + assert "shared-tag" in kwargs["metadata"]["tags"] + assert kwargs["metadata"]["tags"].count("shared-tag") == 1 + + +def test_update_kwargs_with_deployment_no_tags(): + """ + Test that when deployment has no tags, kwargs metadata is not affected. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + }, + }, + ], + ) + + kwargs: dict = {"metadata": {}} + deployment = router.get_deployment_by_model_group_name( + model_group_name="gpt-4o-mini" + ) + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + + # No tags key should be added if deployment has no tags + assert "tags" not in kwargs["metadata"] From ebce0e5f8c6056d417831e8d60c4c17fba097e6d Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 10 Feb 2026 16:26:30 -0800 Subject: [PATCH 117/117] [Release - 02/10/2026] v1.81.10-nightly --- .circleci/config.yml | 2 - ci_cd/.grype.yaml | 33 ++++++++ ci_cd/security_scans.sh | 14 ++-- docs/my-website/docs/proxy/config_settings.md | 4 + litellm/_lazy_imports_registry.py | 4 - .../adapters/handler.py | 5 +- .../mcp_server/mcp_server_manager.py | 12 +-- .../spend_tracking/spend_tracking_utils.py | 62 ++++++++++---- model_prices_and_context_window.json | 11 +++ pyproject.toml | 4 +- requirements.txt | 2 +- tests/code_coverage_tests/check_licenses.py | 11 ++- tests/code_coverage_tests/liccheck.ini | 2 + .../code_coverage_tests/recursive_detector.py | 1 + .../image_gen_tests/test_image_generation.py | 84 ++++++++++++++++++- tests/local_testing/test_get_model_info.py | 10 ++- .../mcp_server/test_mcp_server_manager.py | 2 +- .../test_budget_endpoints.py | 1 + .../test_spend_management_endpoints.py | 1 + .../test_secret_managers_main.py | 84 ++++++++++++------- tests/test_litellm/test_constants.py | 10 ++- tests/test_litellm/test_utils.py | 1 + tests/test_litellm/test_video_generation.py | 7 +- ui/litellm-dashboard/package.json | 1 + 24 files changed, 293 insertions(+), 75 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39182e4c6f3..34c3f05cd25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3802,7 +3802,6 @@ jobs: - run: name: Get new version command: | - cd litellm-proxy-extras NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV @@ -3827,7 +3826,6 @@ jobs: - run: name: Publish to PyPI command: | - cd litellm-proxy-extras echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc python -m pip install --upgrade pip build twine setuptools wheel rm -rf build dist diff --git a/ci_cd/.grype.yaml b/ci_cd/.grype.yaml index 642e2dd9d03..b9bc9db58f5 100644 --- a/ci_cd/.grype.yaml +++ b/ci_cd/.grype.yaml @@ -1,3 +1,36 @@ ignore: - vulnerability: CVE-2026-22184 reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists + # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable + - vulnerability: CVE-2025-55130 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: CVE-2025-59465 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: CVE-2025-55131 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: CVE-2025-59466 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: CVE-2026-21637 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: CVE-2025-55132 + reason: Node in Wolfi apk; only used for Admin UI build/prisma + - vulnerability: GHSA-hx9q-6w63-j58v + reason: orjson dumps recursion; allowlisted + - vulnerability: GHSA-73rr-hh4g-fpgx + reason: diff npm transitive dep; override in package.json, allowlisted + - vulnerability: CVE-2026-0865 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2025-15282 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2026-0672 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2025-15366 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2025-15367 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2025-11468 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2025-12781 + reason: Python 3.13 in Wolfi base; no fixed apk build yet + - vulnerability: CVE-2026-1299 + reason: Python 3.13 in Wolfi base; no fixed apk build yet diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 3ffa13c444f..2db72ae5c69 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -140,12 +140,14 @@ run_grype_scans() { "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel - "CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI - "CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI - "CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI - "CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI - "CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI - "CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI + "CVE-2025-59465" # Node only used for Admin UI build/prisma + "CVE-2025-55131" # Node only used for Admin UI build/prisma + "CVE-2025-59466" # Node only used for Admin UI build/prisma + "CVE-2025-55130" # Node only used for Admin UI build/prisma + "CVE-2025-59467" # Node only used for Admin UI build/prisma + "CVE-2026-21637" # Node only used for Admin UI build/prisma + "CVE-2025-55132" # Node only used for Admin UI build/prisma + "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted "CVE-2025-15281" # No fix available yet "CVE-2026-0865" # No fix available yet "CVE-2025-15282" # No fix available yet diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index c78a5c9243e..5b6c6669b91 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -548,6 +548,10 @@ router_settings: | DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small" | DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3 | DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10 +| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600 +| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200 +| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10 +| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60 | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 051b957ed1d..ebe9af9d85c 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1039,10 +1039,6 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.perplexity.chat.transformation", "PerplexityChatConfig", ), - "PerplexityResponsesConfig": ( - ".llms.perplexity.responses.transformation", - "PerplexityResponsesConfig", - ), "AzureOpenAIO1Config": ( ".llms.azure.chat.o_series_transformation", "AzureOpenAIO1Config", diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 296ae97aead..c6caaddf98b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -64,7 +64,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: model = completion_kwargs.get("model") if isinstance(model, str) and model and not model.startswith("responses/"): - reasoning_effort = completion_kwargs.get("reasoning_effort") + # Prefix model with "responses/" to route to OpenAI Responses API + completion_kwargs["model"] = f"responses/{model}" + + reasoning_effort = completion_kwargs.get("reasoning_effort") if isinstance(reasoning_effort, str) and reasoning_effort: completion_kwargs["reasoning_effort"] = { "effort": reasoning_effort, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 532aea249bf..24eae430d85 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -341,7 +341,7 @@ class MCPServerManager: verbose_logger.info( f"Loading OpenAPI spec from {spec_path} for server {server_name}" ) - self._register_openapi_tools( + await self._register_openapi_tools( spec_path=spec_path, server=new_server, base_url=server_config.get("url", ""), @@ -353,7 +353,9 @@ class MCPServerManager: self.initialize_tool_name_to_mcp_server_name_mapping() - def _register_openapi_tools(self, spec_path: str, server: MCPServer, base_url: str): + async def _register_openapi_tools( + self, spec_path: str, server: MCPServer, base_url: str + ): """ Register tools from an OpenAPI specification for a given server. @@ -375,15 +377,15 @@ class MCPServerManager: get_base_url as get_openapi_base_url, ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - load_openapi_spec, + load_openapi_spec_async, ) from litellm.proxy._experimental.mcp_server.tool_registry import ( global_mcp_tool_registry, ) try: - # Load OpenAPI spec - spec = load_openapi_spec(spec_path) + # Load OpenAPI spec (async to avoid "called from within a running event loop") + spec = await load_openapi_spec_async(spec_path) # Use base_url from config if provided, otherwise extract from spec if not base_url: diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index cb8b9ec0395..2f392c48e9d 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -641,7 +641,9 @@ def _sanitize_request_body_for_spend_logs_payload( return {k: _sanitize_value(v) for k, v in request_body.items()} -def _convert_to_json_serializable_dict(obj: Any) -> Any: +def _convert_to_json_serializable_dict( + obj: Any, visited: Optional[set] = None, max_depth: int = 20 +) -> Any: """ Convert object to JSON-serializable dict, handling Pydantic models safely. @@ -650,23 +652,55 @@ def _convert_to_json_serializable_dict(obj: Any) -> Any: Args: obj: Object to convert (dict, list, Pydantic model, or primitive) + visited: Set of object IDs to track circular references + max_depth: Maximum recursion depth to prevent infinite recursion Returns: JSON-serializable version of the object """ - if isinstance(obj, BaseModel): - # Use Pydantic's model_dump() instead of pickle - return obj.model_dump() - elif isinstance(obj, dict): - return {k: _convert_to_json_serializable_dict(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [_convert_to_json_serializable_dict(item) for item in obj] - elif hasattr(obj, "__dict__"): - # Handle objects with __dict__ attribute - return _convert_to_json_serializable_dict(obj.__dict__) - else: - # Primitives (str, int, float, bool, None) pass through - return obj + if max_depth <= 0: + # Return a placeholder if max depth is exceeded + return "" + + if visited is None: + visited = set() + + # Get the object's memory address to track visited objects + obj_id = id(obj) + if obj_id in visited: + # Circular reference detected, return placeholder + return "" + + # Only track mutable objects (dict, list, objects with __dict__) + if isinstance(obj, (dict, list)) or hasattr(obj, "__dict__"): + visited.add(obj_id) + + try: + if isinstance(obj, BaseModel): + # Use Pydantic's model_dump() instead of pickle + result = obj.model_dump() + # Recursively process the dumped dict + return _convert_to_json_serializable_dict(result, visited, max_depth - 1) + elif isinstance(obj, dict): + return { + k: _convert_to_json_serializable_dict(v, visited, max_depth - 1) + for k, v in obj.items() + } + elif isinstance(obj, list): + return [ + _convert_to_json_serializable_dict(item, visited, max_depth - 1) + for item in obj + ] + elif hasattr(obj, "__dict__"): + # Handle objects with __dict__ attribute + return _convert_to_json_serializable_dict(obj.__dict__, visited, max_depth - 1) + else: + # Primitives (str, int, float, bool, None) pass through + return obj + finally: + # Remove from visited set when done processing this object + if obj_id in visited: + visited.remove(obj_id) def _get_proxy_server_request_for_spend_logs_payload( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d794aa50d2e..35538ab1003 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6115,6 +6115,17 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/moonshotai.kimi-k2-thinking": { + "input_cost_per_token": 7.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.03e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, "bedrock/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", diff --git a/pyproject.toml b/pyproject.toml index eb68cd2f4a9..acb8bc2ada3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.81.9" +version = "1.81.10" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -175,7 +175,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.81.9" +version = "1.81.10" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index be4c6e2f7e2..bd313b105e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging prometheus_client==0.20.0 # for /metrics endpoint on proxy ddtrace==2.19.0 # for advanced DD tracing / profiling -orjson==3.11.2 # fast /embedding responses +orjson==3.11.7 # fast /embedding responses polars==1.31.0 # for data processing apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.19.0 # admin UI, SSO diff --git a/tests/code_coverage_tests/check_licenses.py b/tests/code_coverage_tests/check_licenses.py index dc7cc7c54ac..f49f6807a02 100644 --- a/tests/code_coverage_tests/check_licenses.py +++ b/tests/code_coverage_tests/check_licenses.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import sys -import pkg_resources + import requests +from packaging.requirements import Requirement from pathlib import Path import json from typing import Dict, List, Optional, Set, Tuple @@ -213,7 +214,7 @@ class LicenseChecker: try: with open(requirements_file) as f: requirements = [ - pkg_resources.Requirement.parse(line) + Requirement(line.strip()) for line in f if line.strip() and not line.startswith("#") ] @@ -225,8 +226,10 @@ class LicenseChecker: for req in requirements: try: - version = next(iter(req.specs))[1] if req.specs else None - except Exception: + version = ( + next(iter(req.specifier)).version if req.specifier else None + ) + except StopIteration: version = None if not self.check_package(req.name, version): diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index dc46f83366a..e6e9d761ad5 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -111,6 +111,7 @@ ddtrace: >=2.19.0 # Unknown license orjson: >=3.10.12 # Unknown license apscheduler: >=3.10.4 # Unknown license fastapi-sso: >=0.16.0 # Unknown license +filelock: >=3.20.0 # Unlicense (public domain) - https://unlicense.org / https://github.com/tox-dev/filelock pyjwt: >=2.9.0 # Unknown license python-multipart: >=0.0.18 # Unknown license pillow: >=11.0.0 # Unknown license @@ -123,6 +124,7 @@ opentelemetry-exporter-otlp: >=1.25.0 # Unknown license sentry_sdk: >=2.21.0 # Unknown license cryptography: >=43.0.1 # Unknown license tzdata: >=2025.1 # Unknown license +urllib3: >=2.0.0 # MIT license - https://github.com/urllib3/urllib3 python-dotenv: >=1.0.0 # Unknown license tiktoken: >=0.8.0 # Unknown license click: >=8.1.7 # Unknown license diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 71e7798b09e..d6bf1941a08 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -42,6 +42,7 @@ IGNORE_FUNCTIONS = [ "_validate_inheritance_chain", # max depth set (default 100) to prevent infinite recursion in policy inheritance validation. "_basic_json_schema_validate", # max depth set. "extract_text_from_a2a_message", # max depth set (default 10) to prevent infinite recursion in A2A message parsing. + "_convert_to_json_serializable_dict", # max depth set (default 20) and circular reference protection to prevent infinite recursion. ] diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 0567f60ecfc..3b4abeeb82f 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -5,7 +5,7 @@ import logging import os import sys import traceback -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch sys.path.insert( @@ -25,7 +25,7 @@ import pytest import litellm import json import tempfile -from base_image_generation_test import BaseImageGenTest +from base_image_generation_test import BaseImageGenTest, TestCustomLogger import logging from litellm._logging import verbose_logger @@ -182,6 +182,86 @@ class TestAimlImageGeneration(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: return {"model": "aiml/flux-pro/v1.1"} + @pytest.mark.asyncio(scope="module") + @pytest.mark.flaky(retries=0) + async def test_basic_image_generation(self): + """Test basic image generation""" + from unittest.mock import AsyncMock, patch + + mock_aiml_response = { + "created": 1703658209, + "data": [{"url": "https://example.com/generated_image.png"}], + } + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_aiml_response + mock_response.text = json.dumps(mock_aiml_response) + mock_response.headers = {} + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_async_post, patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + ) as mock_sync_post: + mock_async_post.return_value = mock_response + mock_sync_post.return_value = mock_response + + try: + litellm._turn_on_debug() + custom_logger = TestCustomLogger() + litellm.logging_callback_manager._reset_all_callbacks() + litellm.callbacks = [custom_logger] + base_image_generation_call_args = self.get_base_image_generation_call_args() + litellm.set_verbose = True + # Pass dummy api_key so validate_environment passes; HTTP is mocked + response = await litellm.aimage_generation( + **base_image_generation_call_args, + prompt="A image of a otter", + api_key="test-key-mocked-no-credits-needed", + ) + print("FAL AI RESPONSE: ", response) + + await asyncio.sleep(1) + + # assert response._hidden_params["response_cost"] is not None + # assert response._hidden_params["response_cost"] > 0 + # print("response_cost", response._hidden_params["response_cost"]) + + logged_standard_logging_payload = custom_logger.standard_logging_payload + print("logged_standard_logging_payload", logged_standard_logging_payload) + assert logged_standard_logging_payload is not None + assert logged_standard_logging_payload["response_cost"] is not None + assert logged_standard_logging_payload["response_cost"] > 0 + import openai + from openai.types.images_response import ImagesResponse + + # print openai version + print("openai version=", openai.__version__) + + response_dict = dict(response) + if "usage" in response_dict: + response_dict["usage"] = dict(response_dict["usage"]) + print("response usage=", response_dict.get("usage")) + + assert response.data is not None # type guard for iteration (base fails here if None) + for d in response.data: + assert isinstance(d, Image) + print("data in response.data", d) + assert d.b64_json is not None or d.url is not None + except litellm.RateLimitError as e: + pass + except litellm.ContentPolicyViolationError: + pass # Azure randomly raises these errors - skip when they occur + except litellm.InternalServerError: + pass + except Exception as e: + if "Your task failed as a result of our safety system." in str(e): + pass + else: + pytest.fail(f"An exception occurred - {str(e)}") + + class TestGoogleImageGen(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: return {"model": "gemini/imagen-4.0-generate-001"} diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index b84fc22af01..d46a087eb73 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -311,7 +311,15 @@ def test_get_model_info_bedrock_models(): for commitment in potential_commitments: k = k.replace(f"{commitment}/", "") base_model = BedrockModelInfo.get_base_model(k) - base_model_info = litellm.model_cost[base_model] + # get_base_model() returns model id without "bedrock/" prefix; cost map keys use "bedrock/" + base_model_key = ( + base_model + if base_model in litellm.model_cost + else f"bedrock/{base_model}" + ) + if base_model_key not in litellm.model_cost: + continue + base_model_info = litellm.model_cost[base_model_key] for base_model_key, base_model_value in base_model_info.items(): if "invoke/" in k: continue diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index abb8dd49159..b4b5811666b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1043,7 +1043,7 @@ class TestMCPServerManager: "litellm.proxy._experimental.mcp_server.tool_registry.global_mcp_tool_registry.register_tool", return_value=None, ): - manager._register_openapi_tools( + await manager._register_openapi_tools( spec_path=str(spec_path), server=server, base_url="https://example.com", diff --git a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py index d8c505223d9..b15b9d622e4 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py @@ -21,6 +21,7 @@ sys.path.insert( def client_and_mocks(monkeypatch): # Setup MagicMock Prisma mock_prisma = MagicMock() + mock_table = MagicMock() mock_table.create = AsyncMock(side_effect=lambda *, data: data) mock_table.update = AsyncMock(side_effect=lambda *, where, data: {**where, **data}) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index dd34cb47da4..08205cd2d9d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -206,6 +206,7 @@ ignored_keys = [ "metadata.additional_usage_values.cache_creation_input_tokens", "metadata.additional_usage_values.cache_read_input_tokens", "metadata.additional_usage_values.inference_geo", + "metadata.additional_usage_values.speed", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", ] diff --git a/tests/test_litellm/secret_managers/test_secret_managers_main.py b/tests/test_litellm/secret_managers/test_secret_managers_main.py index eaef6956cd5..4a6e303586a 100644 --- a/tests/test_litellm/secret_managers/test_secret_managers_main.py +++ b/tests/test_litellm/secret_managers/test_secret_managers_main.py @@ -46,15 +46,24 @@ def mock_env(): yield os.environ -@patch("litellm.secret_managers.main.oidc_cache") -@patch("litellm.secret_managers.main._get_oidc_http_handler") -@patch("httpx.Client") # Prevent any real HTTP connections -def test_oidc_google_success(mock_httpx_client, mock_get_http_handler, mock_oidc_cache): - mock_oidc_cache.get_cache.return_value = None - mock_handler = MockHTTPHandler(timeout=600.0) - mock_get_http_handler.return_value = mock_handler +def test_oidc_google_success(): + """Test Google OIDC token fetch with mocked handler (no real network calls).""" secret_name = "oidc/google/[invalid url, do not cite]" - result = get_secret(secret_name) + mock_handler = MockHTTPHandler(timeout=600.0) + mock_get_http_handler = Mock(return_value=mock_handler) + mock_oidc_cache = Mock() + mock_oidc_cache.get_cache.return_value = None + + with patch("litellm.secret_managers.main.oidc_cache", mock_oidc_cache): + with patch( + "litellm.secret_managers.main._get_oidc_http_handler", + mock_get_http_handler, + ): + with patch( + "litellm.secret_managers.main.HTTPHandler", + side_effect=lambda timeout=None: mock_handler, + ): + result = get_secret(secret_name) assert result == "mocked_token" assert mock_handler.last_params == {"audience": "[invalid url, do not cite]"} @@ -63,32 +72,49 @@ def test_oidc_google_success(mock_httpx_client, mock_get_http_handler, mock_oidc ) -@patch("litellm.secret_managers.main.oidc_cache") -@patch("litellm.secret_managers.main._get_oidc_http_handler") -def test_oidc_google_cached(mock_get_http_handler, mock_oidc_cache): +def test_oidc_google_cached(): + """Test Google OIDC uses cache and does not call HTTP (no real network calls).""" + secret_name = "oidc/google/[invalid url, do not cite]" + mock_get_http_handler = Mock() + mock_oidc_cache = Mock() mock_oidc_cache.get_cache.return_value = "cached_token" - secret_name = "oidc/google/[invalid url, do not cite]" - result = get_secret(secret_name) + with patch("litellm.secret_managers.main.oidc_cache", mock_oidc_cache): + with patch( + "litellm.secret_managers.main._get_oidc_http_handler", + mock_get_http_handler, + ): + with patch( + "litellm.secret_managers.main.HTTPHandler", + Mock(side_effect=AssertionError("HTTPHandler should not be used")), + ): + result = get_secret(secret_name) assert result == "cached_token", f"Expected cached token, got {result}" mock_oidc_cache.get_cache.assert_called_with(key=secret_name) - # Verify HTTP handler was never called since we had a cached token mock_get_http_handler.assert_not_called() -@patch("litellm.secret_managers.main.oidc_cache") -@patch("litellm.secret_managers.main._get_oidc_http_handler") -def test_oidc_google_failure(mock_get_http_handler, mock_oidc_cache): +def test_oidc_google_failure(): + """Test Google OIDC raises when provider returns error (no real network calls).""" + secret_name = "oidc/google/https://example.com/api" mock_handler = MockHTTPHandler(timeout=600.0) mock_handler.status_code = 400 - mock_get_http_handler.return_value = mock_handler + mock_get_http_handler = Mock(return_value=mock_handler) + mock_oidc_cache = Mock() mock_oidc_cache.get_cache.return_value = None - - secret_name = "oidc/google/https://example.com/api" - with pytest.raises(ValueError, match="Google OIDC provider failed"): - get_secret(secret_name) + with patch("litellm.secret_managers.main.oidc_cache", mock_oidc_cache): + with patch( + "litellm.secret_managers.main._get_oidc_http_handler", + mock_get_http_handler, + ): + with patch( + "litellm.secret_managers.main.HTTPHandler", + side_effect=lambda timeout=None: mock_handler, + ): + with pytest.raises(ValueError, match="Google OIDC provider failed"): + get_secret(secret_name) def test_oidc_circleci_success(monkeypatch): @@ -151,20 +177,18 @@ def test_oidc_azure_file_success(mock_env, tmp_path): @patch("litellm.secret_managers.main.get_azure_ad_token_provider") -@patch.dict(os.environ, {}, clear=False) # Ensure AZURE_FEDERATED_TOKEN_FILE is not set -def test_oidc_azure_ad_token_success(mock_get_azure_ad_token_provider): - # Ensure the env var is not set so it falls through to Azure AD token provider - if "AZURE_FEDERATED_TOKEN_FILE" in os.environ: - del os.environ["AZURE_FEDERATED_TOKEN_FILE"] - +def test_oidc_azure_ad_token_success(mock_get_azure_ad_token_provider, monkeypatch): + # Force-unset so we always hit the Azure AD token provider path (CI may set AZURE_FEDERATED_TOKEN_FILE) + monkeypatch.delenv("AZURE_FEDERATED_TOKEN_FILE", raising=False) + # Mock the token provider function that gets returned and called mock_token_provider = Mock(return_value="azure_ad_token") mock_get_azure_ad_token_provider.return_value = mock_token_provider - + # Also mock the Azure Identity SDK to prevent any real Azure calls with patch("azure.identity.get_bearer_token_provider") as mock_bearer: mock_bearer.return_value = mock_token_provider - + secret_name = "oidc/azure/api://azure-audience" result = get_secret(secret_name) diff --git a/tests/test_litellm/test_constants.py b/tests/test_litellm/test_constants.py index 77f2f308f88..23447a02e04 100644 --- a/tests/test_litellm/test_constants.py +++ b/tests/test_litellm/test_constants.py @@ -38,6 +38,11 @@ def test_all_numeric_constants_can_be_overridden(): print("all numeric constants", json.dumps(numeric_constants, indent=4)) + # Constants that use a different env var name than the constant name + constant_to_env_var = { + "MAX_CALLBACKS": "LITELLM_MAX_CALLBACKS", + } + # Verify all numeric constants have environment variable support for name, value in numeric_constants: # Skip constants that are not meant to be overridden (if any) @@ -47,8 +52,11 @@ def test_all_numeric_constants_can_be_overridden(): # Create a test value that's different from the default test_value = value + 1 if isinstance(value, int) else value + 0.1 + # Use the env var name that the constants module actually reads + env_var_name = constant_to_env_var.get(name, name) + # Set the environment variable - with mock.patch.dict(os.environ, {name: str(test_value)}): + with mock.patch.dict(os.environ, {env_var_name: str(test_value)}): print("overriding", name, "with", test_value) importlib.reload(constants) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 352125d16c4..794b3b87187 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -661,6 +661,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_url_context": {"type": "boolean"}, "supports_reasoning": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, + "supports_preset": {"type": "boolean"}, "tool_use_system_prompt_tokens": {"type": "number"}, "tpm": {"type": "number"}, "supported_endpoints": { diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 5446a0a7b3f..c8cc292519b 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -798,7 +798,7 @@ def test_openai_transform_video_content_request_empty_params(): def test_video_content_handler_uses_get_for_openai(): """HTTP handler must use GET (not POST) for OpenAI content download.""" from litellm.types.router import GenericLiteLLMParams - + handler = BaseLLMHTTPHandler() config = OpenAIVideoConfig() @@ -807,7 +807,12 @@ def test_video_content_handler_uses_get_for_openai(): mock_response.content = b"mp4-bytes" mock_client.get.return_value = mock_response + # Patch both where _get_httpx_client is used and where it is defined so the mock + # is used regardless of import order / CI environment with patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client", + return_value=mock_client, + ), patch( "litellm.llms.custom_httpx.llm_http_handler._get_httpx_client", return_value=mock_client, ): diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 74caf14a594..164368eb6ba 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -79,6 +79,7 @@ "vitest": "^3.2.4" }, "overrides": { + "diff": ">=8.0.3", "prismjs": ">=1.30.0", "webpack-dev-server": ">=5.2.1", "mermaid": ">=11.10.0",