From ed5cbdac2f308a46d1cde2c642564e431b30e846 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 5 Dec 2025 10:00:46 +0900 Subject: [PATCH 01/66] feat: add support for using MCPs on /chat/completions --- docs/my-website/docs/mcp.md | 26 ++ litellm/main.py | 236 +++++++++++++++--- .../mcp/litellm_proxy_mcp_handler.py | 140 +++++++++-- tests/mcp_tests/test_mcp_chat_completions.py | 143 +++++++++++ .../mcp/test_litellm_proxy_mcp_handler.py | 144 +++++++++++ 5 files changed, 642 insertions(+), 47 deletions(-) create mode 100644 tests/mcp_tests/test_mcp_chat_completions.py create mode 100644 tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index a9f7e249133..3f25beea969 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -421,6 +421,32 @@ if __name__ == "__main__": +#### Use MCP tools with `/chat/completions` + +LiteLLM Proxy also supports MCP-aware tooling on the classic `/v1/chat/completions` endpoint. Provide the MCP tool definition directly in the `tools` array and LiteLLM will fetch and transform the MCP server's tools into OpenAI-compatible function calls. When `require_approval` is set to `"never"`, the proxy automatically executes the returned tool calls and feeds the results back into the model before returning the assistant response. + +```bash title="Chat Completions with MCP Tools" showLineNumbers +curl --location '/v1/chat/completions' \ +--header 'Content-Type: application/json' \ +--header "Authorization: Bearer $LITELLM_API_KEY" \ +--data '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Summarize the latest open PR."} + ], + "tools": [ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/github", + "server_label": "github_mcp", + "require_approval": "never" + } + ] +}' +``` + +If you omit `require_approval` or set it to any value other than `"never"`, the MCP tool calls are returned to the client so that you can review and execute them manually, matching the upstream OpenAI behavior. + ```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers diff --git a/litellm/main.py b/litellm/main.py index 20b2cbb7db8..56ed35bea5f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -28,6 +28,7 @@ from typing import ( Callable, Coroutine, Dict, + Iterable, List, Literal, Mapping, @@ -69,6 +70,7 @@ from litellm.constants import ( ) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( calculate_request_duration, get_audio_file_for_health_check, @@ -103,6 +105,7 @@ from litellm.llms.vertex_ai.common_utils import ( from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.router import GenericLiteLLMParams +from litellm.types.llms.openai import ToolParam from litellm.types.utils import RawRequestTypedDict, StreamingChoices from litellm.utils import ( CustomStreamWrapper, @@ -299,7 +302,6 @@ MOCK_RESPONSE_TYPE = Union[str, Exception, dict, ModelResponse, ModelResponseStr class LiteLLM: - def __init__( self, *, @@ -918,6 +920,118 @@ def mock_completion( raise Exception("Mock completion response failed - {}".format(e)) +async def _call_acompletion_internal( + **call_args: Any, +) -> Union[ModelResponse, CustomStreamWrapper]: + """Invoke acompletion while skipping MCP interception to avoid recursion.""" + safe_args = dict(call_args) + safe_args["_skip_mcp_handler"] = True + safe_args.pop("acompletion", None) + return await acompletion(**safe_args) + + +async def _handle_chat_completion_with_mcp( + call_args: Dict[str, Any] +) -> Optional[Union[ModelResponse, CustomStreamWrapper]]: + """Handle MCP-enabled tool execution for chat completion requests.""" + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.responses.utils import ResponsesAPIRequestUtils + + tools = call_args.get("tools") + if not tools: + return None + + mcp_tools, other_tools = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools) + if not mcp_tools: + return None + + base_call_args = dict(call_args) + + user_api_key_auth = call_args.get("user_api_key_auth") or ( + (call_args.get("metadata", {}) or {}).get("user_api_key_auth") + ) + ( + deduplicated_mcp_tools, + tool_server_map, + ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=mcp_tools, + ) + + openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( + deduplicated_mcp_tools, + target_format="chat", + ) + + base_call_args["tools"] = openai_tools or None + + should_auto_execute = LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( + mcp_tools_with_litellm_proxy=mcp_tools + ) + + ( + mcp_auth_header, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=base_call_args.get("secret_fields"), + tools=tools, + ) + + if not should_auto_execute: + return await _call_acompletion_internal(**base_call_args) + + mock_tool_calls = base_call_args.pop("mock_tool_calls", None) + + initial_call_args = dict(base_call_args) + initial_call_args["stream"] = False + if mock_tool_calls is not None: + initial_call_args["mock_tool_calls"] = mock_tool_calls + + initial_response = await _call_acompletion_internal(**initial_call_args) + if not isinstance(initial_response, ModelResponse): + return initial_response + + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response( + response=initial_response + ) + + if not tool_calls: + if base_call_args.get("stream"): + retry_args = dict(base_call_args) + retry_args["stream"] = call_args.get("stream") + return await _call_acompletion_internal(**retry_args) + return initial_response + + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, + tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + if not tool_results: + return initial_response + + follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( + original_messages=call_args.get("messages", []), + response=initial_response, + tool_results=tool_results, + ) + + follow_up_call_args = dict(base_call_args) + follow_up_call_args["messages"] = follow_up_messages + follow_up_call_args["stream"] = call_args.get("stream") + + return await _call_acompletion_internal(**follow_up_call_args) + + def responses_api_bridge_check( model: str, custom_llm_provider: str, @@ -1091,6 +1205,66 @@ def completion( # type: ignore # noqa: PLR0915 tools = validate_and_fix_openai_tools(tools=tools) # validate tool_choice tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) + + skip_mcp_handler = kwargs.pop("_skip_mcp_handler", False) + if not skip_mcp_handler: + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools) + + if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( + tools=tools_for_mcp + ): + call_args_for_mcp: Dict[str, Any] = { + "model": model, + "messages": messages, + "functions": functions, + "function_call": function_call, + "timeout": timeout, + "temperature": temperature, + "top_p": top_p, + "n": n, + "stream": stream, + "stream_options": stream_options, + "stop": stop, + "max_tokens": max_tokens, + "max_completion_tokens": max_completion_tokens, + "modalities": modalities, + "prediction": prediction, + "audio": audio, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "user": user, + "response_format": response_format, + "seed": seed, + "tools": tools, + "tool_choice": tool_choice, + "parallel_tool_calls": parallel_tool_calls, + "logprobs": logprobs, + "top_logprobs": top_logprobs, + "deployment_id": deployment_id, + "reasoning_effort": reasoning_effort, + "verbosity": verbosity, + "safety_identifier": safety_identifier, + "service_tier": service_tier, + "base_url": base_url, + "api_version": api_version, + "api_key": api_key, + "model_list": model_list, + "extra_headers": extra_headers, + "thinking": thinking, + "web_search_options": web_search_options, + "shared_session": shared_session, + } + call_args_for_mcp.update(kwargs) + + mcp_result = run_async_function( + _handle_chat_completion_with_mcp, call_args_for_mcp + ) + if mcp_result is not None: + return mcp_result ######### unpacking kwargs ##################### args = locals() api_base = kwargs.get("api_base", None) @@ -1181,7 +1355,6 @@ def completion( # type: ignore # noqa: PLR0915 prompt_id=prompt_id, non_default_params=non_default_params ) ): - ( model, messages, @@ -2102,7 +2275,7 @@ def completion( # type: ignore # noqa: PLR0915 config = litellm.GenAIHubOrchestrationConfig.get_config() for k, v in config.items(): if ( - k not in optional_params + k not in optional_params ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in optional_params[k] = v @@ -2272,7 +2445,6 @@ def completion( # type: ignore # noqa: PLR0915 try: if use_base_llm_http_handler: - response = base_llm_http_handler.completion( model=model, messages=messages, @@ -3385,9 +3557,9 @@ def completion( # type: ignore # noqa: PLR0915 "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None ): - optional_params["aws_region_name"] = ( - aws_bedrock_client.meta.region_name - ) + optional_params[ + "aws_region_name" + ] = aws_bedrock_client.meta.region_name bedrock_route = BedrockModelInfo.get_bedrock_route(model) if bedrock_route == "converse": @@ -3605,7 +3777,6 @@ def completion( # type: ignore # noqa: PLR0915 if api_key is not None and "Authorization" not in headers: headers["Authorization"] = f"Bearer {api_key}" - response = base_llm_http_handler.completion( model=model, stream=stream, @@ -3741,7 +3912,6 @@ def completion( # type: ignore # noqa: PLR0915 ) raise e elif custom_llm_provider == "gradient_ai": - api_base = litellm.api_base or api_base response = base_llm_http_handler.completion( model=model, @@ -4359,7 +4529,7 @@ def embedding( # noqa: PLR0915 litellm_params=litellm_params_dict, ) elif custom_llm_provider == "github_copilot": - api_key = (api_key or litellm.api_key) + api_key = api_key or litellm.api_key response = base_llm_http_handler.embedding( model=model, input=input, @@ -5524,9 +5694,9 @@ def adapter_completion( new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs) response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs) # type: ignore - translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = ( - None - ) + translated_response: Optional[ + Union[BaseModel, AdapterCompletionStreamWrapper] + ] = None if isinstance(response, ModelResponse): translated_response = translation_obj.translate_completion_output_params( response=response @@ -6231,9 +6401,9 @@ def speech( # noqa: PLR0915 ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY ] = query_params - litellm_params_dict[ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY] = ( - voice_id - ) + litellm_params_dict[ + ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY + ] = voice_id if api_base is not None: litellm_params_dict["api_base"] = api_base @@ -6283,16 +6453,16 @@ def speech( # noqa: PLR0915 text_to_speech_provider_config = VertexAITextToSpeechConfig() # Cast to specific Vertex AI config type to access dispatch method - vertex_config = cast( - VertexAITextToSpeechConfig, text_to_speech_provider_config - ) + vertex_config = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) # Store Vertex AI specific params in litellm_params_dict - litellm_params_dict.update({ - "vertex_project": generic_optional_params.vertex_project, - "vertex_location": generic_optional_params.vertex_location, - "vertex_credentials": generic_optional_params.vertex_credentials, - }) + litellm_params_dict.update( + { + "vertex_project": generic_optional_params.vertex_project, + "vertex_location": generic_optional_params.vertex_location, + "vertex_credentials": generic_optional_params.vertex_credentials, + } + ) response = vertex_config.dispatch_text_to_speech( model=model, @@ -6663,9 +6833,9 @@ def stream_chunk_builder( # noqa: PLR0915 ] if len(content_chunks) > 0: - response["choices"][0]["message"]["content"] = ( - processor.get_combined_content(content_chunks) - ) + response["choices"][0]["message"][ + "content" + ] = processor.get_combined_content(content_chunks) thinking_blocks = [ chunk @@ -6676,9 +6846,9 @@ def stream_chunk_builder( # noqa: PLR0915 ] if len(thinking_blocks) > 0: - response["choices"][0]["message"]["thinking_blocks"] = ( - processor.get_combined_thinking_content(thinking_blocks) - ) + response["choices"][0]["message"][ + "thinking_blocks" + ] = processor.get_combined_thinking_content(thinking_blocks) reasoning_chunks = [ chunk @@ -6689,9 +6859,9 @@ def stream_chunk_builder( # noqa: PLR0915 ] if len(reasoning_chunks) > 0: - response["choices"][0]["message"]["reasoning_content"] = ( - processor.get_combined_reasoning_content(reasoning_chunks) - ) + response["choices"][0]["message"][ + "reasoning_content" + ] = processor.get_combined_reasoning_content(reasoning_chunks) annotation_chunks = [ chunk diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index a41d6f4f5ad..4dda665f70c 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,10 +1,21 @@ -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + Literal, +) from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name from litellm.responses.main import aresponses from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ResponsesAPIResponse, ToolParam +from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: from mcp.types import Tool as MCPTool @@ -163,7 +174,7 @@ class LiteLLM_Proxy_MCP_Handler: if len(allowed_mcp_servers) == 1: tool_server_map[tool_name] = allowed_mcp_servers[0] else: - tool_server_map[tool_name], _ = split_server_prefix_from_name( + _, tool_server_map[tool_name] = split_server_prefix_from_name( tool_name ) @@ -274,15 +285,23 @@ class LiteLLM_Proxy_MCP_Handler: return deduplicated_mcp_tools, tool_server_map @staticmethod - def _transform_mcp_tools_to_openai(mcp_tools: List[Any]) -> List[Any]: + def _transform_mcp_tools_to_openai( + mcp_tools: List[Any], + target_format: Literal["responses", "chat"] = "responses", + ) -> List[Any]: """Transform MCP tools to OpenAI-compatible format.""" from litellm.experimental_mcp_client.tools import ( transform_mcp_tool_to_openai_responses_api_tool, + transform_mcp_tool_to_openai_tool, ) - openai_tools = [] + openai_tools: List[Any] = [] for mcp_tool in mcp_tools: - openai_tool = transform_mcp_tool_to_openai_responses_api_tool(mcp_tool) + openai_tool: Any + if target_format == "chat": + openai_tool = transform_mcp_tool_to_openai_tool(mcp_tool) + else: + openai_tool = transform_mcp_tool_to_openai_responses_api_tool(mcp_tool) openai_tools.append(openai_tool) return openai_tools @@ -325,22 +344,59 @@ class LiteLLM_Proxy_MCP_Handler: return tool_calls + @staticmethod + def _extract_tool_calls_from_chat_response(response: ModelResponse) -> List[Any]: + """Extract tool calls from a chat completion response.""" + tool_calls: List[Any] = [] + + try: + for choice in response.choices: + message = getattr(choice, "message", None) + if message is None: + continue + tool_call_entries = getattr(message, "tool_calls", None) + if tool_call_entries: + for tool_call in tool_call_entries: + if hasattr(tool_call, "model_dump"): + tool_calls.append(tool_call.model_dump()) + else: + tool_calls.append(tool_call) + except Exception: + verbose_logger.exception( + "Failed to extract tool calls from chat completion response" + ) + + return tool_calls + @staticmethod def _extract_tool_call_details( tool_call, ) -> Tuple[Optional[str], Optional[str], Optional[str]]: """Extract tool name, arguments, and call_id from a tool call.""" if isinstance(tool_call, dict): - tool_name = tool_call.get("name") - tool_arguments = tool_call.get("arguments") tool_call_id = tool_call.get("call_id") or tool_call.get("id") + + # OpenAI chat completions wrap tool info under a `function` block + function_block = tool_call.get("function") + if isinstance(function_block, dict): + tool_name = function_block.get("name") + tool_arguments = function_block.get("arguments") + else: + tool_name = tool_call.get("name") + tool_arguments = tool_call.get("arguments") else: - tool_name = getattr(tool_call, "name", None) - tool_arguments = getattr(tool_call, "arguments", None) tool_call_id = getattr(tool_call, "call_id", None) or getattr( tool_call, "id", None ) + function_obj = getattr(tool_call, "function", None) + if function_obj is not None: + tool_name = getattr(function_obj, "name", None) + tool_arguments = getattr(function_obj, "arguments", None) + else: + tool_name = getattr(tool_call, "name", None) + tool_arguments = getattr(tool_call, "arguments", None) + return tool_name, tool_arguments, tool_call_id @staticmethod @@ -399,8 +455,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _execute_tool_calls( - tool_server_map: dict[str, str], - tool_calls: List[Any], + tool_server_map: dict[str, str], + tool_calls: List[Any], user_api_key_auth: Any, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -453,7 +509,11 @@ class LiteLLM_Proxy_MCP_Handler: # Format result for inclusion in response result_text = LiteLLM_Proxy_MCP_Handler._parse_mcp_result(result) tool_results.append( - {"tool_call_id": tool_call_id, "result": result_text} + { + "tool_call_id": tool_call_id, + "result": result_text, + "name": tool_name, + } ) except BlockedPiiEntityError as e: @@ -462,7 +522,11 @@ class LiteLLM_Proxy_MCP_Handler: ) error_message = f"Tool call blocked: PII entity '{getattr(e, 'entity_type', 'unknown')}' detected by guardrail '{getattr(e, 'guardrail_name', 'unknown')}'. {str(e)}" tool_results.append( - {"tool_call_id": tool_call_id, "result": error_message} + { + "tool_call_id": tool_call_id, + "result": error_message, + "name": tool_name, + } ) except GuardrailRaisedException as e: verbose_logger.error( @@ -470,7 +534,11 @@ class LiteLLM_Proxy_MCP_Handler: ) error_message = f"Tool call blocked: Guardrail '{getattr(e, 'guardrail_name', 'unknown')}' violation. {str(e)}" tool_results.append( - {"tool_call_id": tool_call_id, "result": error_message} + { + "tool_call_id": tool_call_id, + "result": error_message, + "name": tool_name, + } ) except HTTPException as e: verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}") @@ -484,11 +552,55 @@ class LiteLLM_Proxy_MCP_Handler: { "tool_call_id": tool_call_id, "result": f"Error executing tool: {str(e)}", + "name": tool_name, } ) return tool_results + @staticmethod + def _create_follow_up_messages_for_chat( + original_messages: List[Any], + response: ModelResponse, + tool_results: List[Dict[str, Any]], + ) -> List[Any]: + """Create follow-up chat messages that include tool execution results.""" + from copy import deepcopy + + from litellm.utils import convert_list_message_to_dict + + follow_up_messages: List[Any] = convert_list_message_to_dict( + deepcopy(original_messages) + ) + + if not follow_up_messages: + follow_up_messages = [] + + message_to_append: Optional[dict] = None + try: + first_choice = response.choices[0] + if isinstance(first_choice, Choices) and getattr( + first_choice, "message", None + ): + message_to_append = first_choice.message.model_dump(exclude_none=True) + except Exception: + verbose_logger.exception("Failed to convert assistant message for MCP flow") + + if message_to_append: + follow_up_messages.append(message_to_append) + + for tool_result in tool_results: + follow_up_messages.append( + { + "role": "tool", + "tool_call_id": tool_result.get("tool_call_id"), + "name": tool_result.get("name"), + "content": tool_result.get("result", ""), + } + ) + + return follow_up_messages + @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, diff --git a/tests/mcp_tests/test_mcp_chat_completions.py b/tests/mcp_tests/test_mcp_chat_completions.py new file mode 100644 index 00000000000..ae13b6ca6e0 --- /dev/null +++ b/tests/mcp_tests/test_mcp_chat_completions.py @@ -0,0 +1,143 @@ +import pytest + +import litellm +from litellm.types.utils import ModelResponse + + +@pytest.mark.asyncio +async def test_acompletion_mcp_auto_exec(monkeypatch): + from types import SimpleNamespace + + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.responses.utils import ResponsesAPIRequestUtils + + dummy_tool = SimpleNamespace( + name="local_search", + description="search", + inputSchema={"type": "object", "properties": {}}, + ) + + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + return [dummy_tool], {"local_search": "local"} + + async def fake_execute(**kwargs): + fake_execute.called = True # type: ignore[attr-defined] + tool_calls = kwargs.get("tool_calls") or [] + assert tool_calls, "tool calls should be present during auto execution" + call_entry = tool_calls[0] + call_id = call_entry.get("id") or call_entry.get("call_id") or "call" + return [ + { + "tool_call_id": call_id, + "result": "executed", + "name": call_entry.get("name", "local_search"), + } + ] + + fake_execute.called = False # type: ignore[attr-defined] + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + fake_process, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + fake_execute, + ) + monkeypatch.setattr( + ResponsesAPIRequestUtils, + "extract_mcp_headers_from_request", + staticmethod(lambda secret_fields, tools: (None, None, None, None)), + ) + + response = await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + tools=[ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/local", + "server_label": "local", + "require_approval": "never", + } + ], + mock_response="Final answer", + mock_tool_calls=[ + { + "id": "call-1", + "type": "function", + "function": {"name": "local_search", "arguments": "{}"}, + } + ], + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Final answer" + assert fake_execute.called is True # type: ignore[attr-defined] + + +@pytest.mark.asyncio +async def test_acompletion_mcp_respects_manual_approval(monkeypatch): + from types import SimpleNamespace + + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.responses.utils import ResponsesAPIRequestUtils + + dummy_tool = SimpleNamespace( + name="local_search", + description="search", + inputSchema={"type": "object", "properties": {}}, + ) + + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + return [dummy_tool], {"local_search": "local"} + + async def fake_execute(**kwargs): + pytest.fail("auto execution should not run when approval is required") + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + fake_process, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + fake_execute, + ) + monkeypatch.setattr( + ResponsesAPIRequestUtils, + "extract_mcp_headers_from_request", + staticmethod(lambda secret_fields, tools: (None, None, None, None)), + ) + + response = await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + tools=[ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/local", + "server_label": "local", + "require_approval": "manual", + } + ], + mock_response="Pending tool", + mock_tool_calls=[ + { + "id": "call-2", + "type": "function", + "function": {"name": "local_search", "arguments": "{}"}, + } + ], + ) + + assert isinstance(response, ModelResponse) + tool_calls = response.choices[0].message.tool_calls + assert tool_calls is not None and len(tool_calls) == 1 diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py new file mode 100644 index 00000000000..9d4e0aeded2 --- /dev/null +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -0,0 +1,144 @@ +import pytest + +from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, +) +from litellm.types.utils import ModelResponse + + +def test_deduplicate_mcp_tools_single_allowed_server(): + tools = [{"name": "search"}, {"name": "search"}] # duplicate on purpose + + deduped, server_map = LiteLLM_Proxy_MCP_Handler._deduplicate_mcp_tools( + tools, + ["everything"], + ) + + assert len(deduped) == 1 + assert server_map == {"search": "everything"} + + +@pytest.mark.parametrize( + "tool_name,expected_server", + [ + ("alpha-tool", "alpha"), + ("beta-another_tool", "beta"), + ], +) +def test_deduplicate_mcp_tools_prefixed_names(tool_name, expected_server): + tools = [{"name": tool_name}] + + _, server_map = LiteLLM_Proxy_MCP_Handler._deduplicate_mcp_tools( + tools, + ["alpha", "beta"], + ) + + assert server_map[tool_name] == expected_server + + +def test_extract_tool_calls_from_chat_response_handles_tool_calls(): + response = ModelResponse( + id="resp-1", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-123", + "type": "function", + "function": {"name": "foo", "arguments": "{}"}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + model="gpt", + created=0, + object="chat.completion", + ) + + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response( + response + ) + + assert len(tool_calls) == 1 + assert tool_calls[0]["function"]["name"] == "foo" + + +def test_create_follow_up_messages_for_chat_appends_tool_results(): + original_messages = [{"role": "user", "content": "hi"}] + response = ModelResponse( + id="resp-2", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-abc", + "type": "function", + "function": {"name": "foo", "arguments": "{}"}, + } + ], + }, + } + ], + model="gpt", + created=0, + object="chat.completion", + ) + tool_results = [ + { + "tool_call_id": "call-abc", + "name": "foo", + "result": "done", + } + ] + + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( + original_messages, + response, + tool_results, + ) + + assert follow_up[0]["role"] == "user" + assert follow_up[-1]["role"] == "tool" + assert follow_up[-1]["name"] == "foo" + assert follow_up[-1]["content"] == "done" + + +def test_transform_mcp_tools_to_openai_uses_chat_format(monkeypatch): + captured = {} + + def fake_transform_chat(tool): + captured.setdefault("chat", []).append(tool) + return {"chat": True} + + def fake_transform_responses(tool): + captured.setdefault("responses", []).append(tool) + return {"responses": True} + + monkeypatch.setattr( + "litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_tool", + fake_transform_chat, + ) + monkeypatch.setattr( + "litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_responses_api_tool", + fake_transform_responses, + ) + + chat_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( + ["tool"], target_format="chat" + ) + resp_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(["tool"]) + + assert chat_tools == [{"chat": True}] + assert resp_tools == [{"responses": True}] + assert captured["chat"] == ["tool"] + assert captured["responses"] == ["tool"] From 6393277bf42a0a77cc40b05f7f89e5a56ae43c49 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 11 Dec 2025 07:42:27 +0900 Subject: [PATCH 02/66] fix: separate MCP handling out of main.py --- litellm/main.py | 223 +++++------------- .../responses/mcp/chat_completions_handler.py | 141 +++++++++++ .../mcp/test_chat_completions_handler.py | 156 ++++++++++++ 3 files changed, 351 insertions(+), 169 deletions(-) create mode 100644 litellm/responses/mcp/chat_completions_handler.py create mode 100644 tests/test_litellm/responses/mcp/test_chat_completions_handler.py diff --git a/litellm/main.py b/litellm/main.py index 56ed35bea5f..42840793d3f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -28,7 +28,6 @@ from typing import ( Callable, Coroutine, Dict, - Iterable, List, Literal, Mapping, @@ -105,7 +104,6 @@ from litellm.llms.vertex_ai.common_utils import ( from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.router import GenericLiteLLMParams -from litellm.types.llms.openai import ToolParam from litellm.types.utils import RawRequestTypedDict, StreamingChoices from litellm.utils import ( CustomStreamWrapper, @@ -920,118 +918,6 @@ def mock_completion( raise Exception("Mock completion response failed - {}".format(e)) -async def _call_acompletion_internal( - **call_args: Any, -) -> Union[ModelResponse, CustomStreamWrapper]: - """Invoke acompletion while skipping MCP interception to avoid recursion.""" - safe_args = dict(call_args) - safe_args["_skip_mcp_handler"] = True - safe_args.pop("acompletion", None) - return await acompletion(**safe_args) - - -async def _handle_chat_completion_with_mcp( - call_args: Dict[str, Any] -) -> Optional[Union[ModelResponse, CustomStreamWrapper]]: - """Handle MCP-enabled tool execution for chat completion requests.""" - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - from litellm.responses.utils import ResponsesAPIRequestUtils - - tools = call_args.get("tools") - if not tools: - return None - - mcp_tools, other_tools = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools) - if not mcp_tools: - return None - - base_call_args = dict(call_args) - - user_api_key_auth = call_args.get("user_api_key_auth") or ( - (call_args.get("metadata", {}) or {}).get("user_api_key_auth") - ) - ( - deduplicated_mcp_tools, - tool_server_map, - ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( - user_api_key_auth=user_api_key_auth, - mcp_tools_with_litellm_proxy=mcp_tools, - ) - - openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( - deduplicated_mcp_tools, - target_format="chat", - ) - - base_call_args["tools"] = openai_tools or None - - should_auto_execute = LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( - mcp_tools_with_litellm_proxy=mcp_tools - ) - - ( - mcp_auth_header, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( - secret_fields=base_call_args.get("secret_fields"), - tools=tools, - ) - - if not should_auto_execute: - return await _call_acompletion_internal(**base_call_args) - - mock_tool_calls = base_call_args.pop("mock_tool_calls", None) - - initial_call_args = dict(base_call_args) - initial_call_args["stream"] = False - if mock_tool_calls is not None: - initial_call_args["mock_tool_calls"] = mock_tool_calls - - initial_response = await _call_acompletion_internal(**initial_call_args) - if not isinstance(initial_response, ModelResponse): - return initial_response - - tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response( - response=initial_response - ) - - if not tool_calls: - if base_call_args.get("stream"): - retry_args = dict(base_call_args) - retry_args["stream"] = call_args.get("stream") - return await _call_acompletion_internal(**retry_args) - return initial_response - - tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - - if not tool_results: - return initial_response - - follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( - original_messages=call_args.get("messages", []), - response=initial_response, - tool_results=tool_results, - ) - - follow_up_call_args = dict(base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = call_args.get("stream") - - return await _call_acompletion_internal(**follow_up_call_args) - - def responses_api_bridge_check( model: str, custom_llm_provider: str, @@ -1207,64 +1093,63 @@ def completion( # type: ignore # noqa: PLR0915 tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) skip_mcp_handler = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler: - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, + if not skip_mcp_handler and tools: + from litellm.responses.mcp.chat_completions_handler import ( + handle_chat_completion_with_mcp, ) - tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools) - if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( - tools=tools_for_mcp - ): - call_args_for_mcp: Dict[str, Any] = { - "model": model, - "messages": messages, - "functions": functions, - "function_call": function_call, - "timeout": timeout, - "temperature": temperature, - "top_p": top_p, - "n": n, - "stream": stream, - "stream_options": stream_options, - "stop": stop, - "max_tokens": max_tokens, - "max_completion_tokens": max_completion_tokens, - "modalities": modalities, - "prediction": prediction, - "audio": audio, - "presence_penalty": presence_penalty, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "user": user, - "response_format": response_format, - "seed": seed, - "tools": tools, - "tool_choice": tool_choice, - "parallel_tool_calls": parallel_tool_calls, - "logprobs": logprobs, - "top_logprobs": top_logprobs, - "deployment_id": deployment_id, - "reasoning_effort": reasoning_effort, - "verbosity": verbosity, - "safety_identifier": safety_identifier, - "service_tier": service_tier, - "base_url": base_url, - "api_version": api_version, - "api_key": api_key, - "model_list": model_list, - "extra_headers": extra_headers, - "thinking": thinking, - "web_search_options": web_search_options, - "shared_session": shared_session, - } - call_args_for_mcp.update(kwargs) + call_args_for_mcp: Dict[str, Any] = { + "model": model, + "messages": messages, + "functions": functions, + "function_call": function_call, + "timeout": timeout, + "temperature": temperature, + "top_p": top_p, + "n": n, + "stream": stream, + "stream_options": stream_options, + "stop": stop, + "max_tokens": max_tokens, + "max_completion_tokens": max_completion_tokens, + "modalities": modalities, + "prediction": prediction, + "audio": audio, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "user": user, + "response_format": response_format, + "seed": seed, + "tools": tools, + "tool_choice": tool_choice, + "parallel_tool_calls": parallel_tool_calls, + "logprobs": logprobs, + "top_logprobs": top_logprobs, + "deployment_id": deployment_id, + "reasoning_effort": reasoning_effort, + "verbosity": verbosity, + "safety_identifier": safety_identifier, + "service_tier": service_tier, + "base_url": base_url, + "api_version": api_version, + "api_key": api_key, + "model_list": model_list, + "extra_headers": extra_headers, + "thinking": thinking, + "web_search_options": web_search_options, + "shared_session": shared_session, + } + call_args_for_mcp.update(kwargs) - mcp_result = run_async_function( - _handle_chat_completion_with_mcp, call_args_for_mcp - ) - if mcp_result is not None: - return mcp_result + completion_callable = globals().get("acompletion") + mcp_result = run_async_function( + handle_chat_completion_with_mcp, + call_args_for_mcp, + completion_callable, + ) + if mcp_result is not None: + return mcp_result ######### unpacking kwargs ##################### args = locals() api_base = kwargs.get("api_base", None) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py new file mode 100644 index 00000000000..d91d13efdf4 --- /dev/null +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -0,0 +1,141 @@ +"""Helpers for handling MCP-aware `/chat/completions` requests.""" + +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Iterable, + Optional, + Union, + cast, +) + +from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, +) +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ToolParam +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +CompletionCallable = Callable[..., Awaitable[Union[ModelResponse, CustomStreamWrapper]]] + + +async def _call_acompletion_internal( + completion_callable: CompletionCallable, **call_args: Any +) -> Union[ModelResponse, CustomStreamWrapper]: + """Invoke `acompletion` while skipping MCP interception to avoid recursion.""" + + safe_args = dict(call_args) + safe_args["_skip_mcp_handler"] = True + safe_args.pop("acompletion", None) + return await completion_callable(**safe_args) + + +async def handle_chat_completion_with_mcp( + call_args: Dict[str, Any], + completion_callable: CompletionCallable, +) -> Optional[Union[ModelResponse, CustomStreamWrapper]]: + """Handle MCP-enabled tool execution for chat completion requests.""" + + tools = call_args.get("tools") + if not tools: + return None + + tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools) + + if not LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( + tools=tools_for_mcp + ): + return None + + mcp_tools, _ = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools) + if not mcp_tools: + return None + + base_call_args = dict(call_args) + + user_api_key_auth = call_args.get("user_api_key_auth") or ( + (call_args.get("metadata", {}) or {}).get("user_api_key_auth") + ) + ( + deduplicated_mcp_tools, + tool_server_map, + ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=mcp_tools, + ) + + openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( + deduplicated_mcp_tools, + target_format="chat", + ) + + base_call_args["tools"] = openai_tools or None + + should_auto_execute = LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( + mcp_tools_with_litellm_proxy=mcp_tools + ) + + ( + mcp_auth_header, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=base_call_args.get("secret_fields"), + tools=tools, + ) + + if not should_auto_execute: + return await _call_acompletion_internal(completion_callable, **base_call_args) + + mock_tool_calls = base_call_args.pop("mock_tool_calls", None) + + initial_call_args = dict(base_call_args) + initial_call_args["stream"] = False + if mock_tool_calls is not None: + initial_call_args["mock_tool_calls"] = mock_tool_calls + + initial_response = await _call_acompletion_internal( + completion_callable, **initial_call_args + ) + if not isinstance(initial_response, ModelResponse): + return initial_response + + tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_chat_response( + response=initial_response + ) + + if not tool_calls: + if base_call_args.get("stream"): + retry_args = dict(base_call_args) + retry_args["stream"] = call_args.get("stream") + return await _call_acompletion_internal(completion_callable, **retry_args) + return initial_response + + tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map=tool_server_map, + tool_calls=tool_calls, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + if not tool_results: + return initial_response + + follow_up_messages = LiteLLM_Proxy_MCP_Handler._create_follow_up_messages_for_chat( + original_messages=call_args.get("messages", []), + response=initial_response, + tool_results=tool_results, + ) + + follow_up_call_args = dict(base_call_args) + follow_up_call_args["messages"] = follow_up_messages + follow_up_call_args["stream"] = call_args.get("stream") + + return await _call_acompletion_internal(completion_callable, **follow_up_call_args) diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py new file mode 100644 index 00000000000..1eab01d5b86 --- /dev/null +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -0,0 +1,156 @@ +import pytest +from unittest.mock import AsyncMock + +from litellm.types.utils import ModelResponse + +from litellm.responses.mcp.chat_completions_handler import ( + handle_chat_completion_with_mcp, +) +from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, +) +from litellm.responses.utils import ResponsesAPIRequestUtils + + +@pytest.mark.asyncio +async def test_handle_chat_completion_returns_none_without_tools(): + completion_callable = AsyncMock() + + result = await handle_chat_completion_with_mcp({}, completion_callable) + + assert result is None + completion_callable.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_handle_chat_completion_without_auto_execution_calls_model(monkeypatch): + tools = [{"type": "function", "function": {"name": "tool"}}] + completion_callable = AsyncMock(return_value="ok") + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + staticmethod(lambda tools: True), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_parse_mcp_tools", + staticmethod(lambda tools: (tools, {})), + ) + async def mock_process(**_): + return ([], {}) + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + mock_process, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_transform_mcp_tools_to_openai", + staticmethod(lambda *_, **__: ["openai-tool"]), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_auto_execute_tools", + staticmethod(lambda **_: False), + ) + monkeypatch.setattr( + ResponsesAPIRequestUtils, + "extract_mcp_headers_from_request", + staticmethod(lambda **_: (None, None, None, None)), + ) + + call_args = {"tools": tools, "messages": []} + result = await handle_chat_completion_with_mcp(call_args, completion_callable) + + assert result == "ok" + completion_callable.assert_awaited_once() + kwargs = completion_callable.await_args.kwargs + assert kwargs.get("_skip_mcp_handler") is True + assert kwargs.get("tools") == ["openai-tool"] + + +@pytest.mark.asyncio +async def test_handle_chat_completion_auto_exec_performs_follow_up(monkeypatch): + tools = [{"type": "function", "function": {"name": "tool"}}] + initial_response = ModelResponse( + id="1", + model="test", + choices=[], + created=0, + object="chat.completion", + ) + follow_up_response = ModelResponse( + id="2", + model="test", + choices=[], + created=0, + object="chat.completion", + ) + completion_callable = AsyncMock( + side_effect=[initial_response, follow_up_response] + ) + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + staticmethod(lambda tools: True), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_parse_mcp_tools", + staticmethod(lambda tools: (tools, {"tool": "server"})), + ) + async def mock_process(**_): + return (tools, {"tool": "server"}) + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + mock_process, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_transform_mcp_tools_to_openai", + staticmethod(lambda *_, **__: tools), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_auto_execute_tools", + staticmethod(lambda **_: True), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_extract_tool_calls_from_chat_response", + staticmethod(lambda **_: ["call"]), + ) + async def mock_execute(**_): + return ["result"] + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + mock_execute, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_create_follow_up_messages_for_chat", + staticmethod(lambda **_: ["follow-up"]), + ) + monkeypatch.setattr( + ResponsesAPIRequestUtils, + "extract_mcp_headers_from_request", + staticmethod(lambda **_: (None, None, None, None)), + ) + + call_args = {"tools": tools, "messages": ["msg"], "stream": True} + result = await handle_chat_completion_with_mcp(call_args, completion_callable) + + assert result is follow_up_response + assert completion_callable.await_count == 2 + first_call = completion_callable.await_args_list[0].kwargs + second_call = completion_callable.await_args_list[1].kwargs + assert first_call["stream"] is False + assert second_call["messages"] == ["follow-up"] + assert second_call["stream"] is True From 4efa21ee7def87c4cc19805be943ac7240ad33cf Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 11 Dec 2025 10:38:29 +0900 Subject: [PATCH 03/66] docs: clarify MCP tool support across providers --- docs/my-website/docs/completion/input.md | 7 ++- docs/my-website/docs/mcp.md | 57 +++++++++++++----------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index bdbd0b04929..7df4f77017a 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -174,11 +174,11 @@ def completion( - `seed`: *integer or null (optional)* - This feature is in Beta. If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. -- `tools`: *array (optional)* - A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. +- `tools`: *array (optional)* - A list of tools the model may call. Use this to provide a list of functions the model may generate JSON inputs for. - - `type`: *string* - The type of the tool. Currently, only function is supported. + - `type`: *string* - The type of the tool. You can set this to `"function"` or `"mcp"` (matching the `/responses` schema) to call LiteLLM-registered MCP servers directly from `/chat/completions`. - - `function`: *object* - Required. + - `function`: *object* - Required for function tools. - `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that function. @@ -247,4 +247,3 @@ def completion( - `eos_token`: *string (optional)* - Initial string applied at the end of a sequence - `hf_model_name`: *string (optional)* - [Sagemaker Only] The corresponding huggingface name of the model, used to pull the right chat template for the model. - diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 3f25beea969..f9c9cbb4562 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -421,32 +421,6 @@ if __name__ == "__main__": -#### Use MCP tools with `/chat/completions` - -LiteLLM Proxy also supports MCP-aware tooling on the classic `/v1/chat/completions` endpoint. Provide the MCP tool definition directly in the `tools` array and LiteLLM will fetch and transform the MCP server's tools into OpenAI-compatible function calls. When `require_approval` is set to `"never"`, the proxy automatically executes the returned tool calls and feeds the results back into the model before returning the assistant response. - -```bash title="Chat Completions with MCP Tools" showLineNumbers -curl --location '/v1/chat/completions' \ ---header 'Content-Type: application/json' \ ---header "Authorization: Bearer $LITELLM_API_KEY" \ ---data '{ - "model": "gpt-4o-mini", - "messages": [ - {"role": "user", "content": "Summarize the latest open PR."} - ], - "tools": [ - { - "type": "mcp", - "server_url": "litellm_proxy/mcp/github", - "server_label": "github_mcp", - "require_approval": "never" - } - ] -}' -``` - -If you omit `require_approval` or set it to any value other than `"never"`, the MCP tool calls are returned to the client so that you can review and execute them manually, matching the upstream OpenAI behavior. - ```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers @@ -1163,6 +1137,37 @@ curl --location '/v1/responses' \ }' ``` +## Use MCP tools with `/chat/completions` + +:::tip Works with all providers +This flow is **provider-agnostic**: the same MCP tool definition works for _every_ LLM backend behind LiteLLM (OpenAI, Azure OpenAI, Anthropic, Amazon Bedrock, Vertex, self-hosted deployments, etc.). +::: + +LiteLLM Proxy also supports MCP-aware tooling on the classic `/v1/chat/completions` endpoint. Provide the MCP tool definition directly in the `tools` array and LiteLLM will fetch and transform the MCP server's tools into OpenAI-compatible function calls. When `require_approval` is set to `"never"`, the proxy automatically executes the returned tool calls and feeds the results back into the model before returning the assistant response. + +```bash title="Chat Completions with MCP Tools" showLineNumbers +curl --location '/v1/chat/completions' \ +--header 'Content-Type: application/json' \ +--header "Authorization: Bearer $LITELLM_API_KEY" \ +--data '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Summarize the latest open PR."} + ], + "tools": [ + { + "type": "mcp", + "server_url": "litellm_proxy/mcp/github", + "server_label": "github_mcp", + "require_approval": "never" + } + ] +}' +``` + +If you omit `require_approval` or set it to any value other than `"never"`, the MCP tool calls are returned to the client so that you can review and execute them manually, matching the upstream OpenAI behavior. + + ## LiteLLM Proxy - Walk through MCP Gateway LiteLLM exposes an MCP Gateway for admins to add all their MCP servers to LiteLLM. The key benefits of using LiteLLM Proxy with MCP are: From 7b1cef86a78848c2f3dc75da6db5060218dd5919 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 11 Dec 2025 15:08:17 +0530 Subject: [PATCH 04/66] Add support for target_storage param --- .../proxy/hooks/managed_files.py | 159 ++++++++- .../migration.sql | 4 + .../files/azure_blob_storage_backend.py | 312 ++++++++++++++++++ .../llms/base_llm/files/storage_backend.py | 79 +++++ .../base_llm/files/storage_backend_factory.py | 41 +++ litellm/proxy/_types.py | 2 + .../openai_files_endpoints/common_utils.py | 296 +++++++++++++++++ .../openai_files_endpoints/files_endpoints.py | 89 ++++- .../storage_backend_service.py | 244 ++++++++++++++ schema.prisma | 2 + .../test_files_endpoint.py | 92 ++++++ 11 files changed, 1300 insertions(+), 20 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20251210125210_add_storage_backend_to_managed_files/migration.sql create mode 100644 litellm/llms/base_llm/files/azure_blob_storage_backend.py create mode 100644 litellm/llms/base_llm/files/storage_backend.py create mode 100644 litellm/llms/base_llm/files/storage_backend_factory.py create mode 100644 litellm/proxy/openai_files_endpoints/storage_backend_service.py diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 608bb495885..6620db5ffa2 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -22,7 +22,6 @@ from litellm.proxy._types import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, - convert_b64_uid_to_unified_uid, get_batch_id_from_unified_batch_id, get_model_id_from_unified_batch_id, ) @@ -42,6 +41,10 @@ from litellm.types.utils import ( LLMResponseTypes, SpecialEnums, ) +from litellm.proxy.openai_files_endpoints.common_utils import ( + get_content_type_from_file_object, + normalize_mime_type_for_provider, +) if TYPE_CHECKING: from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -108,6 +111,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if file_object is not None: db_data["file_object"] = file_object.model_dump_json() + # Extract storage metadata from hidden params if present + hidden_params = getattr(file_object, "_hidden_params", {}) or {} + if "storage_backend" in hidden_params: + db_data["storage_backend"] = hidden_params["storage_backend"] + if "storage_url" in hidden_params: + db_data["storage_url"] = hidden_params["storage_url"] + + verbose_logger.debug( + f"Storage metadata: storage_backend={db_data.get('storage_backend')}, " + f"storage_url={db_data.get('storage_url')}" + ) result = await self.prisma_client.db.litellm_managedfiletable.create( data=db_data @@ -268,7 +282,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) return False - async def async_pre_call_hook( + async def async_pre_call_hook( # noqa: PLR0915 self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, @@ -287,15 +301,31 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): await self.check_managed_file_id_access(data, user_api_key_dict) ### HANDLE TRANSFORMATIONS ### - if call_type == CallTypes.completion.value: + # Check both completion and acompletion call types + is_completion_call = ( + call_type == CallTypes.completion.value + or call_type == CallTypes.acompletion.value + ) + + if is_completion_call: messages = data.get("messages") + model = data.get("model", "") if messages: file_ids = self.get_file_ids_from_messages(messages) if file_ids: + # Check if any files are stored in storage backends and need base64 conversion + # This is needed for Vertex AI/Gemini which requires base64 content + is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower()) + if is_vertex_ai: + await self._convert_storage_files_to_base64( + messages=messages, + file_ids=file_ids, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + ) + model_file_id_mapping = await self.get_model_file_id_mapping( file_ids, user_api_key_dict.parent_otel_span ) - data["model_file_id_mapping"] = model_file_id_mapping elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value: # Handle managed files in responses API input @@ -865,3 +895,124 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) else: raise Exception(f"LiteLLM Managed File object with id={file_id} not found") + + async def _convert_storage_files_to_base64( + self, + messages: List[AllMessageValues], + file_ids: List[str], + litellm_parent_otel_span: Optional[Span], + ) -> None: + """ + Convert files stored in storage backends to base64 format for Vertex AI/Gemini. + + This method checks if any managed files are stored in storage backends, + downloads them, and converts them to base64 format in the messages. + """ + # Check each file_id to see if it's stored in a storage backend + for file_id in file_ids: + # Check if this is a base64 encoded unified file ID + decoded_unified_file_id = _is_base64_encoded_unified_file_id(file_id) + + if not decoded_unified_file_id: + continue + + # Check database for storage backend info + # IMPORTANT: The database stores the base64 encoded unified_file_id (not the decoded version) + # So we query with the original file_id (which is base64 encoded) + db_file = await self.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": file_id} + ) + + if not db_file or not db_file.storage_backend or not db_file.storage_url: + continue + + # File is stored in a storage backend, download and convert to base64 + try: + from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend + + storage_backend_name = db_file.storage_backend + storage_url = db_file.storage_url + + # Get storage backend (uses same env vars as callback) + try: + storage_backend = get_storage_backend(storage_backend_name) + except ValueError as e: + verbose_logger.warning( + f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}" + ) + continue + + file_content = await storage_backend.download_file(storage_url) + + # Determine content type from file object + content_type = self._get_content_type_from_file_object(db_file.file_object) + + # Convert to base64 + base64_data = base64.b64encode(file_content).decode("utf-8") + base64_data_uri = f"data:{content_type};base64,{base64_data}" + + # Update messages to use base64 instead of file_id + self._update_messages_with_base64_data(messages, file_id, base64_data_uri, content_type) + except Exception as e: + verbose_logger.exception( + f"Error converting file {file_id} from storage backend to base64: {str(e)}" + ) + # Continue with other files even if one fails + continue + + def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str: + """ + Determine content type from file object. + + Uses the MIME type utility for consistent detection and normalization. + + Args: + file_object: The file object from the database (can be dict, JSON string, or None) + + Returns: + str: MIME type (defaults to "application/octet-stream" if cannot be determined) + """ + # Use utility function for detection + content_type = get_content_type_from_file_object(file_object) + + # Normalize for Gemini/Vertex AI (requires image/jpeg, not image/jpg) + content_type = normalize_mime_type_for_provider(content_type, provider="gemini") + + return content_type + + def _update_messages_with_base64_data( + self, + messages: List[AllMessageValues], + file_id: str, + base64_data_uri: str, + content_type: str, + ) -> None: + """ + Update messages to replace file_id with base64 data URI. + + Args: + messages: List of messages to update + file_id: The file ID to replace + base64_data_uri: The base64 data URI to use as replacement + content_type: The MIME type of the file (e.g., "image/jpeg", "application/pdf") + """ + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content and isinstance(content, list): + for element in content: + if element.get("type") == "file": + file_element = cast(ChatCompletionFileObject, element) + file_element_file = file_element.get("file", {}) + + if file_element_file.get("file_id") == file_id: + # Replace file_id with base64 data + file_element_file["file_data"] = base64_data_uri + # Set format to help Gemini determine mime type + file_element_file["format"] = content_type + # Remove file_id to ensure only file_data is used + file_element_file.pop("file_id", None) + + verbose_logger.debug( + f"Converted file {file_id} from storage backend to base64 with format {content_type}" + ) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210125210_add_storage_backend_to_managed_files/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210125210_add_storage_backend_to_managed_files/migration.sql new file mode 100644 index 00000000000..26f8d31d271 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210125210_add_storage_backend_to_managed_files/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "storage_backend" TEXT; +ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "storage_url" TEXT; + diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py new file mode 100644 index 00000000000..db3aa50d89a --- /dev/null +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -0,0 +1,312 @@ +""" +Azure Blob Storage backend implementation for file storage. + +This module implements the Azure Blob Storage backend for storing files +in Azure Data Lake Storage Gen2. It inherits from AzureBlobStorageLogger +to reuse all authentication and Azure Storage operations. +""" + +import time +from typing import Optional +from urllib.parse import quote + +from litellm._logging import verbose_logger +from litellm._uuid import uuid + +from .storage_backend import BaseFileStorageBackend +from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger + + +class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): + """ + Azure Blob Storage backend implementation. + + Inherits from AzureBlobStorageLogger to reuse: + - Authentication (account key and Azure AD) + - Service client management + - Token management + - All Azure Storage helper methods + + Reads configuration from the same environment variables as AzureBlobStorageLogger. + """ + + def __init__(self, **kwargs): + """ + Initialize Azure Blob Storage backend. + + Inherits all functionality from AzureBlobStorageLogger which handles: + - Reading environment variables + - Authentication (account key and Azure AD) + - Service client management + - Token management + + Environment variables (same as AzureBlobStorageLogger): + - AZURE_STORAGE_ACCOUNT_NAME (required) + - AZURE_STORAGE_FILE_SYSTEM (required) + - AZURE_STORAGE_ACCOUNT_KEY (optional, if using account key auth) + - AZURE_STORAGE_TENANT_ID (optional, if using Azure AD) + - AZURE_STORAGE_CLIENT_ID (optional, if using Azure AD) + - AZURE_STORAGE_CLIENT_SECRET (optional, if using Azure AD) + + Note: We skip periodic_flush since we're not using this as a logger. + """ + # Initialize AzureBlobStorageLogger (handles all auth and config) + AzureBlobStorageLogger.__init__(self, **kwargs) + + # Disable logging functionality - we're only using this for file storage + # The periodic_flush task will be created but will do nothing since we override it + + async def periodic_flush(self): + """ + Override to do nothing - we're not using this as a logger. + This prevents the periodic flush task from doing any work. + """ + # Do nothing - this class is used for file storage, not logging + return + + async def async_log_success_event(self, *args, **kwargs): + """ + Override to do nothing - we're not using this as a logger. + """ + # Do nothing - this class is used for file storage, not logging + pass + + async def async_log_failure_event(self, *args, **kwargs): + """ + Override to do nothing - we're not using this as a logger. + """ + # Do nothing - this class is used for file storage, not logging + pass + + def _generate_file_name( + self, original_filename: str, file_naming_strategy: str + ) -> str: + """Generate file name based on naming strategy.""" + if file_naming_strategy == "original_filename": + # Use original filename, but sanitize it + return quote(original_filename, safe="") + elif file_naming_strategy == "timestamp": + # Use timestamp + extension = original_filename.split(".")[-1] if "." in original_filename else "" + timestamp = int(time.time() * 1000) # milliseconds + return f"{timestamp}.{extension}" if extension else str(timestamp) + else: # default to "uuid" + # Use UUID + extension = original_filename.split(".")[-1] if "." in original_filename else "" + file_uuid = str(uuid.uuid4()) + return f"{file_uuid}.{extension}" if extension else file_uuid + + async def upload_file( + self, + file_content: bytes, + filename: str, + content_type: str, + path_prefix: Optional[str] = None, + file_naming_strategy: str = "uuid", + ) -> str: + """ + Upload a file to Azure Blob Storage. + + Returns the blob URL in format: https://{account}.blob.core.windows.net/{container}/{path} + """ + try: + # Generate file name + file_name = self._generate_file_name(filename, file_naming_strategy) + + # Build full path + if path_prefix: + # Remove leading/trailing slashes and normalize + prefix = path_prefix.strip("/") + full_path = f"{prefix}/{file_name}" + else: + full_path = file_name + + if self.azure_storage_account_key: + # Use Azure SDK with account key (reuse logger's method) + storage_url = await self._upload_file_with_account_key( + file_content=file_content, + full_path=full_path, + ) + else: + # Use REST API with Azure AD token (reuse logger's methods) + storage_url = await self._upload_file_with_azure_ad( + file_content=file_content, + full_path=full_path, + ) + + verbose_logger.debug( + f"Successfully uploaded file to Azure Blob Storage: {storage_url}" + ) + return storage_url + + except Exception as e: + verbose_logger.exception(f"Error uploading file to Azure Blob Storage: {str(e)}") + raise + + async def _upload_file_with_account_key( + self, file_content: bytes, full_path: str + ) -> str: + """Upload file using Azure SDK with account key authentication.""" + # Reuse the logger's service client method + service_client = await self.get_service_client() + file_system_client = service_client.get_file_system_client( + file_system=self.azure_storage_file_system + ) + + # Create filesystem (container) if it doesn't exist + if not await file_system_client.exists(): + await file_system_client.create_file_system() + verbose_logger.debug(f"Created filesystem: {self.azure_storage_file_system}") + + # Extract directory and filename (similar to logger's pattern) + path_parts = full_path.split("/") + if len(path_parts) > 1: + directory_path = "/".join(path_parts[:-1]) + file_name = path_parts[-1] + + # Create directory if needed (like logger does) + directory_client = file_system_client.get_directory_client(directory_path) + if not await directory_client.exists(): + await directory_client.create_directory() + verbose_logger.debug(f"Created directory: {directory_path}") + + # Get file client from directory (same pattern as logger) + file_client = directory_client.get_file_client(file_name) + else: + # No directory, create file directly in root + file_client = file_system_client.get_file_client(full_path) + + # Create, append, and flush (same pattern as logger's upload_to_azure_data_lake_with_azure_account_key) + await file_client.create_file() + await file_client.append_data(data=file_content, offset=0, length=len(file_content)) + await file_client.flush_data(position=len(file_content), offset=0) + + # Return blob URL (not DFS URL) + blob_url = f"https://{self.azure_storage_account_name}.blob.core.windows.net/{self.azure_storage_file_system}/{full_path}" + return blob_url + + async def _upload_file_with_azure_ad( + self, file_content: bytes, full_path: str + ) -> str: + """Upload file using REST API with Azure AD authentication.""" + # Reuse the logger's token management + await self.set_valid_azure_ad_token() + + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, + ) + + async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback + ) + + # Use DFS endpoint for upload + base_url = f"https://{self.azure_storage_account_name}.dfs.core.windows.net/{self.azure_storage_file_system}/{full_path}" + + # Execute 3-step upload process: create, append, flush + # Reuse the logger's helper methods + await self._create_file(async_client, base_url) + # Append data - logger's _append_data expects string, so we create our own for bytes + await self._append_data_bytes(async_client, base_url, file_content) + await self._flush_data(async_client, base_url, len(file_content)) + + # Return blob URL (not DFS URL) + blob_url = f"https://{self.azure_storage_account_name}.blob.core.windows.net/{self.azure_storage_file_system}/{full_path}" + return blob_url + + async def _append_data_bytes( + self, client, base_url: str, file_content: bytes + ): + """Append binary data to file using REST API.""" + from litellm.constants import AZURE_STORAGE_MSFT_VERSION + + headers = { + "x-ms-version": AZURE_STORAGE_MSFT_VERSION, + "Content-Type": "application/octet-stream", + "Authorization": f"Bearer {self.azure_auth_token}", + } + response = await client.patch( + f"{base_url}?action=append&position=0", + headers=headers, + content=file_content, + ) + response.raise_for_status() + + async def download_file(self, storage_url: str) -> bytes: + """ + Download a file from Azure Blob Storage. + + Args: + storage_url: Blob URL in format: https://{account}.blob.core.windows.net/{container}/{path} + + Returns: + bytes: File content + """ + try: + # Parse blob URL to extract path + # URL format: https://{account}.blob.core.windows.net/{container}/{path} + if ".blob.core.windows.net/" not in storage_url: + raise ValueError(f"Invalid Azure Blob Storage URL: {storage_url}") + + # Extract path after container name + container_and_path = storage_url.split(".blob.core.windows.net/", 1)[1] + path_parts = container_and_path.split("/", 1) + if len(path_parts) < 2: + raise ValueError(f"Invalid Azure Blob Storage URL format: {storage_url}") + file_path = path_parts[1] # Path after container name + + if self.azure_storage_account_key: + # Use Azure SDK (reuse logger's service client) + return await self._download_file_with_account_key(file_path) + else: + # Use REST API (reuse logger's token management) + return await self._download_file_with_azure_ad(file_path) + + except Exception as e: + verbose_logger.exception(f"Error downloading file from Azure Blob Storage: {str(e)}") + raise + + async def _download_file_with_account_key(self, file_path: str) -> bytes: + """Download file using Azure SDK with account key.""" + # Reuse the logger's service client method + service_client = await self.get_service_client() + file_system_client = service_client.get_file_system_client( + file_system=self.azure_storage_file_system + ) + # Ensure filesystem exists (should already exist, but check for safety) + if not await file_system_client.exists(): + raise ValueError(f"Filesystem {self.azure_storage_file_system} does not exist") + file_client = file_system_client.get_file_client(file_path) + # Download file + download_response = await file_client.download_file() + file_content = await download_response.readall() + return file_content + + async def _download_file_with_azure_ad(self, file_path: str) -> bytes: + """Download file using REST API with Azure AD token.""" + # Reuse the logger's token management + await self.set_valid_azure_ad_token() + + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, + ) + from litellm.constants import AZURE_STORAGE_MSFT_VERSION + + async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback + ) + + # Use blob endpoint for download (simpler than DFS) + blob_url = f"https://{self.azure_storage_account_name}.blob.core.windows.net/{self.azure_storage_file_system}/{file_path}" + + headers = { + "x-ms-version": AZURE_STORAGE_MSFT_VERSION, + "Authorization": f"Bearer {self.azure_auth_token}", + } + + response = await async_client.get(blob_url, headers=headers) + response.raise_for_status() + return response.content + diff --git a/litellm/llms/base_llm/files/storage_backend.py b/litellm/llms/base_llm/files/storage_backend.py new file mode 100644 index 00000000000..d9570452950 --- /dev/null +++ b/litellm/llms/base_llm/files/storage_backend.py @@ -0,0 +1,79 @@ +""" +Base storage backend interface for file storage backends. + +This module defines the abstract base class that all file storage backends +(e.g., Azure Blob Storage, S3, GCS) must implement. +""" + +from abc import ABC, abstractmethod +from typing import Optional + + +class BaseFileStorageBackend(ABC): + """ + Abstract base class for file storage backends. + + All storage backends (Azure Blob Storage, S3, GCS, etc.) must implement + these methods to provide a consistent interface for file operations. + """ + + @abstractmethod + async def upload_file( + self, + file_content: bytes, + filename: str, + content_type: str, + path_prefix: Optional[str] = None, + file_naming_strategy: str = "uuid", + ) -> str: + """ + Upload a file to the storage backend. + + Args: + file_content: The file content as bytes + filename: Original filename (may be used for naming strategy) + content_type: MIME type of the file + path_prefix: Optional path prefix for organizing files + file_naming_strategy: Strategy for naming files ("uuid", "timestamp", "original_filename") + + Returns: + str: The storage URL where the file can be accessed/downloaded + + Raises: + Exception: If upload fails + """ + pass + + @abstractmethod + async def download_file(self, storage_url: str) -> bytes: + """ + Download a file from the storage backend. + + Args: + storage_url: The storage URL returned from upload_file + + Returns: + bytes: The file content + + Raises: + Exception: If download fails + """ + pass + + async def delete_file(self, storage_url: str) -> None: + """ + Delete a file from the storage backend. + + This is optional and can be overridden by backends that support deletion. + Default implementation does nothing. + + Args: + storage_url: The storage URL of the file to delete + + Raises: + Exception: If deletion fails + """ + # Default implementation: no-op + # Backends can override if they support deletion + pass + diff --git a/litellm/llms/base_llm/files/storage_backend_factory.py b/litellm/llms/base_llm/files/storage_backend_factory.py new file mode 100644 index 00000000000..1685f3fbd26 --- /dev/null +++ b/litellm/llms/base_llm/files/storage_backend_factory.py @@ -0,0 +1,41 @@ +""" +Factory for creating storage backend instances. + +This module provides a factory function to instantiate the correct storage backend +based on the backend type. Backends use the same configuration as their corresponding +callbacks (e.g., azure_storage uses the same env vars as AzureBlobStorageLogger). +""" + +from litellm._logging import verbose_logger + +from .azure_blob_storage_backend import AzureBlobStorageBackend +from .storage_backend import BaseFileStorageBackend + + +def get_storage_backend(backend_type: str) -> BaseFileStorageBackend: + """ + Factory function to create a storage backend instance. + + Backends are configured using the same environment variables as their + corresponding callbacks. For example, "azure_storage" uses the same + env vars as AzureBlobStorageLogger. + + Args: + backend_type: Backend type identifier (e.g., "azure_storage") + + Returns: + BaseFileStorageBackend: Instance of the appropriate storage backend + + Raises: + ValueError: If backend_type is not supported + """ + verbose_logger.debug(f"Creating storage backend: type={backend_type}") + + if backend_type == "azure_storage": + return AzureBlobStorageBackend() + else: + raise ValueError( + f"Unsupported storage backend type: {backend_type}. " + f"Supported types: azure_storage" + ) + diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 083ac07340a..b7f128a2475 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3680,6 +3680,8 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase): flat_model_file_ids: List[str] created_by: Optional[str] updated_by: Optional[str] + storage_backend: Optional[str] = None + storage_url: Optional[str] = None class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index c5b58e06d4b..d51336ef0b3 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1,7 +1,12 @@ import base64 +import mimetypes import re +from dataclasses import dataclass, field from typing import List, Literal, Optional, Union +from fastapi import Request + +from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.utils import SpecialEnums @@ -339,3 +344,294 @@ def handle_model_based_routing( # No model-based routing needed return False, None, None, None + + +# ============================================================================ +# MIME TYPE DETECTION AND NORMALIZATION +# ============================================================================ + + +# Gemini-supported image MIME types +GEMINI_SUPPORTED_IMAGE_TYPES = { + "image/png", + "image/jpeg", + "image/webp", +} + +# Gemini-supported video MIME types +GEMINI_SUPPORTED_VIDEO_TYPES = { + "video/3gpp", + "video/wmv", + "video/webm", + "video/mp4", + "video/mpg", + "video/mpegps", + "video/mpeg", + "video/quicktime", + "video/x-flv", +} + +# Gemini-supported audio MIME types +GEMINI_SUPPORTED_AUDIO_TYPES = { + "audio/webm", + "audio/wav", + "audio/pcm", + "audio/opus", + "audio/mp4", + "audio/mpga", + "audio/mpeg", + "audio/m4a", + "audio/mp3", + "audio/flac", + "audio/aac", +} + +# Gemini-supported document MIME types +GEMINI_SUPPORTED_DOCUMENT_TYPES = { + "text/plain", + "application/pdf", +} + +# Mapping of common file extensions to MIME types +# This extends Python's mimetypes with custom mappings +EXTENSION_TO_MIME_TYPE = { + ".jpg": "image/jpeg", # Normalize jpg to jpeg + ".jpeg": "image/jpeg", + ".png": "image/png", + ".webp": "image/webp", + ".pdf": "application/pdf", + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".m4a": "audio/mp4", +} + + +def detect_content_type_from_filename(filename: str) -> str: + """ + Detect content type from filename using extension. + + Uses Python's mimetypes module with custom overrides for common cases. + Normalizes jpg to jpeg for consistency. + """ + if not filename: + return "application/octet-stream" + + # Try custom mapping first + filename_lower = filename.lower() + for ext, mime_type in EXTENSION_TO_MIME_TYPE.items(): + if filename_lower.endswith(ext): + return mime_type + + # Fall back to Python's mimetypes + mime_type_guess, _ = mimetypes.guess_type(filename) + if mime_type_guess is not None: + return mime_type_guess + + return "application/octet-stream" + + +def normalize_mime_type_for_provider( + mime_type: str, provider: Optional[str] = None +) -> str: + """ + Normalize MIME type for specific provider requirements. + + Currently handles: + - Gemini: Normalizes image/jpg to image/jpeg + + Args: + mime_type: Original MIME type + provider: Provider name (e.g., "gemini", "vertex_ai") + + Returns: + str: Normalized MIME type + """ + normalized = mime_type.lower().strip() + + # Gemini/Vertex AI requires image/jpeg, not image/jpg + if provider and ("gemini" in provider.lower() or "vertex_ai" in provider.lower()): + if normalized == "image/jpg": + normalized = "image/jpeg" + + # General normalization: always normalize jpg to jpeg + if normalized == "image/jpg": + normalized = "image/jpeg" + + return normalized + + +def is_gemini_supported_mime_type(mime_type: str) -> bool: + """ + Check if a MIME type is supported by Gemini multimodal models. + + Supported categories: + - Images: image/png, image/jpeg, image/webp + - Video: 3gpp, wmv, webm, mp4, mpg, mpegps, mpeg, quicktime, x-flv + - Audio: webm, wav, pcm, opus, mp4, mpga, mpeg, m4a, mp3, flac, aac + - Documents: text/plain, application/pdf + + Args: + mime_type: MIME type to check + + Returns: + bool: True if supported, False otherwise + """ + normalized = normalize_mime_type_for_provider(mime_type, provider="gemini") + return normalized in ( + GEMINI_SUPPORTED_IMAGE_TYPES + | GEMINI_SUPPORTED_VIDEO_TYPES + | GEMINI_SUPPORTED_AUDIO_TYPES + | GEMINI_SUPPORTED_DOCUMENT_TYPES + ) + + +def get_content_type_from_file_object(file_object: Optional[dict]) -> str: + """ + Determine content type from file object (from database or API response). + + Extracts filename from file object and uses detect_content_type_from_filename. + Falls back to default if file object is invalid or filename not found. + + Args: + file_object: File object dictionary (can be None) + + Returns: + str: MIME type (defaults to "application/octet-stream" if cannot be determined) + """ + if not file_object: + return "application/octet-stream" + + # Handle JSON string + if isinstance(file_object, str): + import json + try: + file_object = json.loads(file_object) + except json.JSONDecodeError: + return "application/octet-stream" + + if not isinstance(file_object, dict): + return "application/octet-stream" + + # Try to get filename + filename = file_object.get("filename", "") + if filename: + return detect_content_type_from_filename(filename) + + return "application/octet-stream" + + +# ============================================================================ +# REQUEST PARAMETER EXTRACTION +# ============================================================================ + + +@dataclass +class FileCreationParams: + """ + Structured parameters extracted from file creation requests. + + Attributes: + target_storage: Storage backend name (e.g., "azure_storage", "default") + target_model_names: List of model names for managed files + model: Model parameter for multi-account routing + """ + + target_storage: str = "default" + target_model_names: List[str] = field(default_factory=list) + model: Optional[str] = None + + def __post_init__(self): + """Normalize and validate parameters after initialization.""" + if self.target_model_names is None: + self.target_model_names = [] + + # Normalize target_storage + if not self.target_storage: + self.target_storage = "default" + + # Strip whitespace from model names + self.target_model_names = [name.strip() for name in self.target_model_names if name.strip()] + + +async def extract_file_creation_params( + request: Request, + request_body: Optional[dict] = None, + target_model_names_form: Optional[str] = None, + target_storage_form: Optional[str] = None, +) -> FileCreationParams: + """ + Extract file creation parameters from request. + + Args: + request: FastAPI request object + request_body: Optional pre-parsed request body + target_model_names_form: target_model_names from form field (comma-separated string) + target_storage_form: target_storage from form field (defaults to "default") + + Returns: + FileCreationParams: Structured parameters extracted from the request + """ + if request_body is None: + request_body = await _read_request_body(request=request) or {} + + # Extract target_storage (simplified - just use form parameter) + target_storage = _extract_target_storage_simple(target_storage_form) + + # Extract target_model_names (simplified - just use form parameter) + target_model_names = _extract_target_model_names_simple(target_model_names_form) + + # Extract model parameter + model = _extract_model_param(request, request_body) + + return FileCreationParams( + target_storage=target_storage, + target_model_names=target_model_names, + model=model, + ) + + +def _extract_target_storage_simple(target_storage_form: Optional[str] = None) -> str: + """ + Extract target_storage parameter from form field. + + Args: + target_storage_form: target_storage from form field + + Returns: + str: Target storage backend name, or "default" + """ + if target_storage_form: + return target_storage_form.strip() + return "default" + + +def _extract_target_model_names_simple(target_model_names_form: Optional[str] = None) -> List[str]: + """ + Extract target_model_names parameter from form field. + """ + if not target_model_names_form: + return [] + + # Parse comma-separated string into list + if isinstance(target_model_names_form, str): + return [name.strip() for name in target_model_names_form.split(",") if name.strip()] + elif isinstance(target_model_names_form, list): + return [str(name).strip() for name in target_model_names_form if name] + + return [] + + +def _extract_model_param(request: Request, request_body: dict) -> Optional[str]: + """ + Extract model parameter from request. + + Priority: + 1. request_body.model + 2. Query parameter (?model=) + 3. Header (x-litellm-model) + """ + return ( + request_body.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 3f08a4ec366..9738dd57382 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -7,7 +7,7 @@ import asyncio import traceback -from typing import Optional, cast, get_args +from typing import Any, Optional, cast, get_args import httpx from fastapi import ( @@ -46,10 +46,12 @@ from litellm.types.llms.openai import ( from .common_utils import ( _is_base64_encoded_unified_file_id, encode_file_id_with_model, + extract_file_creation_params, get_credentials_for_model, handle_model_based_routing, prepare_data_with_credentials, ) +from .storage_backend_service import StorageBackendFileService router = APIRouter() @@ -135,17 +137,38 @@ async def route_create_file( router_model: Optional[str], custom_llm_provider: str, model: Optional[str] = None, + target_storage: Optional[str] = "default", ) -> OpenAIFileObject: """ Route file creation request to the appropriate provider. Priority: - 1. If model parameter provided -> use model credentials and encode ID - 2. If enable_loadbalancing_on_batch_endpoints -> deprecated loadbalancing - 3. If target_model_names_list -> managed files (requires DB) - 4. Else -> use custom_llm_provider with files_settings + 1. If target_storage is specified and not "default" -> use storage backend + 2. If model parameter provided -> use model credentials and encode ID + 3. If enable_loadbalancing_on_batch_endpoints -> deprecated loadbalancing + 4. If target_model_names_list -> managed files (requires DB) + 5. Else -> use custom_llm_provider with files_settings """ + # Handle custom storage backend + if target_storage and target_storage != "default": + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + + # Extract file data + file_data = extract_file_data(cast(Any, _create_file_request.get("file"))) + + # Use storage backend service to handle upload + file_object = await StorageBackendFileService.upload_file_to_storage_backend( + file_data=file_data, + target_storage=target_storage, + target_model_names=target_model_names_list, + purpose=purpose, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + ) + + return file_object + # NEW: Handle model-based routing (no DB required) if model is not None: # Get credentials from model_list via router @@ -254,6 +277,7 @@ async def create_file( fastapi_response: Response, purpose: str = Form(...), target_model_names: str = Form(default=""), + target_storage: str = Form(default="default"), provider: Optional[str] = None, custom_llm_provider: str = Form(default="openai"), file: UploadFile = File(...), @@ -297,18 +321,18 @@ async def create_file( or "openai" ) - # NEW: Extract model parameter for multi-account routing + # Extract file creation parameters using utility function request_body = await _read_request_body(request=request) or {} - model_param = ( - request_body.get("model") - or request.query_params.get("model") - or request.headers.get("x-litellm-model") + file_params = await extract_file_creation_params( + request=request, + request_body=request_body, + target_model_names_form=target_model_names, + target_storage_form=target_storage, ) - - target_model_names_list = ( - target_model_names.split(",") if target_model_names else [] - ) - target_model_names_list = [model.strip() for model in target_model_names_list] + + target_storage = file_params.target_storage + target_model_names_list = file_params.target_model_names + model_param = file_params.model # Prepare the data for forwarding # Replace with: @@ -368,6 +392,7 @@ async def create_file( router_model=router_model, custom_llm_provider=custom_llm_provider, model=model_param, + target_storage=target_storage, ) if response is None: @@ -447,7 +472,7 @@ async def create_file( dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -async def get_file_content( +async def get_file_content( # noqa: PLR0915 request: Request, fastapi_response: Response, file_id: str, @@ -525,6 +550,38 @@ async def get_file_content( param="None", code=500, ) + + # Check if file is stored in a storage backend (check DB) + if hasattr(managed_files_obj, "prisma_client") and managed_files_obj.prisma_client: + db_file = await managed_files_obj.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": file_id} + ) + if db_file and db_file.storage_backend and db_file.storage_url: + # File is stored in a storage backend, download it + from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend + + storage_backend_name = db_file.storage_backend + storage_url = db_file.storage_url + + try: + # Get storage backend (uses same env vars as callback) + storage_backend = get_storage_backend(storage_backend_name) + file_content = await storage_backend.download_file(storage_url) + + # Return file content + from fastapi.responses import Response as FastAPIResponse + return FastAPIResponse( + content=file_content, + media_type="application/octet-stream", + ) + except ValueError as e: + raise ProxyException( + message=f"Storage backend error: {str(e)}", + type="invalid_request_error", + param="file_id", + code=400, + ) + model = cast(Optional[str], data.get("model")) if model: response = await llm_router.afile_content( diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py new file mode 100644 index 00000000000..727a7876a5c --- /dev/null +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -0,0 +1,244 @@ +""" +Storage backend service for file upload operations. + +This module provides a service class for handling file uploads to custom +storage backends (e.g., Azure Blob Storage) and managing associated metadata. +""" + +import base64 +import time +from typing import Any, List, Mapping, cast + +from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid as uuid_module +from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend +from litellm.llms.base_llm.files.transformation import BaseFileEndpoints +from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.utils import ProxyLogging +from litellm.types.llms.openai import OpenAIFileObject +from litellm.types.utils import SpecialEnums + + +class StorageBackendFileService: + """ + Service for handling file uploads to storage backends. + + This service encapsulates the logic for: + - Uploading files to storage backends + - Creating file objects with storage metadata + - Generating unified file IDs for managed files + - Storing files in the managed files system + """ + + @staticmethod + async def upload_file_to_storage_backend( + file_data: Mapping[str, Any], + target_storage: str, + target_model_names: List[str], + purpose: str, + proxy_logging_obj: ProxyLogging, + user_api_key_dict: UserAPIKeyAuth, + ) -> OpenAIFileObject: + """ + Upload a file to a storage backend and create a file object. + + Args: + file_data: File data dictionary from extract_file_data() + target_storage: Storage backend name (e.g., "azure_storage") + target_model_names: List of model names for managed files + purpose: File purpose (e.g., "user_data", "batch") + proxy_logging_obj: Proxy logging object for accessing hooks + user_api_key_dict: User API key authentication data + + Returns: + OpenAIFileObject: Created file object with storage metadata + + Raises: + ProxyException: If storage backend is invalid or upload fails + """ + # Get storage backend instance + try: + storage_backend = get_storage_backend(target_storage) + except ValueError as e: + raise ProxyException( + message=str(e), + type="invalid_request_error", + param="target_storage", + code=400, + ) + + # Extract file information + file_content = file_data["content"] + filename = file_data.get("filename", "file") + content_type = file_data.get("content_type", "application/octet-stream") + + # Upload to storage backend + storage_url = await storage_backend.upload_file( + file_content=file_content, + filename=filename, + content_type=content_type, + path_prefix="", + file_naming_strategy="uuid", + ) + + verbose_proxy_logger.debug( + f"Storage backend upload complete: backend={target_storage}, url={storage_url}" + ) + + # Create file object with storage metadata + file_object = StorageBackendFileService._create_file_object_with_storage_metadata( + file_content=file_content, + filename=filename, + purpose=purpose, + target_storage=target_storage, + storage_url=storage_url, + ) + + # Store in managed files if target_model_names provided + if target_model_names: + await StorageBackendFileService._store_in_managed_files( + file_object=file_object, + file_data=file_data, + target_model_names=target_model_names, + target_storage=target_storage, + storage_url=storage_url, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + ) + + return file_object + + @staticmethod + def _create_file_object_with_storage_metadata( + file_content: bytes, + filename: str, + purpose: str, + target_storage: str, + storage_url: str, + ) -> OpenAIFileObject: + """ + Create an OpenAIFileObject with storage backend metadata. + + Args: + file_content: File content bytes + filename: Original filename + purpose: File purpose + target_storage: Storage backend name + storage_url: URL where file is stored + + Returns: + OpenAIFileObject: File object with storage metadata in _hidden_params + """ + file_id = f"file-{uuid_module.uuid4().hex[:24]}" + file_object = OpenAIFileObject( + id=file_id, + object="file", + purpose=purpose, + created_at=int(time.time()), + bytes=len(file_content), + filename=filename, + status="uploaded", + ) + + # Store storage metadata in hidden params + if not hasattr(file_object, "_hidden_params") or file_object._hidden_params is None: + file_object._hidden_params = {} + file_object._hidden_params.update({ + "storage_backend": target_storage, + "storage_url": storage_url, + }) + + return file_object + + @staticmethod + def _create_unified_file_id( + file_type: str, + target_model_names: List[str], + file_id: str, + ) -> str: + """ + Create a base64-encoded unified file ID for managed files. + + Args: + file_type: MIME type of the file + target_model_names: List of model names + file_id: Original file ID + + Returns: + str: Base64-encoded unified file ID + """ + unified_file_id_str = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( + file_type, + str(uuid_module.uuid4()), + ",".join(target_model_names), + file_id, + None, + ) + + base64_unified_file_id = ( + base64.urlsafe_b64encode(unified_file_id_str.encode()).decode().rstrip("=") + ) + + return base64_unified_file_id + + @staticmethod + async def _store_in_managed_files( + file_object: OpenAIFileObject, + file_data: Mapping[str, Any], + target_model_names: List[str], + target_storage: str, + storage_url: str, + proxy_logging_obj: ProxyLogging, + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + """ + Store file in managed files system with unified file ID. + + Args: + file_object: File object to store + file_data: File data dictionary + target_model_names: List of model names + target_storage: Storage backend name + storage_url: URL where file is stored + proxy_logging_obj: Proxy logging object + user_api_key_dict: User API key authentication data + """ + managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files") + if not managed_files_obj or not isinstance(managed_files_obj, BaseFileEndpoints): + verbose_proxy_logger.warning( + "Managed files hook not available, skipping managed files storage" + ) + return + managed_files_obj = cast(Any, managed_files_obj) + + # Create model mappings using storage URL + model_mappings = { + model_name: storage_url + for model_name in target_model_names + } + + # Create unified file ID + file_type = file_data.get("content_type", "application/octet-stream") + base64_unified_file_id = StorageBackendFileService._create_unified_file_id( + file_type=file_type, + target_model_names=target_model_names, + file_id=file_object.id, + ) + + # Update file object ID to unified ID + file_object.id = base64_unified_file_id + + verbose_proxy_logger.debug( + f"Storing file in managed files: unified_id={base64_unified_file_id}, " + f"storage_backend={target_storage}, storage_url={storage_url}" + ) + + # Store in managed files + await managed_files_obj.store_unified_file_id( + file_id=base64_unified_file_id, + file_object=file_object, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_mappings=model_mappings, + user_api_key_dict=user_api_key_dict, + ) + diff --git a/schema.prisma b/schema.prisma index e227c41f93a..1aecd4c9149 100644 --- a/schema.prisma +++ b/schema.prisma @@ -573,6 +573,8 @@ model LiteLLM_ManagedFileTable { file_object Json? // Stores the OpenAIFileObject model_mappings Json flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id + storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default") + storage_url String? // The actual storage URL where the file is stored created_at DateTime @default(now()) created_by String? updated_at DateTime @updatedAt diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 521faae3ca5..754b6941483 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -18,6 +18,7 @@ from litellm.proxy._types import LiteLLM_UserTableFiltered, UserAPIKeyAuth from litellm.proxy.hooks import get_proxy_hook from litellm.proxy.management_endpoints.internal_user_endpoints import ui_view_users from litellm.proxy.proxy_server import app +from litellm.types.llms.openai import OpenAIFileObject client = TestClient(app) from litellm.caching.caching import DualCache @@ -225,6 +226,97 @@ def test_mock_create_audio_file(mocker: MockerFixture, monkeypatch, llm_router: assert openai_call_found, "OpenAI call not found with expected parameters" +def test_target_storage_invokes_storage_backend( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """ + Ensure target_storage is parsed and invokes the storage backend service. + """ + setup_proxy_logging_object(monkeypatch, llm_router) + + async_mock = mocker.AsyncMock( + return_value=OpenAIFileObject( + id="file-test", + object="file", + purpose="user_data", + created_at=0, + bytes=3, + filename="abc.txt", + status="uploaded", + ) + ) + mocker.patch( + "litellm.proxy.openai_files_endpoints.files_endpoints.StorageBackendFileService.upload_file_to_storage_backend", + new=async_mock, + ) + + test_file_content = b"abc" + test_file = ("abc.txt", test_file_content, "text/plain") + + response = client.post( + "/v1/files", + files={"file": test_file}, + data={ + "purpose": "user_data", + "target_storage": "azure_storage", + }, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + async_mock.assert_awaited_once() + called_kwargs = async_mock.call_args.kwargs + assert called_kwargs["target_storage"] == "azure_storage" + assert called_kwargs["target_model_names"] == [] + assert called_kwargs["purpose"] == "user_data" + + +def test_target_storage_with_target_models( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """ + Ensure target_storage and target_model_names are parsed and passed through. + """ + setup_proxy_logging_object(monkeypatch, llm_router) + + async_mock = mocker.AsyncMock( + return_value=OpenAIFileObject( + id="file-test", + object="file", + purpose="user_data", + created_at=0, + bytes=3, + filename="abc.txt", + status="uploaded", + ) + ) + mocker.patch( + "litellm.proxy.openai_files_endpoints.files_endpoints.StorageBackendFileService.upload_file_to_storage_backend", + new=async_mock, + ) + + test_file_content = b"abc" + test_file = ("abc.txt", test_file_content, "text/plain") + + response = client.post( + "/v1/files", + files={"file": test_file}, + data={ + "purpose": "user_data", + "target_storage": "azure_storage", + "target_model_names": "gemini-2.0-flash", + }, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + async_mock.assert_awaited_once() + called_kwargs = async_mock.call_args.kwargs + assert called_kwargs["target_storage"] == "azure_storage" + assert called_kwargs["target_model_names"] == ["gemini-2.0-flash"] + assert called_kwargs["purpose"] == "user_data" + + @pytest.mark.skip(reason="mock respx fails on ci/cd - unclear why") def test_create_file_and_call_chat_completion_e2e( mocker: MockerFixture, monkeypatch, llm_router: Router From 118a06ddfbef1ed9a6b36df588501e917af6d083 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 11 Dec 2025 16:01:52 +0530 Subject: [PATCH 05/66] bump openai package to 2.9.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 604e58132fb..6e4a428435e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.8.0 # openai + http req. httpx==0.28.1 -openai==2.8.0 # openai req. +openai==2.9.0 # openai req. fastapi==0.120.1 # server dep starlette==0.49.1 # starlette fastapi dep backoff==2.2.1 # server dep From 8cccf3e9f04383f1a5c306471ed0550e5a906eb0 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 12 Dec 2025 06:41:58 +0900 Subject: [PATCH 06/66] fix: update MCP handler invocation to pass full call context from main.py --- litellm/main.py | 47 +-------------- .../responses/mcp/chat_completions_handler.py | 60 ++++++++++++++++++- .../mcp/test_chat_completions_handler.py | 21 +++++-- 3 files changed, 77 insertions(+), 51 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 42840793d3f..47e6aae7d8f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1098,54 +1098,11 @@ def completion( # type: ignore # noqa: PLR0915 handle_chat_completion_with_mcp, ) - call_args_for_mcp: Dict[str, Any] = { - "model": model, - "messages": messages, - "functions": functions, - "function_call": function_call, - "timeout": timeout, - "temperature": temperature, - "top_p": top_p, - "n": n, - "stream": stream, - "stream_options": stream_options, - "stop": stop, - "max_tokens": max_tokens, - "max_completion_tokens": max_completion_tokens, - "modalities": modalities, - "prediction": prediction, - "audio": audio, - "presence_penalty": presence_penalty, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "user": user, - "response_format": response_format, - "seed": seed, - "tools": tools, - "tool_choice": tool_choice, - "parallel_tool_calls": parallel_tool_calls, - "logprobs": logprobs, - "top_logprobs": top_logprobs, - "deployment_id": deployment_id, - "reasoning_effort": reasoning_effort, - "verbosity": verbosity, - "safety_identifier": safety_identifier, - "service_tier": service_tier, - "base_url": base_url, - "api_version": api_version, - "api_key": api_key, - "model_list": model_list, - "extra_headers": extra_headers, - "thinking": thinking, - "web_search_options": web_search_options, - "shared_session": shared_session, - } - call_args_for_mcp.update(kwargs) - + mcp_handler_context = locals().copy() completion_callable = globals().get("acompletion") mcp_result = run_async_function( handle_chat_completion_with_mcp, - call_args_for_mcp, + mcp_handler_context, completion_callable, ) if mcp_result is not None: diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index d91d13efdf4..1957e5fa92e 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -21,6 +21,62 @@ from litellm.utils import CustomStreamWrapper CompletionCallable = Callable[..., Awaitable[Union[ModelResponse, CustomStreamWrapper]]] +_CHAT_COMPLETION_CALL_ARG_KEYS = [ + "model", + "messages", + "functions", + "function_call", + "timeout", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "audio", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "response_format", + "seed", + "tools", + "tool_choice", + "parallel_tool_calls", + "logprobs", + "top_logprobs", + "deployment_id", + "reasoning_effort", + "verbosity", + "safety_identifier", + "service_tier", + "base_url", + "api_version", + "api_key", + "model_list", + "extra_headers", + "thinking", + "web_search_options", + "shared_session", +] + + +def _build_call_args_from_context(call_context: Dict[str, Any]) -> Dict[str, Any]: + """Build kwargs for `acompletion` from the `completion` call context.""" + + call_args = { + key: call_context.get(key) + for key in _CHAT_COMPLETION_CALL_ARG_KEYS + if key in call_context + } + additional_kwargs = dict(call_context.get("kwargs") or {}) + call_args.update(additional_kwargs) + return call_args + async def _call_acompletion_internal( completion_callable: CompletionCallable, **call_args: Any @@ -34,11 +90,13 @@ async def _call_acompletion_internal( async def handle_chat_completion_with_mcp( - call_args: Dict[str, Any], + call_context: Dict[str, Any], completion_callable: CompletionCallable, ) -> Optional[Union[ModelResponse, CustomStreamWrapper]]: """Handle MCP-enabled tool execution for chat completion requests.""" + call_args = _build_call_args_from_context(call_context) + tools = call_args.get("tools") if not tools: return None diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 1eab01d5b86..96e7c39aee2 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -55,20 +55,31 @@ async def test_handle_chat_completion_without_auto_execution_calls_model(monkeyp "_should_auto_execute_tools", staticmethod(lambda **_: False), ) + captured_secret_fields = {} + + def mock_extract(**kwargs): + captured_secret_fields["value"] = kwargs.get("secret_fields") + return (None, None, None, None) + monkeypatch.setattr( ResponsesAPIRequestUtils, "extract_mcp_headers_from_request", - staticmethod(lambda **_: (None, None, None, None)), + staticmethod(mock_extract), ) - call_args = {"tools": tools, "messages": []} - result = await handle_chat_completion_with_mcp(call_args, completion_callable) + call_context = { + "tools": tools, + "messages": [], + "kwargs": {"secret_fields": {"api_key": "value"}}, + } + result = await handle_chat_completion_with_mcp(call_context, completion_callable) assert result == "ok" completion_callable.assert_awaited_once() kwargs = completion_callable.await_args.kwargs assert kwargs.get("_skip_mcp_handler") is True assert kwargs.get("tools") == ["openai-tool"] + assert captured_secret_fields["value"] == {"api_key": "value"} @pytest.mark.asyncio @@ -144,8 +155,8 @@ async def test_handle_chat_completion_auto_exec_performs_follow_up(monkeypatch): staticmethod(lambda **_: (None, None, None, None)), ) - call_args = {"tools": tools, "messages": ["msg"], "stream": True} - result = await handle_chat_completion_with_mcp(call_args, completion_callable) + call_context = {"tools": tools, "messages": ["msg"], "stream": True} + result = await handle_chat_completion_with_mcp(call_context, completion_callable) assert result is follow_up_response assert completion_callable.await_count == 2 From 4e20f0793fbcd1de6e40097791a77e9410bd8dd9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 15:37:39 -0800 Subject: [PATCH 07/66] Move Usage into its own folder --- .../src/app/(dashboard)/usage/page.tsx | 2 +- ui/litellm-dashboard/src/app/page.tsx | 2 +- .../src/components/activity_metrics.tsx | 4 +-- .../common_components/chartUtils.tsx | 2 +- ui/litellm-dashboard/src/components/usage.tsx | 2 +- .../EntityUsage}/entity_usage.test.tsx | 2 +- .../components/EntityUsage}/entity_usage.tsx | 14 ++++---- .../EntityUsage}/top_key_view.test.tsx | 0 .../components/EntityUsage}/top_key_view.tsx | 12 +++---- .../EntityUsage}/top_model_view.test.tsx | 0 .../EntityUsage}/top_model_view.tsx | 4 +-- .../{ => usage/components}/new_usage.test.tsx | 4 +-- .../{ => usage/components}/new_usage.tsx | 32 +++++++++---------- 13 files changed, 40 insertions(+), 40 deletions(-) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/entity_usage.test.tsx (99%) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/entity_usage.tsx (98%) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/top_key_view.test.tsx (100%) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/top_key_view.tsx (96%) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/top_model_view.test.tsx (100%) rename ui/litellm-dashboard/src/components/{ => usage/components/EntityUsage}/top_model_view.tsx (95%) rename ui/litellm-dashboard/src/components/{ => usage/components}/new_usage.test.tsx (99%) rename ui/litellm-dashboard/src/components/{ => usage/components}/new_usage.tsx (97%) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx index d77b947df36..bc73a0d7b33 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx @@ -1,6 +1,6 @@ "use client"; -import NewUsagePage from "@/components/new_usage"; +import NewUsagePage from "@/components/Usage/components/new_usage"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useTeams from "@/app/(dashboard)/hooks/useTeams"; diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 20f5480c970..357a7481df8 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -18,7 +18,7 @@ import { MCPServers } from "@/components/mcp_tools"; import ModelHubTable from "@/components/model_hub_table"; import Navbar from "@/components/navbar"; import { getUiConfig, Organization, proxyBaseUrl, setGlobalLitellmHeaderName } from "@/components/networking"; -import NewUsagePage from "@/components/new_usage"; +import NewUsagePage from "@/components/Usage/components/new_usage"; import OldTeams from "@/components/OldTeams"; import { fetchUserModels } from "@/components/organisms/create_key_button"; import Organizations, { fetchOrganizations } from "@/components/organizations"; diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index a791ece9bb7..c336da216bc 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,10 +1,10 @@ import React from "react"; import { Card, Grid, Text, Title } from "@tremor/react"; import { AreaChart, BarChart } from "@tremor/react"; -import { DailyData, ModelActivityData, KeyMetricWithMetadata, TopApiKeyData } from "./usage/types"; +import { DailyData, ModelActivityData, KeyMetricWithMetadata, TopApiKeyData } from "./Usage/types"; import { Collapse } from "antd"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { valueFormatter } from "../components/usage/utils/value_formatters"; +import { valueFormatter } from "./Usage/utils/value_formatters"; import { CustomTooltip, CustomLegend } from "./common_components/chartUtils"; interface ActivityMetricsProps { diff --git a/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx b/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx index b6318ca8bc2..cc91b2a486d 100644 --- a/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx +++ b/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx @@ -1,6 +1,6 @@ import React from "react"; import type { CustomTooltipProps } from "@tremor/react"; -import { SpendMetrics } from "../usage/types"; +import { SpendMetrics } from "../Usage/types"; interface ChartDataPoint { date: string; diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 0900a0a9cc1..6aa4a10b687 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -49,7 +49,7 @@ import { adminGlobalActivityPerModel, getProxyUISettings, } from "./networking"; -import TopKeyView from "./top_key_view"; +import TopKeyView from "./Usage/components/EntityUsage/top_key_view"; import { formatNumberWithCommas } from "@/utils/dataUtils"; console.log("process.env.NODE_ENV", process.env.NODE_ENV); diff --git a/ui/litellm-dashboard/src/components/entity_usage.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/entity_usage.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.test.tsx index d0d2337e185..dba5eda04c6 100644 --- a/ui/litellm-dashboard/src/components/entity_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.test.tsx @@ -1,7 +1,7 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import EntityUsage from "./entity_usage"; -import * as networking from "./networking"; +import * as networking from "../../../networking"; beforeAll(() => { if (typeof window !== "undefined" && !window.ResizeObserver) { diff --git a/ui/litellm-dashboard/src/components/entity_usage.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx similarity index 98% rename from ui/litellm-dashboard/src/components/entity_usage.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx index cced9e8ff98..1d99c99844a 100644 --- a/ui/litellm-dashboard/src/components/entity_usage.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx @@ -21,21 +21,21 @@ import { TabPanels, Subtitle, } from "@tremor/react"; -import { ActivityMetrics, processActivityData } from "./activity_metrics"; -import { DailyData, BreakdownMetrics, KeyMetricWithMetadata, EntityMetricWithMetadata, TagUsage } from "./usage/types"; +import { ActivityMetrics, processActivityData } from "../../../activity_metrics"; +import { DailyData, BreakdownMetrics, KeyMetricWithMetadata, EntityMetricWithMetadata, TagUsage } from "../../types"; import { organizationDailyActivityCall, tagDailyActivityCall, teamDailyActivityCall, customerDailyActivityCall, agentDailyActivityCall, -} from "./networking"; +} from "../../../networking"; import TopKeyView from "./top_key_view"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { valueFormatterSpend } from "./usage/utils/value_formatters"; -import { getProviderLogoAndName } from "./provider_info_helpers"; -import { UsageExportHeader } from "./EntityUsageExport"; -import type { EntityType } from "./EntityUsageExport/types"; +import { valueFormatterSpend } from "../../utils/value_formatters"; +import { getProviderLogoAndName } from "../../../provider_info_helpers"; +import { UsageExportHeader } from "../../../EntityUsageExport"; +import type { EntityType } from "../../../EntityUsageExport/types"; import TopModelView from "./top_model_view"; interface EntityMetrics { diff --git a/ui/litellm-dashboard/src/components/top_key_view.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/top_key_view.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.test.tsx diff --git a/ui/litellm-dashboard/src/components/top_key_view.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.tsx similarity index 96% rename from ui/litellm-dashboard/src/components/top_key_view.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.tsx index 565dc03a38a..43dc3244b4d 100644 --- a/ui/litellm-dashboard/src/components/top_key_view.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.tsx @@ -1,13 +1,13 @@ import React, { useState } from "react"; import { BarChart } from "@tremor/react"; -import KeyInfoView from "./templates/key_info_view"; -import { keyInfoV1Call } from "./networking"; -import { transformKeyInfo } from "../components/key_team_helpers/transform_key_info"; -import { DataTable } from "./view_logs/table"; +import KeyInfoView from "../../../templates/key_info_view"; +import { keyInfoV1Call } from "../../../networking"; +import { transformKeyInfo } from "../../../key_team_helpers/transform_key_info"; +import { DataTable } from "../../../view_logs/table"; import { Tooltip } from "antd"; import { Button } from "@tremor/react"; -import { formatNumberWithCommas } from "../utils/dataUtils"; -import { TagUsage } from "./usage/types"; +import { formatNumberWithCommas } from "../../../../utils/dataUtils"; +import { TagUsage } from "../../types"; import { ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/outline"; interface TopKeyViewProps { diff --git a/ui/litellm-dashboard/src/components/top_model_view.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/top_model_view.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.test.tsx diff --git a/ui/litellm-dashboard/src/components/top_model_view.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.tsx similarity index 95% rename from ui/litellm-dashboard/src/components/top_model_view.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.tsx index f090d2da5cb..15ae660d4be 100644 --- a/ui/litellm-dashboard/src/components/top_model_view.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.tsx @@ -1,7 +1,7 @@ import { BarChart } from "@tremor/react"; -import { formatNumberWithCommas } from "../utils/dataUtils"; +import { formatNumberWithCommas } from "../../../../utils/dataUtils"; import { useState } from "react"; -import { DataTable } from "./view_logs/table"; +import { DataTable } from "../../../view_logs/table"; interface TopModel { key: string; diff --git a/ui/litellm-dashboard/src/components/new_usage.test.tsx b/ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/new_usage.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx index 340201452fb..1358bf23295 100644 --- a/ui/litellm-dashboard/src/components/new_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx @@ -1,8 +1,8 @@ import { render, screen, fireEvent, waitFor, act } from "@testing-library/react"; import { describe, it, expect, vi, beforeEach, beforeAll } from "vitest"; import NewUsagePage from "./new_usage"; -import type { Organization } from "./networking"; -import * as networking from "./networking"; +import type { Organization } from "../../networking"; +import * as networking from "../../networking"; import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/usage/components/new_usage.tsx similarity index 97% rename from ui/litellm-dashboard/src/components/new_usage.tsx rename to ui/litellm-dashboard/src/components/usage/components/new_usage.tsx index fd89ddb869b..d91c0743d8b 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/new_usage.tsx @@ -33,22 +33,22 @@ import React, { useCallback, useEffect, useMemo, useState } from "react"; import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { Button } from "@tremor/react"; -import { all_admin_roles } from "../utils/roles"; -import { ActivityMetrics, processActivityData } from "./activity_metrics"; -import CloudZeroExportModal from "./cloudzero_export_modal"; -import EntityUsage, { EntityList } from "./entity_usage"; -import EntityUsageExportModal from "./EntityUsageExport"; -import { Team } from "./key_team_helpers/key_list"; -import { Organization, tagListCall, userDailyActivityAggregatedCall, userDailyActivityCall } from "./networking"; -import { getProviderLogoAndName } from "./provider_info_helpers"; -import AdvancedDatePicker from "./shared/advanced_date_picker"; -import { ChartLoader } from "./shared/chart_loader"; -import { Tag } from "./tag_management/types"; -import TopKeyView from "./top_key_view"; -import { DailyData, KeyMetricWithMetadata, MetricWithMetadata } from "./usage/types"; -import { valueFormatterSpend } from "./usage/utils/value_formatters"; -import UserAgentActivity from "./user_agent_activity"; -import ViewUserSpend from "./view_user_spend"; +import { all_admin_roles } from "../../../utils/roles"; +import { ActivityMetrics, processActivityData } from "../../activity_metrics"; +import CloudZeroExportModal from "../../cloudzero_export_modal"; +import EntityUsage, { EntityList } from "./EntityUsage/entity_usage"; +import EntityUsageExportModal from "../../EntityUsageExport"; +import { Team } from "../../key_team_helpers/key_list"; +import { Organization, tagListCall, userDailyActivityAggregatedCall, userDailyActivityCall } from "../../networking"; +import { getProviderLogoAndName } from "../../provider_info_helpers"; +import AdvancedDatePicker from "../../shared/advanced_date_picker"; +import { ChartLoader } from "../../shared/chart_loader"; +import { Tag } from "../../tag_management/types"; +import TopKeyView from "./EntityUsage/top_key_view"; +import { DailyData, KeyMetricWithMetadata, MetricWithMetadata } from "../types"; +import { valueFormatterSpend } from "../utils/value_formatters"; +import UserAgentActivity from "../../user_agent_activity"; +import ViewUserSpend from "../../view_user_spend"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; interface NewUsagePageProps { From 424934296f81a25a7dd613036e17486d04edc761 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 15:40:26 -0800 Subject: [PATCH 08/66] adding all files --- ui/litellm-dashboard/tests/top_key_view.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/tests/top_key_view.test.tsx b/ui/litellm-dashboard/tests/top_key_view.test.tsx index a034a20c849..5cc2de57378 100644 --- a/ui/litellm-dashboard/tests/top_key_view.test.tsx +++ b/ui/litellm-dashboard/tests/top_key_view.test.tsx @@ -1,7 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { renderWithProviders, screen, fireEvent } from "./test-utils"; -import TopKeyView from "../src/components/top_key_view"; -import { TagUsage } from "../src/components/usage/types"; +import TopKeyView from "../src/components/Usage/components/EntityUsage/top_key_view"; +import { TagUsage } from "../src/components/Usage/types"; // Mock the networking module vi.mock("../src/components/networking", () => ({ From 7e58931ec1f7df7bd6f4dc9c17fcf076c1acf777 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 11 Dec 2025 15:43:40 -0800 Subject: [PATCH 09/66] Prompt Management - new API for integrating providers (#17829) * Prompt Management API - new API to interact with Prompt Management integrations (no PR required) (#17800) * feat: initial commit adding prompt management api * feat: initial commit adding prompt management api * fix: refactoring to make sure get prompt is async * fix: additional fixes * fix: partially working generic api prompt management --- .../braintrust_prompt_wrapper_README.md | 279 ++++++++++ .../braintrust_prompt_wrapper_server.py | 274 ++++++++++ .../anthropic_cache_control_hook.py | 77 ++- .../bitbucket/bitbucket_prompt_manager.py | 75 ++- litellm/integrations/custom_logger.py | 5 + .../integrations/custom_prompt_management.py | 8 +- .../dotprompt/dotprompt_manager.py | 77 ++- .../generic_prompt_management/__init__.py | 80 +++ .../generic_prompt_manager.py | 501 ++++++++++++++++++ .../gitlab/gitlab_prompt_manager.py | 75 ++- .../langfuse/langfuse_prompt_management.py | 31 +- .../integrations/prompt_management_base.py | 160 +++++- litellm/litellm_core_utils/litellm_logging.py | 22 +- .../index.html} | 0 .../proxy/_experimental/out/guardrails.html | 1 - .../out/{login.html => login/index.html} | 0 .../out/{logs.html => logs/index.html} | 0 .../{model-hub.html => model-hub/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../proxy/_experimental/out/onboarding.html | 1 - .../index.html} | 0 .../index.html} | 0 .../out/{teams.html => teams/index.html} | 0 .../{test-key.html => test-key/index.html} | 0 .../out/{usage.html => usage/index.html} | 0 .../out/{users.html => users/index.html} | 0 .../index.html} | 0 litellm/proxy/_new_secret_config.yaml | 24 +- litellm/proxy/utils.py | 21 +- litellm/types/llms/custom_http.py | 1 + litellm/types/prompts/init_prompts.py | 11 +- 32 files changed, 1656 insertions(+), 67 deletions(-) create mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md create mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py create mode 100644 litellm/integrations/generic_prompt_management/__init__.py create mode 100644 litellm/integrations/generic_prompt_management/generic_prompt_manager.py rename litellm/proxy/_experimental/out/{api-reference.html => api-reference/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/guardrails.html rename litellm/proxy/_experimental/out/{login.html => login/index.html} (100%) rename litellm/proxy/_experimental/out/{logs.html => logs/index.html} (100%) rename litellm/proxy/_experimental/out/{model-hub.html => model-hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) rename litellm/proxy/_experimental/out/{models-and-endpoints.html => models-and-endpoints/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html rename litellm/proxy/_experimental/out/{organizations.html => organizations/index.html} (100%) rename litellm/proxy/_experimental/out/{playground.html => playground/index.html} (100%) rename litellm/proxy/_experimental/out/{teams.html => teams/index.html} (100%) rename litellm/proxy/_experimental/out/{test-key.html => test-key/index.html} (100%) rename litellm/proxy/_experimental/out/{usage.html => usage/index.html} (100%) rename litellm/proxy/_experimental/out/{users.html => users/index.html} (100%) rename litellm/proxy/_experimental/out/{virtual-keys.html => virtual-keys/index.html} (100%) diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md new file mode 100644 index 00000000000..1bf52d922c6 --- /dev/null +++ b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md @@ -0,0 +1,279 @@ +# Braintrust Prompt Wrapper for LiteLLM + +This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API. + +## Architecture + +``` +┌─────────────┐ ┌──────────────────────┐ ┌─────────────┐ +│ LiteLLM │ ──────> │ Wrapper Server │ ──────> │ Braintrust │ +│ Client │ │ (This Server) │ │ API │ +└─────────────┘ └──────────────────────┘ └─────────────┘ + Uses generic Transforms Stores actual + prompt manager Braintrust format prompt templates + to LiteLLM format +``` + +## Components + +### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`) + +A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint. + +**Expected API Response Format:** +```json +{ + "prompt_id": "string", + "prompt_template": [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello {name}"} + ], + "prompt_template_model": "gpt-4", + "prompt_template_optional_params": { + "temperature": 0.7, + "max_tokens": 100 + } +} +``` + +### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`) + +A FastAPI server that: +- Implements the `/beta/litellm_prompt_management` endpoint +- Fetches prompts from Braintrust API +- Transforms Braintrust response format to LiteLLM format + +## Setup + +### Install Dependencies + +```bash +pip install fastapi uvicorn httpx litellm +``` + +### Set Environment Variables + +```bash +export BRAINTRUST_API_KEY="your-braintrust-api-key" +``` + +## Usage + +### Step 1: Start the Wrapper Server + +```bash +python braintrust_prompt_wrapper_server.py +``` + +The server will start on `http://localhost:8080` by default. + +You can customize the port and host: +```bash +export PORT=8000 +export HOST=0.0.0.0 +python braintrust_prompt_wrapper_server.py +``` + +### Step 2: Use with LiteLLM + +```python +import litellm +from litellm.integrations.generic_prompt_management import GenericPromptManager + +# Configure the generic prompt manager to use your wrapper server +generic_config = { + "api_base": "http://localhost:8080", + "api_key": "your-braintrust-api-key", # Will be passed to Braintrust + "timeout": 30, +} + +# Create the prompt manager +prompt_manager = GenericPromptManager(**generic_config) + +# Use with completion +response = litellm.completion( + model="generic_prompt/gpt-4", + prompt_id="your-braintrust-prompt-id", + prompt_variables={"name": "World"}, # Variables to substitute + messages=[{"role": "user", "content": "Additional message"}] +) + +print(response) +``` + +### Step 3: Direct API Testing + +You can also test the wrapper API directly: + +```bash +# Test with curl +curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ + "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" + +# Health check +curl http://localhost:8080/health + +# Service info +curl http://localhost:8080/ +``` + +## API Documentation + +Once the server is running, visit: +- Swagger UI: `http://localhost:8080/docs` +- ReDoc: `http://localhost:8080/redoc` + +## Braintrust Format Transformation + +The wrapper automatically transforms Braintrust's response format: + +**Braintrust API Response:** +```json +{ + "id": "prompt-123", + "prompt_data": { + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant" + } + ] + }, + "options": { + "model": "gpt-4", + "params": { + "temperature": 0.7, + "max_tokens": 100 + } + } + } +} +``` + +**Transformed to LiteLLM Format:** +```json +{ + "prompt_id": "prompt-123", + "prompt_template": [ + { + "role": "system", + "content": "You are a helpful assistant" + } + ], + "prompt_template_model": "gpt-4", + "prompt_template_optional_params": { + "temperature": 0.7, + "max_tokens": 100 + } +} +``` + +## Supported Parameters + +The wrapper automatically maps these Braintrust parameters to LiteLLM: + +- `temperature` +- `max_tokens` / `max_completion_tokens` +- `top_p` +- `frequency_penalty` +- `presence_penalty` +- `n` +- `stop` +- `response_format` +- `tool_choice` +- `function_call` +- `tools` + +## Variable Substitution + +The generic prompt manager supports simple variable substitution: + +```python +# In your Braintrust prompt: +# "Hello {name}, welcome to {place}!" + +# In your code: +prompt_variables = { + "name": "Alice", + "place": "Wonderland" +} + +# Result: +# "Hello Alice, welcome to Wonderland!" +``` + +Supports both `{variable}` and `{{variable}}` syntax. + +## Error Handling + +The wrapper provides detailed error messages: + +- **401**: Missing or invalid Braintrust API token +- **404**: Prompt not found in Braintrust +- **502**: Failed to connect to Braintrust API +- **500**: Error transforming response + +## Production Deployment + +For production use: + +1. **Use HTTPS**: Deploy behind a reverse proxy with SSL +2. **Authentication**: Add authentication to the wrapper endpoint if needed +3. **Rate Limiting**: Implement rate limiting to prevent abuse +4. **Caching**: Consider caching prompt responses +5. **Monitoring**: Add logging and monitoring + +Example with Docker: + +```dockerfile +FROM python:3.11-slim + +WORKDIR /app + +RUN pip install fastapi uvicorn httpx + +COPY braintrust_prompt_wrapper_server.py . + +ENV PORT=8080 +ENV HOST=0.0.0.0 + +EXPOSE 8080 + +CMD ["python", "braintrust_prompt_wrapper_server.py"] +``` + +## Extending to Other Providers + +This pattern can be used with any prompt management provider: + +1. Create a wrapper server that implements `/beta/litellm_prompt_management` +2. Transform the provider's response to LiteLLM format +3. Use the generic prompt manager to connect + +Example providers: +- Langsmith +- PromptLayer +- Humanloop +- Custom internal systems + +## Troubleshooting + +### "No Braintrust API token provided" +- Set `BRAINTRUST_API_KEY` environment variable +- Or pass token in `Authorization: Bearer TOKEN` header + +### "Failed to connect to Braintrust API" +- Check your internet connection +- Verify Braintrust API is accessible +- Check firewall settings + +### "Prompt not found" +- Verify the prompt ID exists in Braintrust +- Check that your API token has access to the prompt + +## License + +This wrapper is part of the LiteLLM project and follows the same license. + diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py new file mode 100644 index 00000000000..6379314c5b6 --- /dev/null +++ b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py @@ -0,0 +1,274 @@ +""" +Mock server that implements the /beta/litellm_prompt_management endpoint +and acts as a wrapper for calling the Braintrust API. + +This server transforms Braintrust's prompt API response into the format +expected by LiteLLM's generic prompt management client. + +Usage: + python braintrust_prompt_wrapper_server.py + + # Then test with: + curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ + "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" +""" + +import json +import os +from typing import Any, Dict, List, Optional + +import httpx +from fastapi import FastAPI, HTTPException, Header, Query +from fastapi.responses import JSONResponse +import uvicorn + + +app = FastAPI( + title="Braintrust Prompt Wrapper", + description="Wrapper server for Braintrust prompts to work with LiteLLM", + version="1.0.0", +) + + +def transform_braintrust_message(message: Dict[str, Any]) -> Dict[str, str]: + """ + Transform a Braintrust message to LiteLLM format. + + Braintrust message format: + { + "role": "system", + "content": "...", + "name": "..." (optional) + } + + LiteLLM format: + { + "role": "system", + "content": "..." + } + """ + result = { + "role": message.get("role", "user"), + "content": message.get("content", ""), + } + + # Include name if present + if "name" in message: + result["name"] = message["name"] + + return result + + +def transform_braintrust_response( + braintrust_response: Dict[str, Any], +) -> Dict[str, Any]: + """ + Transform Braintrust API response to LiteLLM prompt management format. + + Braintrust response format: + { + "objects": [{ + "id": "prompt_id", + "prompt_data": { + "prompt": { + "type": "chat", + "messages": [...], + "tools": "..." + }, + "options": { + "model": "gpt-4", + "params": { + "temperature": 0.7, + "max_tokens": 100, + ... + } + } + } + }] + } + + LiteLLM format: + { + "prompt_id": "prompt_id", + "prompt_template": [...], + "prompt_template_model": "gpt-4", + "prompt_template_optional_params": {...} + } + """ + # Extract the first object from the objects array if it exists + if "objects" in braintrust_response and len(braintrust_response["objects"]) > 0: + prompt_object = braintrust_response["objects"][0] + else: + prompt_object = braintrust_response + + prompt_data = prompt_object.get("prompt_data", {}) + prompt_info = prompt_data.get("prompt", {}) + options = prompt_data.get("options", {}) + + # Extract messages + messages = prompt_info.get("messages", []) + transformed_messages = [transform_braintrust_message(msg) for msg in messages] + + # Extract model + model = options.get("model") + + # Extract optional parameters + params = options.get("params", {}) + optional_params: Dict[str, Any] = {} + + # Map common parameters + param_mapping = { + "temperature": "temperature", + "max_tokens": "max_tokens", + "max_completion_tokens": "max_tokens", # Alternative name + "top_p": "top_p", + "frequency_penalty": "frequency_penalty", + "presence_penalty": "presence_penalty", + "n": "n", + "stop": "stop", + } + + for braintrust_param, litellm_param in param_mapping.items(): + if braintrust_param in params: + value = params[braintrust_param] + if value is not None: + optional_params[litellm_param] = value + + # Handle response_format + if "response_format" in params: + optional_params["response_format"] = params["response_format"] + + # Handle tool_choice + if "tool_choice" in params: + optional_params["tool_choice"] = params["tool_choice"] + + # Handle function_call + if "function_call" in params: + optional_params["function_call"] = params["function_call"] + + # Add tools if present + if "tools" in prompt_info and prompt_info["tools"]: + optional_params["tools"] = prompt_info["tools"] + + # Handle tool_functions from prompt_data + if "tool_functions" in prompt_data and prompt_data["tool_functions"]: + optional_params["tool_functions"] = prompt_data["tool_functions"] + + return { + "prompt_id": prompt_object.get("id"), + "prompt_template": transformed_messages, + "prompt_template_model": model, + "prompt_template_optional_params": optional_params if optional_params else None, + } + + +@app.get("/beta/litellm_prompt_management") +async def get_prompt( + prompt_id: str = Query(..., description="The Braintrust prompt ID to fetch"), + authorization: Optional[str] = Header( + None, description="Bearer token for Braintrust API" + ), +) -> JSONResponse: + """ + Fetch a prompt from Braintrust and transform it to LiteLLM format. + + Args: + prompt_id: The Braintrust prompt ID + authorization: Bearer token for Braintrust API (from header) + + Returns: + JSONResponse with the transformed prompt data + """ + # Extract token from Authorization header or environment + braintrust_token = None + if authorization and authorization.startswith("Bearer "): + braintrust_token = authorization.replace("Bearer ", "") + else: + braintrust_token = os.getenv("BRAINTRUST_API_KEY") + + if not braintrust_token: + raise HTTPException( + status_code=401, + detail="No Braintrust API token provided. Pass via Authorization header or set BRAINTRUST_API_KEY environment variable.", + ) + + # Call Braintrust API + braintrust_url = f"https://api.braintrust.dev/v1/prompt/{prompt_id}" + headers = { + "Authorization": f"Bearer {braintrust_token}", + "Accept": "application/json", + } + print(f"headers: {headers}") + print(f"braintrust_url: {braintrust_url}") + print(f"braintrust_token: {braintrust_token}") + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(braintrust_url, headers=headers) + response.raise_for_status() + braintrust_data = response.json() + except httpx.HTTPStatusError as e: + raise HTTPException( + status_code=e.response.status_code, + detail=f"Braintrust API error: {e.response.text}", + ) + except httpx.RequestError as e: + raise HTTPException( + status_code=502, + detail=f"Failed to connect to Braintrust API: {str(e)}", + ) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=502, + detail=f"Failed to parse Braintrust API response: {str(e)}", + ) + + print(f"braintrust_data: {braintrust_data}") + # Transform the response + try: + transformed_data = transform_braintrust_response(braintrust_data) + print(f"transformed_data: {transformed_data}") + return JSONResponse(content=transformed_data) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to transform Braintrust response: {str(e)}", + ) + + +@app.get("/health") +async def health_check(): + """Health check endpoint.""" + return {"status": "healthy", "service": "braintrust-prompt-wrapper"} + + +@app.get("/") +async def root(): + """Root endpoint with service information.""" + return { + "service": "Braintrust Prompt Wrapper for LiteLLM", + "version": "1.0.0", + "endpoints": { + "prompt_management": "/beta/litellm_prompt_management?prompt_id=", + "health": "/health", + }, + "documentation": "/docs", + } + + +def main(): + """Run the server.""" + port = int(os.getenv("PORT", "8080")) + host = os.getenv("HOST", "0.0.0.0") + + print(f"🚀 Starting Braintrust Prompt Wrapper Server on {host}:{port}") + print(f"📚 API Documentation available at http://{host}:{port}/docs") + print( + f"🔑 Make sure to set BRAINTRUST_API_KEY environment variable or pass token in Authorization header" + ) + + uvicorn.run(app, host=host, port=port) + + +if __name__ == "__main__": + main() diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 8b0a96842e1..45b932a73af 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -7,16 +7,18 @@ Users can define """ import copy -from typing import Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Optional, Tuple, Union, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.types.integrations.anthropic_cache_control_hook import ( CacheControlInjectionPoint, CacheControlMessageInjectionPoint, ) from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedContent +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams @@ -29,6 +31,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -141,6 +144,78 @@ class AnthropicCacheControlHook(CustomPromptManagement): """Return the integration name for this hook.""" return "anthropic_cache_control_hook" + def should_run_prompt_management( + self, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], + dynamic_callback_params: StandardCallbackDynamicParams, + ) -> bool: + """Always return False since this is not a true prompt management system.""" + return False + + def _compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """Not used - this hook only modifies messages, doesn't fetch prompts.""" + return PromptManagementClient( + prompt_id=prompt_id, + prompt_template=[], + prompt_template_model=None, + prompt_template_optional_params=None, + completed_messages=None, + ) + + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """Not used - this hook only modifies messages, doesn't fetch prompts.""" + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Any, + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """Async version - delegates to sync since no async operations needed.""" + return self.get_chat_completion_prompt( + model=model, + messages=messages, + non_default_params=non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + @staticmethod def should_use_anthropic_cache_control_hook(non_default_params: Dict) -> bool: if non_default_params.get("cache_control_injection_points", None): diff --git a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py index 39759910730..aa6bff5509e 100644 --- a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py +++ b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py @@ -13,6 +13,7 @@ from litellm.integrations.prompt_management_base import ( PromptManagementClient, ) from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams from .bitbucket_client import BitBucketClient @@ -414,7 +415,8 @@ class BitBucketPromptManager(CustomPromptManagement): def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: """ @@ -423,11 +425,12 @@ class BitBucketPromptManager(CustomPromptManagement): For BitBucket, we always return True and handle the prompt loading in the _compile_prompt_helper method. """ - return True + return prompt_id is not None def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, @@ -442,6 +445,9 @@ class BitBucketPromptManager(CustomPromptManagement): 3. Converts the rendered text into chat messages 4. Extracts model and optional parameters from metadata """ + if prompt_id is None: + raise ValueError("prompt_id is required for BitBucket prompt manager") + try: # Load the prompt from BitBucket if not already loaded if prompt_id not in self.prompt_manager.prompts: @@ -481,6 +487,31 @@ class BitBucketPromptManager(CustomPromptManagement): except Exception as e: raise ValueError(f"Error compiling prompt '{prompt_id}': {e}") + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """ + Async version of compile prompt helper. Since BitBucket operations use sync client, + this simply delegates to the sync version. + """ + if prompt_id is None: + raise ValueError("prompt_id is required for BitBucket prompt manager") + + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + def get_chat_completion_prompt( self, model: str, @@ -489,6 +520,7 @@ class BitBucketPromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -505,6 +537,39 @@ class BitBucketPromptManager(CustomPromptManagement): prompt_id, prompt_variables, dynamic_callback_params, - prompt_label, - prompt_version, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Any, + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Async version - delegates to PromptManagementBase async implementation. + """ + return await PromptManagementBase.async_get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + litellm_logging_obj=litellm_logging_obj, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + tools=tools, + prompt_label=prompt_label, + prompt_version=prompt_version, ) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 66d5553f5ca..6488128b215 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -20,6 +20,7 @@ from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER from litellm.types.integrations.argilla import ArgillaItem from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import ( AdapterCompletionStreamWrapper, CallTypes, @@ -158,9 +159,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, + prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -178,6 +182,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 401280647bf..875ad8f1ef5 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -6,6 +6,7 @@ from litellm.integrations.prompt_management_base import ( PromptManagementClient, ) from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams @@ -29,6 +30,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -48,14 +50,16 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: return True def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index 53a12914496..e3901bd9359 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams from .prompt_manager import PromptManager, PromptTemplate @@ -82,7 +83,8 @@ class DotpromptManager(CustomPromptManagement): def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: """ @@ -90,6 +92,8 @@ class DotpromptManager(CustomPromptManagement): Returns True if the prompt_id exists in our prompt manager. """ + if prompt_id is None: + return False try: return prompt_id in self.prompt_manager.list_prompts() except Exception: @@ -98,7 +102,8 @@ class DotpromptManager(CustomPromptManagement): def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, @@ -114,6 +119,9 @@ class DotpromptManager(CustomPromptManagement): 4. Extracts model and optional parameters from metadata """ + if prompt_id is None: + raise ValueError("prompt_id is required for dotprompt manager") + try: # Get the prompt template (versioned or base) @@ -153,6 +161,31 @@ class DotpromptManager(CustomPromptManagement): except Exception as e: raise ValueError(f"Error compiling prompt '{prompt_id}': {e}") + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """ + Async version of compile prompt helper. Since dotprompt operations are synchronous, + this simply delegates to the sync version. + """ + if prompt_id is None: + raise ValueError("prompt_id is required for dotprompt manager") + + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + def get_chat_completion_prompt( self, model: str, @@ -161,6 +194,7 @@ class DotpromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -177,8 +211,43 @@ class DotpromptManager(CustomPromptManagement): prompt_id, prompt_variables, dynamic_callback_params, - prompt_label, - prompt_version, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Any, + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Async version - delegates to PromptManagementBase async implementation. + """ + from litellm.integrations.prompt_management_base import PromptManagementBase + + return await PromptManagementBase.async_get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + litellm_logging_obj=litellm_logging_obj, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + tools=tools, + prompt_label=prompt_label, + prompt_version=prompt_version, ) def _convert_to_messages(self, rendered_content: str) -> List[AllMessageValues]: diff --git a/litellm/integrations/generic_prompt_management/__init__.py b/litellm/integrations/generic_prompt_management/__init__.py new file mode 100644 index 00000000000..7466dc9c68d --- /dev/null +++ b/litellm/integrations/generic_prompt_management/__init__.py @@ -0,0 +1,80 @@ +"""Generic prompt management integration for LiteLLM.""" + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from .generic_prompt_manager import GenericPromptManager + from litellm.types.prompts.init_prompts import PromptLiteLLMParams, PromptSpec + from litellm.integrations.custom_prompt_management import CustomPromptManagement + +from litellm.types.prompts.init_prompts import SupportedPromptIntegrations + +from .generic_prompt_manager import GenericPromptManager + +# Global instances +global_generic_prompt_config: Optional[dict] = None + + +def set_global_generic_prompt_config(config: dict) -> None: + """ + Set the global generic prompt configuration. + + Args: + config: Dictionary containing generic prompt configuration + - api_base: Base URL for the API + - api_key: Optional API key for authentication + - timeout: Request timeout in seconds (default: 30) + """ + import litellm + + litellm.global_generic_prompt_config = config # type: ignore + + +def prompt_initializer( + litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec" +) -> "CustomPromptManagement": + """ + Initialize a prompt from a generic prompt management API. + """ + prompt_id = getattr(litellm_params, "prompt_id", None) + + api_base = litellm_params.api_base + api_key = litellm_params.api_key + if not api_base: + raise ValueError("api_base is required in generic_prompt_config") + + provider_specific_query_params = litellm_params.provider_specific_query_params + + try: + generic_prompt_manager = GenericPromptManager( + api_base=api_base, + api_key=api_key, + prompt_id=prompt_id, + additional_provider_specific_query_params=provider_specific_query_params, + **litellm_params.model_dump( + exclude_none=True, + exclude={ + "prompt_id", + "api_key", + "provider_specific_query_params", + "api_base", + }, + ), + ) + + return generic_prompt_manager + except Exception as e: + raise e + + +prompt_initializer_registry = { + SupportedPromptIntegrations.GENERIC_PROMPT_MANAGEMENT.value: prompt_initializer, +} + +# Export public API +__all__ = [ + "GenericPromptManager", + "set_global_generic_prompt_config", + "global_generic_prompt_config", + "prompt_initializer_registry", +] diff --git a/litellm/integrations/generic_prompt_management/generic_prompt_manager.py b/litellm/integrations/generic_prompt_management/generic_prompt_manager.py new file mode 100644 index 00000000000..9490d9fde1c --- /dev/null +++ b/litellm/integrations/generic_prompt_management/generic_prompt_manager.py @@ -0,0 +1,501 @@ +""" +Generic prompt manager that integrates with LiteLLM's prompt management system. +Fetches prompts from any API that implements the /beta/litellm_prompt_management endpoint. +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import httpx + +from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.integrations.prompt_management_base import ( + PromptManagementBase, + PromptManagementClient, +) +from litellm.llms.custom_httpx.http_handler import ( + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec +from litellm.types.utils import StandardCallbackDynamicParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + +class GenericPromptManager(CustomPromptManagement): + """ + Generic prompt manager that integrates with LiteLLM's prompt management system. + + This class enables using prompts from any API that implements the + /beta/litellm_prompt_management endpoint. + + Usage: + # Configure API access + generic_config = { + "api_base": "https://your-api.com", + "api_key": "your-api-key", # optional + "timeout": 30, # optional, defaults to 30 + } + + # Use with completion + response = litellm.completion( + model="generic_prompt/gpt-4", + prompt_id="my_prompt_id", + prompt_variables={"variable": "value"}, + generic_prompt_config=generic_config, + messages=[{"role": "user", "content": "Additional message"}] + ) + """ + + def __init__( + self, + api_base: str, + api_key: Optional[str] = None, + timeout: int = 30, + prompt_id: Optional[str] = None, + additional_provider_specific_query_params: Optional[Dict[str, Any]] = None, + **kwargs, + ): + """ + Initialize the Generic Prompt Manager. + + Args: + api_base: Base URL for the API (e.g., "https://your-api.com") + api_key: Optional API key for authentication + timeout: Request timeout in seconds (default: 30) + prompt_id: Optional prompt ID to pre-load + """ + super().__init__(**kwargs) + self.api_base = api_base.rstrip("/") + self.api_key = api_key + self.timeout = timeout + self.prompt_id = prompt_id + self.additional_provider_specific_query_params = ( + additional_provider_specific_query_params + ) + self._prompt_cache: Dict[str, PromptManagementClient] = {} + + @property + def integration_name(self) -> str: + """Integration name used in model names like 'generic_prompt/gpt-4'.""" + return "generic_prompt" + + def _get_headers(self) -> Dict[str, str]: + """Get HTTP headers for API requests.""" + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + return headers + + def _fetch_prompt_from_api( + self, prompt_id: Optional[str], prompt_spec: Optional[PromptSpec] + ) -> Dict[str, Any]: + """ + Fetch a prompt from the API. + + Args: + prompt_id: The ID of the prompt to fetch + + Returns: + The prompt data from the API + + Raises: + Exception: If the API request fails + """ + if prompt_id is None and prompt_spec is None: + raise ValueError("prompt_id or prompt_spec is required") + + url = f"{self.api_base}/beta/litellm_prompt_management" + params = { + "prompt_id": prompt_id, + **(self.additional_provider_specific_query_params or {}), + } + http_client = _get_httpx_client() + + try: + + response = http_client.get( + url, + params=params, + headers=self._get_headers(), + ) + + response.raise_for_status() + return response.json() + except httpx.HTTPError as e: + raise Exception(f"Failed to fetch prompt '{prompt_id}' from API: {e}") + except json.JSONDecodeError as e: + raise Exception(f"Failed to parse prompt response for '{prompt_id}': {e}") + + async def async_fetch_prompt_from_api( + self, prompt_id: Optional[str], prompt_spec: Optional[PromptSpec] + ) -> Dict[str, Any]: + """ + Fetch a prompt from the API asynchronously. + """ + if prompt_id is None and prompt_spec is None: + raise ValueError("prompt_id or prompt_spec is required") + + url = f"{self.api_base}/beta/litellm_prompt_management" + params = { + "prompt_id": prompt_id, + **( + prompt_spec.litellm_params.provider_specific_query_params + if prompt_spec + and prompt_spec.litellm_params.provider_specific_query_params + else {} + ), + } + + http_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PromptManagement, + ) + + try: + response = await http_client.get( + url, + params=params, + headers=self._get_headers(), + ) + response.raise_for_status() + return response.json() + except httpx.HTTPError as e: + raise Exception(f"Failed to fetch prompt '{prompt_id}' from API: {e}") + except json.JSONDecodeError as e: + raise Exception(f"Failed to parse prompt response for '{prompt_id}': {e}") + + def _parse_api_response( + self, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], + api_response: Dict[str, Any], + ) -> PromptManagementClient: + """ + Parse the API response into a PromptManagementClient structure. + + Expected API response format: + { + "prompt_id": "string", + "prompt_template": [ + {"role": "system", "content": "..."}, + {"role": "user", "content": "..."} + ], + "prompt_template_model": "gpt-4", # optional + "prompt_template_optional_params": { # optional + "temperature": 0.7, + "max_tokens": 100 + } + } + + Args: + prompt_id: The ID of the prompt + api_response: The response from the API + + Returns: + PromptManagementClient structure + """ + return PromptManagementClient( + prompt_id=prompt_id, + prompt_template=api_response.get("prompt_template", []), + prompt_template_model=api_response.get("prompt_template_model"), + prompt_template_optional_params=api_response.get( + "prompt_template_optional_params" + ), + completed_messages=None, + ) + + def should_run_prompt_management( + self, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], + dynamic_callback_params: StandardCallbackDynamicParams, + ) -> bool: + """ + Determine if prompt management should run based on the prompt_id. + + For Generic Prompt Manager, we always return True and handle the prompt loading + in the _compile_prompt_helper method. + """ + if prompt_id is not None or ( + prompt_spec is not None + and prompt_spec.litellm_params.provider_specific_query_params is not None + ): + return True + return False + + def _get_cache_key( + self, + prompt_id: Optional[str], + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> str: + return f"{prompt_id}:{prompt_label}:{prompt_version}" + + def _common_caching_logic( + self, + prompt_id: Optional[str], + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + prompt_variables: Optional[dict] = None, + ) -> Optional[PromptManagementClient]: + """ + Common caching logic for the prompt manager. + """ + # Check cache first + cache_key = self._get_cache_key(prompt_id, prompt_label, prompt_version) + if cache_key in self._prompt_cache: + cached_prompt = self._prompt_cache[cache_key] + # Return a copy with variables applied if needed + if prompt_variables: + return self._apply_variables(cached_prompt, prompt_variables) + return cached_prompt + return None + + def _compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """ + Compile a prompt template into a PromptManagementClient structure. + + This method: + 1. Fetches the prompt from the API (with caching) + 2. Applies any prompt variables (if the API supports it) + 3. Returns the structured prompt data + + Args: + prompt_id: The ID of the prompt + prompt_variables: Variables to substitute in the template (optional) + dynamic_callback_params: Dynamic callback parameters + prompt_label: Optional label for the prompt version + prompt_version: Optional specific version number + + Returns: + PromptManagementClient structure + """ + cached_prompt = self._common_caching_logic( + prompt_id=prompt_id, + prompt_label=prompt_label, + prompt_version=prompt_version, + prompt_variables=prompt_variables, + ) + if cached_prompt: + return cached_prompt + + cache_key = self._get_cache_key(prompt_id, prompt_label, prompt_version) + try: + # Fetch from API + api_response = self._fetch_prompt_from_api(prompt_id, prompt_spec) + + # Parse the response + prompt_client = self._parse_api_response( + prompt_id, prompt_spec, api_response + ) + + # Cache the result + self._prompt_cache[cache_key] = prompt_client + + # Apply variables if provided + if prompt_variables: + prompt_client = self._apply_variables(prompt_client, prompt_variables) + + return prompt_client + + except Exception as e: + raise ValueError(f"Error compiling prompt '{prompt_id}': {e}") + + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + + # Check cache first + cached_prompt = self._common_caching_logic( + prompt_id=prompt_id, + prompt_label=prompt_label, + prompt_version=prompt_version, + prompt_variables=prompt_variables, + ) + if cached_prompt: + return cached_prompt + + cache_key = self._get_cache_key(prompt_id, prompt_label, prompt_version) + + try: + # Fetch from API + + api_response = await self.async_fetch_prompt_from_api( + prompt_id=prompt_id, prompt_spec=prompt_spec + ) + + # Parse the response + prompt_client = self._parse_api_response( + prompt_id, prompt_spec, api_response + ) + + # Cache the result + self._prompt_cache[cache_key] = prompt_client + + # Apply variables if provided + if prompt_variables: + prompt_client = self._apply_variables(prompt_client, prompt_variables) + + return prompt_client + + except Exception as e: + raise ValueError( + f"Error compiling prompt '{prompt_id}': {e}, prompt_spec: {prompt_spec}" + ) + + def _apply_variables( + self, + prompt_client: PromptManagementClient, + variables: Dict[str, Any], + ) -> PromptManagementClient: + """ + Apply variables to the prompt template. + + This performs simple string substitution using {variable_name} syntax. + + Args: + prompt_client: The prompt client structure + variables: Variables to substitute + + Returns: + Updated PromptManagementClient with variables applied + """ + # Create a copy of the prompt template with variables applied + updated_messages: List[AllMessageValues] = [] + for message in prompt_client["prompt_template"]: + updated_message = dict(message) # type: ignore + if "content" in updated_message and isinstance( + updated_message["content"], str + ): + content = updated_message["content"] + for key, value in variables.items(): + content = content.replace(f"{{{key}}}", str(value)) + content = content.replace( + f"{{{{{key}}}}}", str(value) + ) # Also support {{key}} + updated_message["content"] = content + updated_messages.append(updated_message) # type: ignore + + return PromptManagementClient( + prompt_id=prompt_client["prompt_id"], + prompt_template=updated_messages, + prompt_template_model=prompt_client["prompt_template_model"], + prompt_template_optional_params=prompt_client[ + "prompt_template_optional_params" + ], + completed_messages=None, + ) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: "LiteLLMLoggingObj", + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Get chat completion prompt and return processed model, messages, and parameters. + """ + + return await PromptManagementBase.async_get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + litellm_logging_obj=litellm_logging_obj, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + tools=tools, + prompt_label=prompt_label, + prompt_version=prompt_version, + ignore_prompt_manager_model=( + ignore_prompt_manager_model + or prompt_spec.litellm_params.ignore_prompt_manager_model + if prompt_spec + else False + ), + ignore_prompt_manager_optional_params=( + ignore_prompt_manager_optional_params + or prompt_spec.litellm_params.ignore_prompt_manager_optional_params + if prompt_spec + else False + ), + ) + + def get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Get chat completion prompt and return processed model, messages, and parameters. + """ + return PromptManagementBase.get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ignore_prompt_manager_model=( + ignore_prompt_manager_model + or prompt_spec.litellm_params.ignore_prompt_manager_model + if prompt_spec + else False + ), + ignore_prompt_manager_optional_params=( + ignore_prompt_manager_optional_params + or prompt_spec.litellm_params.ignore_prompt_manager_optional_params + if prompt_spec + else False + ), + ) + + def clear_cache(self) -> None: + """Clear the prompt cache.""" + self._prompt_cache.clear() diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index 9931c007dc7..85335a811a3 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -13,6 +13,7 @@ from litellm.integrations.prompt_management_base import ( PromptManagementClient, ) from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams GITLAB_PREFIX = "gitlab::" @@ -454,19 +455,24 @@ class GitLabPromptManager(CustomPromptManagement): def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: - return True + return prompt_id is not None def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ) -> PromptManagementClient: + if prompt_id is None: + raise ValueError("prompt_id is required for GitLab prompt manager") + try: decoded_id = decode_prompt_id(prompt_id) if decoded_id not in self.prompt_manager.prompts: @@ -505,6 +511,31 @@ class GitLabPromptManager(CustomPromptManagement): except Exception as e: raise ValueError(f"Error compiling prompt '{prompt_id}': {e}") + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """ + Async version of compile prompt helper. Since GitLab operations use sync client, + this simply delegates to the sync version. + """ + if prompt_id is None: + raise ValueError("prompt_id is required for GitLab prompt manager") + + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + def get_chat_completion_prompt( self, model: str, @@ -513,6 +544,7 @@ class GitLabPromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -526,8 +558,41 @@ class GitLabPromptManager(CustomPromptManagement): prompt_id, prompt_variables, dynamic_callback_params, - prompt_label, - prompt_version, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Any, + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Async version - delegates to PromptManagementBase async implementation. + """ + return await PromptManagementBase.async_get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + litellm_logging_obj=litellm_logging_obj, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + tools=tools, + prompt_label=prompt_label, + prompt_version=prompt_version, ) diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index ebab9840036..8e562238cc7 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -13,6 +13,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.litellm_core_utils.asyncify import run_async_function from litellm.types.llms.openai import AllMessageValues, ChatCompletionSystemMessage +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import ( @@ -183,6 +184,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, + prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, @@ -200,9 +202,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: + if prompt_id is None: + return False langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), langfuse_secret=dynamic_callback_params.get("langfuse_secret"), @@ -217,12 +222,16 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ) -> PromptManagementClient: + if prompt_id is None: + raise ValueError("prompt_id is required for Langfuse prompt management") + langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), langfuse_secret=dynamic_callback_params.get("langfuse_secret"), @@ -257,6 +266,24 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge completed_messages=None, ) + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + def log_success_event(self, kwargs, response_obj, start_time, end_time): return run_async_function( self.async_log_success_event, kwargs, response_obj, start_time, end_time diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 90321ad0fa8..b32f78c0dea 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -1,14 +1,18 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Tuple -from typing_extensions import TypedDict +from typing_extensions import TYPE_CHECKING, TypedDict from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + class PromptManagementClient(TypedDict): - prompt_id: str + prompt_id: Optional[str] prompt_template: List[AllMessageValues] prompt_template_model: Optional[str] prompt_template_optional_params: Optional[Dict[str, Any]] @@ -24,7 +28,8 @@ class PromptManagementBase(ABC): @abstractmethod def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: pass @@ -32,7 +37,8 @@ class PromptManagementBase(ABC): @abstractmethod def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, @@ -40,6 +46,18 @@ class PromptManagementBase(ABC): ) -> PromptManagementClient: pass + @abstractmethod + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + pass + def merge_messages( self, prompt_template: List[AllMessageValues], @@ -55,10 +73,41 @@ class PromptManagementBase(ABC): dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + prompt_spec: Optional[PromptSpec] = None, ) -> PromptManagementClient: compiled_prompt_client = self._compile_prompt_helper( prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + + try: + messages = compiled_prompt_client["prompt_template"] + client_messages + except Exception as e: + raise ValueError( + f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}" + ) + + compiled_prompt_client["completed_messages"] = messages + return compiled_prompt_client + + async def async_compile_prompt( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + client_messages: List[AllMessageValues], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + compiled_prompt_client = await self.async_compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, prompt_label=prompt_label, @@ -83,6 +132,39 @@ class PromptManagementBase(ABC): else: return model.replace("{}/".format(self.integration_name), "") + def post_compile_prompt_processing( + self, + prompt_template: PromptManagementClient, + messages: List[AllMessageValues], + non_default_params: dict, + model: str, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, + ): + completed_messages = prompt_template["completed_messages"] or messages + + prompt_template_optional_params = ( + prompt_template["prompt_template_optional_params"] or {} + ) + + updated_non_default_params = { + **non_default_params, + **( + prompt_template_optional_params + if not ignore_prompt_manager_optional_params + else {} + ), + } + + if not ignore_prompt_manager_model: + model = self._get_model_from_prompt( + prompt_management_client=prompt_template, model=model + ) + else: + model = model + + return model, completed_messages, updated_non_default_params + def get_chat_completion_prompt( self, model: str, @@ -91,6 +173,7 @@ class PromptManagementBase(ABC): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -100,7 +183,9 @@ class PromptManagementBase(ABC): if prompt_id is None: raise ValueError("prompt_id is required for Prompt Management Base class") if not self.should_run_prompt_management( - prompt_id=prompt_id, dynamic_callback_params=dynamic_callback_params + prompt_id=prompt_id, + prompt_spec=prompt_spec, + dynamic_callback_params=dynamic_callback_params, ): return model, messages, non_default_params @@ -113,26 +198,53 @@ class PromptManagementBase(ABC): prompt_version=prompt_version, ) - completed_messages = prompt_template["completed_messages"] or messages - - prompt_template_optional_params = ( - prompt_template["prompt_template_optional_params"] or {} + return self.post_compile_prompt_processing( + prompt_template=prompt_template, + messages=messages, + non_default_params=non_default_params, + model=model, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) - if not ignore_prompt_manager_optional_params: - updated_non_default_params = { - **non_default_params, - **prompt_template_optional_params, - } - else: - updated_non_default_params = non_default_params + async def async_get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: "LiteLLMLoggingObj", + prompt_spec: Optional[PromptSpec] = None, + tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, + ) -> Tuple[str, List[AllMessageValues], dict]: + if not self.should_run_prompt_management( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + dynamic_callback_params=dynamic_callback_params, + ): + return model, messages, non_default_params - if not ignore_prompt_manager_model: - model = self._get_model_from_prompt( - prompt_management_client=prompt_template, model=model - ) - else: - model = model + prompt_template = await self.async_compile_prompt( + prompt_id=prompt_id, + prompt_variables=prompt_variables, + client_messages=messages, + dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) - - return model, completed_messages, updated_non_default_params + return self.post_compile_prompt_processing( + prompt_template=prompt_template, + messages=messages, + non_default_params=non_default_params, + model=model, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, + ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e1277138456..f2f6a785969 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -85,6 +85,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ) from litellm.types.mcp import MCPPostCallResponseObject +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CachingDetails, @@ -265,6 +266,7 @@ def _get_cached_prometheus_logger(): global _PrometheusLogger if _PrometheusLogger is None: from litellm.integrations.prometheus import PrometheusLogger + _PrometheusLogger = PrometheusLogger return _PrometheusLogger @@ -601,8 +603,9 @@ class Logging(LiteLLMLoggingBaseClass): model: str, messages: List[AllMessageValues], non_default_params: Dict, - prompt_id: Optional[str], prompt_variables: Optional[dict], + prompt_id: Optional[str] = None, + prompt_spec: Optional[PromptSpec] = None, prompt_management_logger: Optional[CustomLogger] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, @@ -613,6 +616,7 @@ class Logging(LiteLLMLoggingBaseClass): model=model, non_default_params=non_default_params, prompt_id=prompt_id, + prompt_spec=prompt_spec, dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -627,6 +631,7 @@ class Logging(LiteLLMLoggingBaseClass): messages=messages, non_default_params=non_default_params or {}, prompt_id=prompt_id, + prompt_spec=prompt_spec, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, prompt_label=prompt_label, @@ -640,8 +645,9 @@ class Logging(LiteLLMLoggingBaseClass): model: str, messages: List[AllMessageValues], non_default_params: Dict, - prompt_id: Optional[str], prompt_variables: Optional[dict], + prompt_id: Optional[str] = None, + prompt_spec: Optional[PromptSpec] = None, prompt_management_logger: Optional[CustomLogger] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, @@ -654,6 +660,7 @@ class Logging(LiteLLMLoggingBaseClass): tools=tools, non_default_params=non_default_params, prompt_id=prompt_id, + prompt_spec=prompt_spec, dynamic_callback_params=self.standard_callback_dynamic_params, ) ) @@ -668,6 +675,7 @@ class Logging(LiteLLMLoggingBaseClass): messages=messages, non_default_params=non_default_params or {}, prompt_id=prompt_id, + prompt_spec=prompt_spec, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, litellm_logging_obj=self, @@ -681,6 +689,7 @@ class Logging(LiteLLMLoggingBaseClass): def _auto_detect_prompt_management_logger( self, prompt_id: str, + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> Optional[CustomLogger]: """ @@ -706,6 +715,7 @@ class Logging(LiteLLMLoggingBaseClass): try: if logger.should_run_prompt_management( prompt_id=prompt_id, + prompt_spec=prompt_spec, dynamic_callback_params=dynamic_callback_params, ): self.model_call_details["prompt_integration"] = ( @@ -724,6 +734,7 @@ class Logging(LiteLLMLoggingBaseClass): non_default_params: Dict, tools: Optional[List[Dict]] = None, prompt_id: Optional[str] = None, + prompt_spec: Optional[PromptSpec] = None, dynamic_callback_params: Optional[StandardCallbackDynamicParams] = None, ) -> Optional[CustomLogger]: """ @@ -756,6 +767,7 @@ class Logging(LiteLLMLoggingBaseClass): if prompt_id and dynamic_callback_params is not None: auto_detected_logger = self._auto_detect_prompt_management_logger( prompt_id=prompt_id, + prompt_spec=prompt_spec, dynamic_callback_params=dynamic_callback_params, ) if auto_detected_logger is not None: @@ -3516,7 +3528,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _literalai_logger # type: ignore elif logging_integration == "prometheus": PrometheusLogger = _get_cached_prometheus_logger() - + for callback in _in_memory_loggers: if isinstance(callback, PrometheusLogger): return callback # type: ignore @@ -3835,9 +3847,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "weave_otel": - from litellm.integrations.opentelemetry import ( - OpenTelemetryConfig, - ) + from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( WeaveOtelLogger, get_weave_otel_config, diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails.html deleted file mode 100644 index c67546be7e8..00000000000 --- a/litellm/proxy/_experimental/out/guardrails.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/login.html b/litellm/proxy/_experimental/out/login/index.html similarity index 100% rename from litellm/proxy/_experimental/out/login.html rename to litellm/proxy/_experimental/out/login/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index ff07168af10..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/playground.html b/litellm/proxy/_experimental/out/playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/playground.html rename to litellm/proxy/_experimental/out/playground/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index bb0981b73d6..df632ad453f 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -10,11 +10,25 @@ model_list: litellm_params: model: openai/gpt-4.1-mini + +# guardrails: +# - guardrail_name: generic-guardrail +# litellm_params: +# guardrail: generic_guardrail_api +# mode: ["pre_call"] +# headers: +# Authorization: Bearer mock-bedrock-token-12345 +# api_base: http://localhost:8080 +# default_on: true + prompts: - prompt_id: "simple_prompt" litellm_params: - prompt_id: "UHJvbXB0VmVyc2lvbjox" - prompt_integration: "arize_phoenix" - api_base: https://app.phoenix.arize.com/s/krrishdholakia - ignore_prompt_manager_model: true # ignores model from prompt manager - ignore_prompt_manager_optional_params: true # ignores optional params from prompt manager - e.g. temperature, max_tokens, etc. \ No newline at end of file + prompt_integration: "generic_prompt_management" + provider_specific_query_params: + project_name: litellm + slug: hello-world-prompt-2bac + api_base: http://localhost:8080 + api_key: os.environ/BRAINTRUST_API_KEY + ignore_prompt_manager_model: true + ignore_prompt_manager_optional_params: true diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ee5f5ffa3ed..275baa88da8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -824,7 +824,7 @@ class ProxyLogging: return data - def _process_prompt_template( + async def _process_prompt_template( self, data: dict, litellm_logging_obj: Any, @@ -833,6 +833,7 @@ class ProxyLogging: call_type: CallTypesLiteral, ) -> None: """Process prompt template if applicable.""" + from litellm.proxy.prompts.prompt_endpoints import ( construct_versioned_prompt_id, get_latest_version_prompt_id, @@ -857,21 +858,24 @@ class ProxyLogging: litellm_prompt_id: Optional[str] = None if prompt_spec is not None: litellm_prompt_id = prompt_spec.litellm_params.prompt_id + data.pop("prompt_id", None) + + if custom_logger and prompt_spec is not None: - if custom_logger and litellm_prompt_id is not None: ( model, messages, optional_params, - ) = litellm_logging_obj.get_chat_completion_prompt( + ) = await litellm_logging_obj.async_get_chat_completion_prompt( model=data.get("model", ""), messages=data.get("messages", []), - non_default_params=get_non_default_completion_params(kwargs=data), + non_default_params=get_non_default_completion_params(kwargs=data) or {}, prompt_id=litellm_prompt_id, + prompt_spec=prompt_spec, prompt_management_logger=custom_logger, - prompt_variables=data.get("prompt_variables", None), - prompt_label=data.get("prompt_label", None), - prompt_version=data.get("prompt_version", None), + prompt_variables=data.pop("prompt_variables", None) or {}, + prompt_label=data.pop("prompt_label", None) or {}, + prompt_version=data.pop("prompt_version", None) or {}, ) data.update(optional_params) @@ -976,8 +980,7 @@ class ProxyLogging: and prompt_id is not None and (call_type == "completion" or call_type == "acompletion") ): - - self._process_prompt_template( + await self._process_prompt_template( data=data, litellm_logging_obj=litellm_logging_obj, prompt_id=prompt_id, diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 9ed25005c05..ca348bad97c 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -25,6 +25,7 @@ class httpxSpecialProvider(str, Enum): MCP = "mcp" RAG = "rag" A2A = "a2a" + PromptManagement = "prompt_management" VerifyTypes = Union[str, bool, ssl.SSLContext] diff --git a/litellm/types/prompts/init_prompts.py b/litellm/types/prompts/init_prompts.py index 5621b4483e3..2d9f807bc26 100644 --- a/litellm/types/prompts/init_prompts.py +++ b/litellm/types/prompts/init_prompts.py @@ -11,6 +11,7 @@ class SupportedPromptIntegrations(str, Enum): CUSTOM = "custom" BITBUCKET = "bitbucket" GITLAB = "gitlab" + GENERIC_PROMPT_MANAGEMENT = "generic_prompt_management" ARIZE_PHOENIX = "arize_phoenix" @@ -21,10 +22,16 @@ class PromptInfo(BaseModel): class PromptLiteLLMParams(BaseModel): - prompt_id: str + prompt_id: Optional[str] = None prompt_integration: str - api_key: Optional[str] = None + api_base: Optional[str] = None + api_key: Optional[str] = None + + provider_specific_query_params: Optional[Dict[str, Any]] = None + + ignore_prompt_manager_model: Optional[bool] = False + ignore_prompt_manager_optional_params: Optional[bool] = False dotprompt_content: Optional[str] = None """ From c1c8a6937eca7f744d163295970a3733ba631873 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 16:11:09 -0800 Subject: [PATCH 10/66] Renaming + fixing tests --- .../src/app/(dashboard)/usage/page.tsx | 4 ++-- ui/litellm-dashboard/src/app/page.tsx | 2 +- ui/litellm-dashboard/src/components/usage.tsx | 2 +- ...ity_usage.test.tsx => EntityUsage.test.tsx} | 10 +++++----- .../{entity_usage.tsx => EntityUsage.tsx} | 4 ++-- ...p_key_view.test.tsx => TopKeyView.test.tsx} | 2 +- .../{top_key_view.tsx => TopKeyView.tsx} | 0 ...del_view.test.tsx => TopModelView.test.tsx} | 2 +- .../{top_model_view.tsx => TopModelView.tsx} | 0 ...w_usage.test.tsx => UsagePageView.test.tsx} | 18 +++++++++--------- .../{new_usage.tsx => UsagePageView.tsx} | 17 +++++------------ .../tests/top_key_view.test.tsx | 2 +- 12 files changed, 28 insertions(+), 35 deletions(-) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{entity_usage.test.tsx => EntityUsage.test.tsx} (97%) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{entity_usage.tsx => EntityUsage.tsx} (99%) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{top_key_view.test.tsx => TopKeyView.test.tsx} (98%) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{top_key_view.tsx => TopKeyView.tsx} (100%) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{top_model_view.test.tsx => TopModelView.test.tsx} (97%) rename ui/litellm-dashboard/src/components/usage/components/EntityUsage/{top_model_view.tsx => TopModelView.tsx} (100%) rename ui/litellm-dashboard/src/components/usage/components/{new_usage.test.tsx => UsagePageView.test.tsx} (96%) rename ui/litellm-dashboard/src/components/usage/components/{new_usage.tsx => UsagePageView.tsx} (99%) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx index bc73a0d7b33..2fecfb6a0b2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx @@ -1,6 +1,6 @@ "use client"; -import NewUsagePage from "@/components/Usage/components/new_usage"; +import UsagePageView from "@/components/Usage/components/UsagePageView"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useTeams from "@/app/(dashboard)/hooks/useTeams"; @@ -9,7 +9,7 @@ const UsagePage = () => { const { teams } = useTeams(); return ( - { @@ -14,7 +14,7 @@ beforeAll(() => { }); // Mock the networking module -vi.mock("./networking", () => ({ +vi.mock("../../../networking", () => ({ tagDailyActivityCall: vi.fn(), teamDailyActivityCall: vi.fn(), organizationDailyActivityCall: vi.fn(), @@ -23,16 +23,16 @@ vi.mock("./networking", () => ({ })); // Mock the child components to simplify testing -vi.mock("./activity_metrics", () => ({ +vi.mock("../../../activity_metrics", () => ({ ActivityMetrics: () =>
Activity Metrics
, processActivityData: () => ({ data: [], metadata: {} }), })); -vi.mock("./top_key_view", () => ({ +vi.mock("./TopKeyView", () => ({ default: () =>
Top Keys
, })); -vi.mock("./top_model_view", () => ({ +vi.mock("./TopModelView", () => ({ default: () =>
Top Models
, })); diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx index 1d99c99844a..aa4eddd1bf3 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/entity_usage.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx @@ -30,13 +30,13 @@ import { customerDailyActivityCall, agentDailyActivityCall, } from "../../../networking"; -import TopKeyView from "./top_key_view"; +import TopKeyView from "./TopKeyView"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { valueFormatterSpend } from "../../utils/value_formatters"; import { getProviderLogoAndName } from "../../../provider_info_helpers"; import { UsageExportHeader } from "../../../EntityUsageExport"; import type { EntityType } from "../../../EntityUsageExport/types"; -import TopModelView from "./top_model_view"; +import TopModelView from "./TopModelView"; interface EntityMetrics { metrics: { diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx similarity index 98% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx index c918fdbd586..273b85d6d20 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx @@ -1,6 +1,6 @@ import { render } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import TopKeyView from "./top_key_view"; +import TopKeyView from "./TopKeyView"; describe("TopKeyView", () => { it("should render", () => { diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_key_view.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.test.tsx similarity index 97% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.test.tsx index d25055c3b91..fc2e63ec044 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.test.tsx @@ -1,6 +1,6 @@ import { render } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import TopModelView from "./top_model_view"; +import TopModelView from "./TopModelView"; describe("TopModelView", () => { it("should render", () => { diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/top_model_view.tsx rename to ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx similarity index 96% rename from ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx rename to ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx index 1358bf23295..da12660b15d 100644 --- a/ui/litellm-dashboard/src/components/usage/components/new_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx @@ -1,6 +1,6 @@ import { render, screen, fireEvent, waitFor, act } from "@testing-library/react"; import { describe, it, expect, vi, beforeEach, beforeAll } from "vitest"; -import NewUsagePage from "./new_usage"; +import NewUsagePage from "./UsagePageView"; import type { Organization } from "../../networking"; import * as networking from "../../networking"; import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; @@ -18,40 +18,40 @@ beforeAll(() => { }); // Mock the networking module -vi.mock("./networking", () => ({ +vi.mock("../../networking", () => ({ userDailyActivityCall: vi.fn(), userDailyActivityAggregatedCall: vi.fn(), tagListCall: vi.fn(), })); // Mock child components to simplify testing -vi.mock("./activity_metrics", () => ({ +vi.mock("../../activity_metrics", () => ({ ActivityMetrics: () =>
Activity Metrics
, processActivityData: () => ({ data: [], metadata: {} }), })); -vi.mock("./view_user_spend", () => ({ +vi.mock("../../view_user_spend", () => ({ default: () =>
View User Spend
, })); -vi.mock("./top_key_view", () => ({ +vi.mock("./EntityUsage/TopKeyView", () => ({ default: () =>
Top Keys
, })); -vi.mock("./entity_usage", () => ({ +vi.mock("./EntityUsage/EntityUsage", () => ({ default: () =>
Entity Usage
, EntityList: [], })); -vi.mock("./user_agent_activity", () => ({ +vi.mock("../../user_agent_activity", () => ({ default: () =>
User Agent Activity
, })); -vi.mock("./cloudzero_export_modal", () => ({ +vi.mock("../../cloudzero_export_modal", () => ({ default: () =>
CloudZero Export Modal
, })); -vi.mock("./EntityUsageExport", () => ({ +vi.mock("../../EntityUsageExport", () => ({ default: () =>
Entity Usage Export Modal
, })); diff --git a/ui/litellm-dashboard/src/components/usage/components/new_usage.tsx b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/usage/components/new_usage.tsx rename to ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx index d91c0743d8b..b8728b85605 100644 --- a/ui/litellm-dashboard/src/components/usage/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx @@ -36,7 +36,7 @@ import { Button } from "@tremor/react"; import { all_admin_roles } from "../../../utils/roles"; import { ActivityMetrics, processActivityData } from "../../activity_metrics"; import CloudZeroExportModal from "../../cloudzero_export_modal"; -import EntityUsage, { EntityList } from "./EntityUsage/entity_usage"; +import EntityUsage, { EntityList } from "./EntityUsage/EntityUsage"; import EntityUsageExportModal from "../../EntityUsageExport"; import { Team } from "../../key_team_helpers/key_list"; import { Organization, tagListCall, userDailyActivityAggregatedCall, userDailyActivityCall } from "../../networking"; @@ -44,14 +44,14 @@ import { getProviderLogoAndName } from "../../provider_info_helpers"; import AdvancedDatePicker from "../../shared/advanced_date_picker"; import { ChartLoader } from "../../shared/chart_loader"; import { Tag } from "../../tag_management/types"; -import TopKeyView from "./EntityUsage/top_key_view"; +import TopKeyView from "./EntityUsage/TopKeyView"; import { DailyData, KeyMetricWithMetadata, MetricWithMetadata } from "../types"; import { valueFormatterSpend } from "../utils/value_formatters"; import UserAgentActivity from "../../user_agent_activity"; import ViewUserSpend from "../../view_user_spend"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; -interface NewUsagePageProps { +interface UsagePageProps { accessToken: string | null; userRole: string | null; userID: string | null; @@ -60,14 +60,7 @@ interface NewUsagePageProps { premiumUser: boolean; } -const NewUsagePage: React.FC = ({ - accessToken, - userRole, - userID, - teams, - organizations, - premiumUser, -}) => { +const UsagePage: React.FC = ({ accessToken, userRole, userID, teams, organizations, premiumUser }) => { const [userSpendData, setUserSpendData] = useState<{ results: DailyData[]; metadata: any; @@ -928,4 +921,4 @@ const getModelActivityData = (userSpendData: { results: DailyData[]; metadata: a return modelData; }; -export default NewUsagePage; +export default UsagePage; diff --git a/ui/litellm-dashboard/tests/top_key_view.test.tsx b/ui/litellm-dashboard/tests/top_key_view.test.tsx index 5cc2de57378..08a762f0900 100644 --- a/ui/litellm-dashboard/tests/top_key_view.test.tsx +++ b/ui/litellm-dashboard/tests/top_key_view.test.tsx @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { renderWithProviders, screen, fireEvent } from "./test-utils"; -import TopKeyView from "../src/components/Usage/components/EntityUsage/top_key_view"; +import TopKeyView from "../src/components/Usage/components/EntityUsage/TopKeyView"; import { TagUsage } from "../src/components/Usage/types"; // Mock the networking module From 52cb54968a7df9e0596b7a5dd817c18d178f7353 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 16:28:03 -0800 Subject: [PATCH 11/66] Change to useAuthorized hook --- .../src/app/(dashboard)/usage/page.tsx | 11 +- ui/litellm-dashboard/src/app/page.tsx | 4 - ui/litellm-dashboard/src/components/usage.tsx | 18 +-- .../components/EntityUsage/EntityUsage.tsx | 10 +- .../EntityUsage/TopKeyView.test.tsx | 115 +++++++----------- .../components/EntityUsage/TopKeyView.tsx | 16 +-- .../usage/components/UsagePageView.test.tsx | 17 +++ .../usage/components/UsagePageView.tsx | 26 +--- .../src/components/view_user_spend.tsx | 14 +-- .../tests/top_key_view.test.tsx | 16 +++ 10 files changed, 92 insertions(+), 155 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx index 2fecfb6a0b2..d55939c7a6e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx @@ -8,16 +8,7 @@ const UsagePage = () => { const { accessToken, userRole, userId, premiumUser } = useAuthorized(); const { teams } = useTeams(); - return ( - - ); + return ; }; export default UsagePage; diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index b767c81e58b..b7157eed5ed 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -446,12 +446,8 @@ export default function CreateKeyPage() { ) : page == "new_usage" ? ( ) : ( = ({ accessToken, token, userRole, use Project Spend {new Date().toLocaleString("default", { month: "long" })} 1 -{" "} {new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0).getDate()} - + @@ -616,14 +609,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use Top Virtual Keys - + diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx index aa4eddd1bf3..3179ce25f80 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx @@ -576,15 +576,7 @@ const EntityUsage: React.FC = ({ Top Virtual Keys - + diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx index 273b85d6d20..6c074623139 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx @@ -1,87 +1,64 @@ -import { render } from "@testing-library/react"; -import { describe, expect, it } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi, beforeEach } from "vitest"; import TopKeyView from "./TopKeyView"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + __esModule: true, + default: vi.fn(), +})); describe("TopKeyView", () => { + const mockUseAuthorized = vi.mocked(useAuthorized); + const mockAuth = { + token: "mock-token", + accessToken: "test-token", + userId: "user-1", + userEmail: "user@example.com", + userRole: "admin", + premiumUser: true, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }; + const baseProps = { + topKeys: [], + teams: null, + showTags: false, + }; + + beforeEach(() => { + mockUseAuthorized.mockReturnValue(mockAuth); + }); + it("should render", () => { - const { container } = render( - , - ); - expect(container).toBeTruthy(); + render(); + expect(screen.getByText("Table View")).toBeInTheDocument(); }); it("should have a table view button", () => { - const { getByText } = render( - , - ); - expect(getByText("Table View")).toBeInTheDocument(); + render(); + expect(screen.getByText("Table View")).toBeInTheDocument(); }); it("should have a chart view", () => { - const { getByText } = render( - , - ); - expect(getByText("Chart View")).toBeInTheDocument(); + render(); + expect(screen.getByText("Chart View")).toBeInTheDocument(); }); ["Key ID", "Key Alias", "Spend (USD)"].forEach((header) => { it(`should have a ${header} column`, () => { - const { getByText } = render( - , - ); - expect(getByText(header)).toBeInTheDocument(); + render(); + expect(screen.getByText(header)).toBeInTheDocument(); }); }); it("should have a Tags column when showTags is true", () => { - const { getByText } = render( - , - ); - expect(getByText("Tags")).toBeInTheDocument(); + render(); + expect(screen.getByText("Tags")).toBeInTheDocument(); }); it("should show the key's information on the table", () => { - const { getByText } = render( + render( { ], }, ]} - accessToken="test-token" - userID={null} - userRole={null} teams={null} - premiumUser={false} showTags={true} />, ); - expect(getByText("Test Key")).toBeInTheDocument(); - expect(getByText(/tag-1/)).toBeInTheDocument(); - expect(getByText(/tag-2/)).toBeInTheDocument(); - expect(getByText("$100.00")).toBeInTheDocument(); + expect(screen.getByText("Test Key")).toBeInTheDocument(); + expect(screen.getByText(/tag-1/)).toBeInTheDocument(); + expect(screen.getByText(/tag-2/)).toBeInTheDocument(); + expect(screen.getByText("$100.00")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx index 43dc3244b4d..8f6bc411630 100644 --- a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx @@ -9,26 +9,16 @@ import { Button } from "@tremor/react"; import { formatNumberWithCommas } from "../../../../utils/dataUtils"; import { TagUsage } from "../../types"; import { ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/outline"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; interface TopKeyViewProps { topKeys: any[]; - accessToken: string | null; - userID: string | null; - userRole: string | null; teams: any[] | null; - premiumUser: boolean; showTags?: boolean; } -const TopKeyView: React.FC = ({ - topKeys, - accessToken, - userID, - userRole, - teams, - premiumUser, - showTags = false, -}) => { +const TopKeyView: React.FC = ({ topKeys, teams, showTags = false }) => { + const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); const [isModalOpen, setIsModalOpen] = useState(false); const [selectedKey, setSelectedKey] = useState(null); const [keyData, setKeyData] = useState(undefined); diff --git a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx index da12660b15d..265103b5ccc 100644 --- a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx @@ -5,6 +5,7 @@ import type { Organization } from "../../networking"; import * as networking from "../../networking"; import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; // Polyfill ResizeObserver for test environment beforeAll(() => { @@ -63,11 +64,17 @@ vi.mock("@/app/(dashboard)/hooks/agents/useAgents", () => ({ useAgents: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + __esModule: true, + default: vi.fn(), +})); + describe("NewUsage", () => { const mockUserDailyActivityAggregatedCall = vi.mocked(networking.userDailyActivityAggregatedCall); const mockTagListCall = vi.mocked(networking.tagListCall); const mockUseCustomers = vi.mocked(useCustomers); const mockUseAgents = vi.mocked(useAgents); + const mockUseAuthorized = vi.mocked(useAuthorized); const mockSpendData = { results: [ @@ -233,6 +240,16 @@ describe("NewUsage", () => { }; beforeEach(() => { + mockUseAuthorized.mockReturnValue({ + token: "mock-token", + accessToken: defaultProps.accessToken, + userId: defaultProps.userID, + userEmail: "test@example.com", + userRole: defaultProps.userRole, + premiumUser: defaultProps.premiumUser, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }); mockUserDailyActivityAggregatedCall.mockClear(); mockTagListCall.mockClear(); mockUserDailyActivityAggregatedCall.mockResolvedValue(mockSpendData); diff --git a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx index b8728b85605..d52e9d7a94b 100644 --- a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx @@ -50,17 +50,15 @@ import { valueFormatterSpend } from "../utils/value_formatters"; import UserAgentActivity from "../../user_agent_activity"; import ViewUserSpend from "../../view_user_spend"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; interface UsagePageProps { - accessToken: string | null; - userRole: string | null; - userID: string | null; teams: Team[]; organizations: Organization[]; - premiumUser: boolean; } -const UsagePage: React.FC = ({ accessToken, userRole, userID, teams, organizations, premiumUser }) => { +const UsagePage: React.FC = ({ teams, organizations }) => { + const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); const [userSpendData, setUserSpendData] = useState<{ results: DailyData[]; metadata: any; @@ -488,14 +486,7 @@ const UsagePage: React.FC = ({ accessToken, userRole, userID, te )} - + @@ -581,14 +572,7 @@ const UsagePage: React.FC = ({ accessToken, userRole, userID, te Top Virtual Keys - + diff --git a/ui/litellm-dashboard/src/components/view_user_spend.tsx b/ui/litellm-dashboard/src/components/view_user_spend.tsx index 611b308e500..51e87f8bc0d 100644 --- a/ui/litellm-dashboard/src/components/view_user_spend.tsx +++ b/ui/litellm-dashboard/src/components/view_user_spend.tsx @@ -2,6 +2,7 @@ import React, { useEffect, useState } from "react"; import { modelAvailableCall } from "./networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; // Define the props type interface UserSpendData { @@ -10,21 +11,12 @@ interface UserSpendData { // Add other properties if needed } interface ViewUserSpendProps { - userID: string | null; - userRole: string | null; - accessToken: string | null; userSpend: number | null; userMaxBudget: number | null; selectedTeam: any | null; } -const ViewUserSpend: React.FC = ({ - userID, - userRole, - accessToken, - userSpend, - userMaxBudget, - selectedTeam, -}) => { +const ViewUserSpend: React.FC = ({ userSpend, userMaxBudget, selectedTeam }) => { + const { accessToken, userRole, userId: userID } = useAuthorized(); console.log(`userSpend: ${userSpend}`); let [spend, setSpend] = useState(userSpend !== null ? userSpend : 0.0); const [maxBudget, setMaxBudget] = useState( diff --git a/ui/litellm-dashboard/tests/top_key_view.test.tsx b/ui/litellm-dashboard/tests/top_key_view.test.tsx index 08a762f0900..3cdb44cabb3 100644 --- a/ui/litellm-dashboard/tests/top_key_view.test.tsx +++ b/ui/litellm-dashboard/tests/top_key_view.test.tsx @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { renderWithProviders, screen, fireEvent } from "./test-utils"; import TopKeyView from "../src/components/Usage/components/EntityUsage/TopKeyView"; import { TagUsage } from "../src/components/Usage/types"; +import useAuthorized from "../src/app/(dashboard)/hooks/useAuthorized"; // Mock the networking module vi.mock("../src/components/networking", () => ({ @@ -13,7 +14,12 @@ vi.mock("../src/components/key_team_helpers/transform_key_info", () => ({ transformKeyInfo: vi.fn((data) => data), })); +vi.mock("../src/app/(dashboard)/hooks/useAuthorized", () => ({ + default: vi.fn(), +})); + describe("TopKeyView", () => { + const mockUseAuthorized = vi.mocked(useAuthorized); const mockProps = { topKeys: [], accessToken: "test-token", @@ -58,6 +64,16 @@ describe("TopKeyView", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseAuthorized.mockReturnValue({ + token: "mock-token", + accessToken: mockProps.accessToken, + userId: mockProps.userID, + userEmail: "test@example.com", + userRole: mockProps.userRole, + premiumUser: mockProps.premiumUser, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }); }); describe("Tags Column Visibility", () => { From 0635b1cbf05b8c698ed632495783cbe10c3bdac0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 16:33:23 -0800 Subject: [PATCH 12/66] rename --- ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx | 2 +- ui/litellm-dashboard/src/app/page.tsx | 2 +- .../components/EntityUsage/EntityUsage.test.tsx | 0 .../components/EntityUsage/EntityUsage.tsx | 0 .../components/EntityUsage/TopKeyView.test.tsx | 0 .../components/EntityUsage/TopKeyView.tsx | 0 .../components/EntityUsage/TopModelView.test.tsx | 0 .../components/EntityUsage/TopModelView.tsx | 0 .../{usage => UsagePage}/components/UsagePageView.test.tsx | 0 .../{usage => UsagePage}/components/UsagePageView.tsx | 0 .../src/components/{usage => UsagePage}/types.ts | 0 .../{usage => UsagePage}/utils/value_formatters.tsx | 0 ui/litellm-dashboard/src/components/activity_metrics.tsx | 4 ++-- .../src/components/common_components/chartUtils.tsx | 2 +- ui/litellm-dashboard/src/components/usage.tsx | 2 +- ui/litellm-dashboard/tests/top_key_view.test.tsx | 4 ++-- 16 files changed, 8 insertions(+), 8 deletions(-) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/EntityUsage.test.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/EntityUsage.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/TopKeyView.test.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/TopKeyView.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/TopModelView.test.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/EntityUsage/TopModelView.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/UsagePageView.test.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/components/UsagePageView.tsx (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/types.ts (100%) rename ui/litellm-dashboard/src/components/{usage => UsagePage}/utils/value_formatters.tsx (100%) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx index d55939c7a6e..477c1163ce7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx @@ -1,6 +1,6 @@ "use client"; -import UsagePageView from "@/components/Usage/components/UsagePageView"; +import UsagePageView from "@/components/UsagePage/components/UsagePageView"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useTeams from "@/app/(dashboard)/hooks/useTeams"; diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index b7157eed5ed..6b94f514d91 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -18,7 +18,7 @@ import { MCPServers } from "@/components/mcp_tools"; import ModelHubTable from "@/components/model_hub_table"; import Navbar from "@/components/navbar"; import { getUiConfig, Organization, proxyBaseUrl, setGlobalLitellmHeaderName } from "@/components/networking"; -import NewUsagePage from "@/components/Usage/components/UsagePageView"; +import NewUsagePage from "@/components/UsagePage/components/UsagePageView"; import OldTeams from "@/components/OldTeams"; import { fetchUserModels } from "@/components/organisms/create_key_button"; import Organizations, { fetchOrganizations } from "@/components/organizations"; diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.test.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/EntityUsage.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.test.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopKeyView.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.test.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/EntityUsage/TopModelView.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/UsagePageView.test.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx diff --git a/ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/components/UsagePageView.tsx rename to ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx diff --git a/ui/litellm-dashboard/src/components/usage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts similarity index 100% rename from ui/litellm-dashboard/src/components/usage/types.ts rename to ui/litellm-dashboard/src/components/UsagePage/types.ts diff --git a/ui/litellm-dashboard/src/components/usage/utils/value_formatters.tsx b/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/usage/utils/value_formatters.tsx rename to ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index c336da216bc..8b1c03e92db 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,10 +1,10 @@ import React from "react"; import { Card, Grid, Text, Title } from "@tremor/react"; import { AreaChart, BarChart } from "@tremor/react"; -import { DailyData, ModelActivityData, KeyMetricWithMetadata, TopApiKeyData } from "./Usage/types"; +import { DailyData, ModelActivityData, KeyMetricWithMetadata, TopApiKeyData } from "./UsagePage/types"; import { Collapse } from "antd"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { valueFormatter } from "./Usage/utils/value_formatters"; +import { valueFormatter } from "./UsagePage/utils/value_formatters"; import { CustomTooltip, CustomLegend } from "./common_components/chartUtils"; interface ActivityMetricsProps { diff --git a/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx b/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx index cc91b2a486d..7fff09056b0 100644 --- a/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx +++ b/ui/litellm-dashboard/src/components/common_components/chartUtils.tsx @@ -1,6 +1,6 @@ import React from "react"; import type { CustomTooltipProps } from "@tremor/react"; -import { SpendMetrics } from "../Usage/types"; +import { SpendMetrics } from "../UsagePage/types"; interface ChartDataPoint { date: string; diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 624e9da8311..2f55f242f6f 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -49,7 +49,7 @@ import { adminGlobalActivityPerModel, getProxyUISettings, } from "./networking"; -import TopKeyView from "./Usage/components/EntityUsage/TopKeyView"; +import TopKeyView from "./UsagePage/components/EntityUsage/TopKeyView"; import { formatNumberWithCommas } from "@/utils/dataUtils"; console.log("process.env.NODE_ENV", process.env.NODE_ENV); diff --git a/ui/litellm-dashboard/tests/top_key_view.test.tsx b/ui/litellm-dashboard/tests/top_key_view.test.tsx index 3cdb44cabb3..51662b8f453 100644 --- a/ui/litellm-dashboard/tests/top_key_view.test.tsx +++ b/ui/litellm-dashboard/tests/top_key_view.test.tsx @@ -1,7 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { renderWithProviders, screen, fireEvent } from "./test-utils"; -import TopKeyView from "../src/components/Usage/components/EntityUsage/TopKeyView"; -import { TagUsage } from "../src/components/Usage/types"; +import TopKeyView from "../src/components/UsagePage/components/EntityUsage/TopKeyView"; +import { TagUsage } from "../src/components/UsagePage/types"; import useAuthorized from "../src/app/(dashboard)/hooks/useAuthorized"; // Mock the networking module From dfdd74f9ac0a2186f2a2b53e9b10080c4f181b45 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 11 Dec 2025 17:37:09 -0800 Subject: [PATCH 13/66] Usage View Select --- .../UsagePage/components/UsagePageView.tsx | 826 +++++++++--------- .../UsageViewSelect/UsageViewSelect.test.tsx | 0 .../UsageViewSelect/UsageViewSelect.tsx | 171 ++++ 3 files changed, 586 insertions(+), 411 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index d52e9d7a94b..1c4dba84586 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -51,6 +51,7 @@ import UserAgentActivity from "../../user_agent_activity"; import ViewUserSpend from "../../view_user_spend"; import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { UsageViewSelect, UsageOption } from "./UsageViewSelect/UsageViewSelect"; interface UsagePageProps { teams: Team[]; @@ -86,7 +87,8 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const [isGlobalExportModalOpen, setIsGlobalExportModalOpen] = useState(false); const [showOrganizationBanner, setShowOrganizationBanner] = useState(true); const [showCustomerBanner, setShowCustomerBanner] = useState(true); - + const [usageView, setUsageView] = useState("global"); + const [showAgentBanner, setShowAgentBanner] = useState(true); const getAllTags = async () => { if (!accessToken) { return; @@ -416,431 +418,433 @@ const UsagePage: React.FC = ({ teams, organizations }) => { {/* Global Date Picker and Tabs - Single Row */}
- -
- - {all_admin_roles.includes(userRole || "") ? Global Usage : Your Usage} - {all_admin_roles.includes(userRole || "") ? ( - Organization Usage - ) : ( - Your Organization Usage - )} - Team Usage - {all_admin_roles.includes(userRole || "") ? Customer Usage : <>} - {all_admin_roles.includes(userRole || "") ? Tag Usage : <>} - {all_admin_roles.includes(userRole || "") ? Agent Usage : <>} - {all_admin_roles.includes(userRole || "") ? User Agent Activity : <>} - - -
- - {/* Your Usage Panel */} - - -
- - Cost - Model Activity - Key Activity - MCP Server Activity - -
+ {/* Your Usage Panel */} + {usageView === "global" && ( + +
+ + Cost + Model Activity + Key Activity + MCP Server Activity + + +
+ + {/* Cost Panel */} + + + {/* Total Spend Card */} + + + Project Spend{" "} + {dateValue.from && dateValue.to && ( + <> + {dateValue.from.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: dateValue.from.getFullYear() !== dateValue.to.getFullYear() ? "numeric" : undefined, + })} + {" - "} + {dateValue.to.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + })} + + )} + + + + + + + + Usage Metrics + + + Total Requests + + {userSpendData.metadata?.total_api_requests?.toLocaleString() || 0} + + + + Successful Requests + + {userSpendData.metadata?.total_successful_requests?.toLocaleString() || 0} + + + + Failed Requests + + {userSpendData.metadata?.total_failed_requests?.toLocaleString() || 0} + + + + Total Tokens + + {userSpendData.metadata?.total_tokens?.toLocaleString() || 0} + + + + Average Cost per Request + + $ + {formatNumberWithCommas( + (totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1), + 4, + )} + + + + + + + {/* Daily Spend Chart */} + + + Daily Spend + {loading ? ( + + ) : ( + new Date(a.date).getTime() - new Date(b.date).getTime(), + )} + index="date" + categories={["metrics.spend"]} + colors={["cyan"]} + valueFormatter={valueFormatterSpend} + yAxisWidth={100} + showLegend={false} + customTooltip={({ payload, active }) => { + if (!active || !payload?.[0]) return null; + const data = payload[0].payload; + return ( +
+

{data.date}

+

+ Spend: ${formatNumberWithCommas(data.metrics.spend, 2)} +

+

Requests: {data.metrics.api_requests}

+

Successful: {data.metrics.successful_requests}

+

Failed: {data.metrics.failed_requests}

+

Tokens: {data.metrics.total_tokens}

+
+ ); + }} /> - - )} - > - Export Data - -
- - {/* Cost Panel */} - - - {/* Total Spend Card */} - - - Project Spend{" "} - {dateValue.from && dateValue.to && ( - <> - {dateValue.from.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - year: - dateValue.from.getFullYear() !== dateValue.to.getFullYear() ? "numeric" : undefined, - })} - {" - "} - {dateValue.to.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - year: "numeric", - })} - - )} - + )} + + + {/* Top API Keys */} + + + Top Virtual Keys + + + - - + {/* Top Models */} + + +
+ {modelViewType === "groups" ? "Top Public Model Names" : "Top Litellm Models"} +
+ + +
+
+ {loading ? ( + + ) : ( + { + if (!active || !payload?.[0]) return null; + const data = payload[0].payload; + return ( +
+

{data.key}

+

Spend: ${formatNumberWithCommas(data.spend, 2)}

+

Total Requests: {data.requests.toLocaleString()}

+

+ Successful: {data.successful_requests.toLocaleString()} +

+

Failed: {data.failed_requests.toLocaleString()}

+

Tokens: {data.tokens.toLocaleString()}

+
+ ); + }} + /> + )} + + - - - Usage Metrics - - - Total Requests - - {userSpendData.metadata?.total_api_requests?.toLocaleString() || 0} - - - - Successful Requests - - {userSpendData.metadata?.total_successful_requests?.toLocaleString() || 0} - - - - Failed Requests - - {userSpendData.metadata?.total_failed_requests?.toLocaleString() || 0} - - - - Total Tokens - - {userSpendData.metadata?.total_tokens?.toLocaleString() || 0} - - - - Average Cost per Request - - $ - {formatNumberWithCommas( - (totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1), - 4, - )} - - - - - - - {/* Daily Spend Chart */} - - - Daily Spend - {loading ? ( - - ) : ( - new Date(a.date).getTime() - new Date(b.date).getTime(), - )} - index="date" - categories={["metrics.spend"]} - colors={["cyan"]} - valueFormatter={valueFormatterSpend} - yAxisWidth={100} - showLegend={false} - customTooltip={({ payload, active }) => { - if (!active || !payload?.[0]) return null; - const data = payload[0].payload; - return ( -
-

{data.date}

-

- Spend: ${formatNumberWithCommas(data.metrics.spend, 2)} -

-

Requests: {data.metrics.api_requests}

-

Successful: {data.metrics.successful_requests}

-

Failed: {data.metrics.failed_requests}

-

Tokens: {data.metrics.total_tokens}

-
- ); - }} - /> - )} - - - {/* Top API Keys */} - - - Top Virtual Keys - - - - - {/* Top Models */} - - -
- - {modelViewType === "groups" ? "Top Public Model Names" : "Top Litellm Models"} - -
- - -
-
- {loading ? ( - - ) : ( - + +
+ Spend by Provider +
+ {loading ? ( + + ) : ( + + + `$${formatNumberWithCommas(value, 2)}`} colors={["cyan"]} - valueFormatter={valueFormatterSpend} - layout="vertical" - yAxisWidth={200} - showLegend={false} - customTooltip={({ payload, active }) => { - if (!active || !payload?.[0]) return null; - const data = payload[0].payload; - return ( -
-

{data.key}

-

Spend: ${formatNumberWithCommas(data.spend, 2)}

-

Total Requests: {data.requests.toLocaleString()}

-

- Successful: {data.successful_requests.toLocaleString()} -

-

Failed: {data.failed_requests.toLocaleString()}

-

Tokens: {data.tokens.toLocaleString()}

-
- ); - }} /> - )} - - - - {/* Spend by Provider */} - - -
- Spend by Provider -
- {loading ? ( - - ) : ( - - - `$${formatNumberWithCommas(value, 2)}`} - colors={["cyan"]} - /> - - - - - - Provider - Spend - Successful - Failed - Tokens + + +
+ + + Provider + Spend + Successful + Failed + Tokens + + + + {getProviderSpend() + .filter((provider) => provider.spend > 0) + .map((provider) => ( + + +
+ {provider.provider && ( + {`${provider.provider} { + const target = e.target as HTMLImageElement; + const parent = target.parentElement; + if (parent) { + const fallbackDiv = document.createElement("div"); + fallbackDiv.className = + "w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs"; + fallbackDiv.textContent = provider.provider?.charAt(0) || "-"; + parent.replaceChild(fallbackDiv, target); + } + }} + /> + )} + {provider.provider} +
+ + ${formatNumberWithCommas(provider.spend, 2)} + + {provider.successful_requests.toLocaleString()} + + + {provider.failed_requests.toLocaleString()} + + {provider.tokens.toLocaleString()} - - - {getProviderSpend() - .filter((provider) => provider.spend > 0) - .map((provider) => ( - - -
- {provider.provider && ( - {`${provider.provider} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = provider.provider?.charAt(0) || "-"; - parent.replaceChild(fallbackDiv, target); - } - }} - /> - )} - {provider.provider} -
- - ${formatNumberWithCommas(provider.spend, 2)} - - {provider.successful_requests.toLocaleString()} - - - {provider.failed_requests.toLocaleString()} - - {provider.tokens.toLocaleString()} - - ))} - -
- - - )} - - + ))} + + + + + )} + + - {/* Usage Metrics */} - - + {/* Usage Metrics */} + + - {/* Activity Panel */} - - - - - - - - - - - - + {/* Activity Panel */} + + + + + + + + + + + + )} + {/* Organization Usage Panel */} - {/* Organization Usage Panel */} - - {showOrganizationBanner && ( - setShowOrganizationBanner(false)} - className="mb-5" - /> - )} - ({ - label: organization.organization_alias, - value: organization.organization_id, - })) || null - } - premiumUser={premiumUser} + {usageView === "organization" && ( + <> + {showOrganizationBanner && ( + setShowOrganizationBanner(false)} + className="mb-5" /> - + )} + ({ + label: organization.organization_alias, + value: organization.organization_id, + })) || null + } + premiumUser={premiumUser} + /> + + )} - {/* Team Usage Panel */} - - ({ - label: team.team_alias, - value: team.team_id, - })) || null - } - premiumUser={premiumUser} - dateValue={dateValue} - /> - + {/* Team Usage Panel */} + {usageView === "team" && ( + ({ + label: team.team_alias, + value: team.team_id, + })) || null + } + premiumUser={premiumUser} + dateValue={dateValue} + /> + )} - {/* Customer Usage Panel */} - - {showCustomerBanner && ( - setShowCustomerBanner(false)} - className="mb-5" - /> - )} - ({ - label: customer.alias || customer.user_id, - value: customer.user_id, - })) || null - } - premiumUser={premiumUser} - dateValue={dateValue} + {/* Customer Usage Panel */} + {usageView === "customer" && ( + <> + {showCustomerBanner && ( + setShowCustomerBanner(false)} + className="mb-5" /> - - {/* Tag Usage Panel */} - - ({ + label: customer.alias || customer.user_id, + value: customer.user_id, + })) || null + } + premiumUser={premiumUser} + dateValue={dateValue} + /> + + )} + {/* Tag Usage Panel */} + {usageView === "tag" && ( + + )} + {usageView === "agent" && ( + <> + {showAgentBanner && ( + setShowAgentBanner(false)} + className="mb-5" /> - - - ({ label: agent.agent_name, value: agent.agent_id })) || null - } - premiumUser={premiumUser} - dateValue={dateValue} - /> - - {/* User Agent Activity Panel */} - - - - - + )} + ({ label: agent.agent_name, value: agent.agent_id })) || null + } + premiumUser={premiumUser} + dateValue={dateValue} + />{" "} + + )} + {/* User Agent Activity Panel */} + {usageView === "user-agent-activity" && ( + + )}
diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx new file mode 100644 index 00000000000..1cbe30ebcfc --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx @@ -0,0 +1,171 @@ +import React from "react"; +import { Select } from "antd"; +import { + GlobalOutlined, + BankOutlined, + TeamOutlined, + ShoppingCartOutlined, + TagsOutlined, + RobotOutlined, + LineChartOutlined, + BarChartOutlined, +} from "@ant-design/icons"; +export type UsageOption = "global" | "organization" | "team" | "customer" | "tag" | "agent" | "user-agent-activity"; +export interface UsageViewSelectProps { + value: UsageOption; + onChange: (value: UsageOption) => void; + isAdmin: boolean; + title?: string; + description?: string; + "data-id"?: string; +} +interface OptionConfig { + value: UsageOption; + label: string; + description: string; + icon: React.ReactNode; + adminOnly?: boolean; + showForAdmin?: string; + showForNonAdmin?: string; + descriptionForAdmin?: string; + descriptionForNonAdmin?: string; +} +const OPTIONS: OptionConfig[] = [ + { + value: "global", + label: "Global Usage", + showForAdmin: "Global Usage", + showForNonAdmin: "Your Usage", + description: "View usage across all resources", + descriptionForAdmin: "View usage across all resources and users", + descriptionForNonAdmin: "View your personal usage statistics", + icon: , + }, + { + value: "organization", + label: "Organization Usage", + showForAdmin: "Organization Usage", + showForNonAdmin: "Your Organization Usage", + description: "View organization-level usage", + descriptionForAdmin: "View usage across all organizations", + descriptionForNonAdmin: "View your organization's usage statistics", + icon: , + }, + { + value: "team", + label: "Team Usage", + description: "View usage by team", + icon: , + }, + { + value: "customer", + label: "Customer Usage", + description: "View usage by customer accounts", + icon: , + adminOnly: true, + }, + { + value: "tag", + label: "Tag Usage", + description: "View usage grouped by tags", + icon: , + adminOnly: true, + }, + { + value: "agent", + label: "Agent Usage (A2A)", + description: "View usage by AI agents", + icon: , + adminOnly: true, + }, + { + value: "user-agent-activity", + label: "User Agent Activity", + description: "View detailed user agent activity logs", + icon: , + adminOnly: true, + }, +]; +export const UsageViewSelect: React.FC = ({ + value, + onChange, + isAdmin, + title = "Usage View", + description = "Select the usage data you want to view", + "data-id": dataId, +}) => { + const getFilteredOptions = () => { + return OPTIONS.filter((option) => { + if (option.adminOnly && !isAdmin) { + return false; + } + return true; + }).map((option) => { + let label = option.label; + let desc = option.description; + if (option.showForAdmin && option.showForNonAdmin) { + label = isAdmin ? option.showForAdmin : option.showForNonAdmin; + } + if (option.descriptionForAdmin && option.descriptionForNonAdmin) { + desc = isAdmin ? option.descriptionForAdmin : option.descriptionForNonAdmin; + } + return { + value: option.value, + label, + description: desc, + icon: option.icon, + }; + }); + }; + const filteredOptions = getFilteredOptions(); + return ( +
+
+
+
+ +
+
+

{title}

+

{description}

+
+
+
+ ({ value: opt.value, @@ -144,12 +147,17 @@ export const UsageViewSelect: React.FC = ({ const opt = filteredOptions.find((o) => o.value === option.value); if (!opt) return option.label; return ( -
+
{opt.icon}
{opt.label}
{opt.description}
+ {opt.badgeText && ( +
+ +
+ )}
); }} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 78d8a3940ce..2ddea2d1e32 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -21,7 +21,7 @@ import { ToolOutlined, UserOutlined, } from "@ant-design/icons"; -import { ConfigProvider, Layout, Menu } from "antd"; +import { Badge, ConfigProvider, Layout, Menu } from "antd"; import { all_admin_roles, internalUserRoles, isAdminRole, rolesWithWriteAccess } from "../utils/roles"; import UsageIndicator from "./usage_indicator"; const { Sider } = Layout; @@ -39,7 +39,7 @@ interface SidebarProps { interface MenuItem { key: string; page: string; - label: string; + label: string | React.ReactNode; roles?: string[]; children?: MenuItem[]; // Add children property for submenus icon?: React.ReactNode; @@ -71,7 +71,11 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau { key: "new_usage", page: "new_usage", - label: "Usage", + label: ( + + Usage + + ), icon: , roles: [...all_admin_roles, ...internalUserRoles], }, From d38f2410325e10bc26ef2c902532632ad055b23d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 10:18:20 -0800 Subject: [PATCH 43/66] [Feat] JWT Auth - auth allow selecting team_id from request header (#17884) * feat: add get_team_id_from_header for JWT Auth * fix Auth builder JWT Auth * test_get_team_id_from_header * test_auth_builder_uses_team_from_header_e2e * Select Team via Request Header --- docs/my-website/docs/proxy/token_auth.md | 20 +++++ litellm/proxy/auth/handle_jwt.py | 66 +++++++++++++++- litellm/proxy/auth/user_api_key_auth.py | 1 + .../proxy/auth/test_handle_jwt.py | 77 ++++++++++++++++++- 4 files changed, 160 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 1db1b2a8965..fe928a596cf 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -247,6 +247,26 @@ OIDC Auth for API: [**See Walkthrough**](https://www.loom.com/share/00fe2deab59a - Validate if any group has model access - If all checks pass, allow the request +### Select Team via Request Header + +When a JWT token contains multiple teams (via `team_ids_jwt_field`), you can explicitly select which team to use for a request by passing the `x-litellm-team-id` header. + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer ' \ +-H 'x-litellm-team-id: team_id_2' \ +-d '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}] +}' +``` + +**Validation:** +- The team ID in the header must exist in the JWT's `team_ids_jwt_field` list or match `team_id_jwt_field` +- If an invalid team is specified, a 403 error is returned +- If no header is provided, LiteLLM auto-selects the first team with access to the requested model + ### Custom JWT Validate diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index ed6877d1469..17ff0de9f7b 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -1029,6 +1029,47 @@ class JWTAuthManager: ) return True + @staticmethod + def get_team_id_from_header( + request_headers: Optional[dict], + allowed_team_ids: Set[str], + ) -> Optional[str]: + """ + Extract team_id from x-litellm-team-id header if present. + Validates that the team is in the user's allowed teams from JWT. + + Args: + request_headers: Dictionary of request headers + allowed_team_ids: Set of team IDs the user is allowed to access (from JWT) + + Returns: + The team_id from header if valid, None otherwise + + Raises: + HTTPException: If team_id is provided but not in allowed_team_ids + """ + if not request_headers: + return None + + # Normalize headers to lowercase for case-insensitive lookup + normalized_headers = {k.lower(): v for k, v in request_headers.items()} + header_team_id = normalized_headers.get("x-litellm-team-id") + + if not header_team_id: + return None + + # Validate that the team_id is in the allowed teams + if header_team_id not in allowed_team_ids: + raise HTTPException( + status_code=403, + detail=f"Team '{header_team_id}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", + ) + + verbose_proxy_logger.debug( + f"Using team_id from x-litellm-team-id header: {header_team_id}" + ) + return header_team_id + @staticmethod async def map_user_to_teams( user_object: Optional[LiteLLM_UserTable], @@ -1140,6 +1181,7 @@ class JWTAuthManager: user_api_key_cache: DualCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, + request_headers: Optional[dict] = None, ) -> JWTAuthBuilderResult: """Main authentication and authorization builder""" # Check if OIDC UserInfo endpoint is enabled @@ -1216,9 +1258,28 @@ class JWTAuthManager: return admin_result # Get team with model access - ## SPECIFIC TEAM ID + ## Check if team_id is specified via x-litellm-team-id header + all_team_ids = JWTAuthManager.get_all_team_ids(jwt_handler, jwt_valid_token) + specific_team_id = jwt_handler.get_team_id(token=jwt_valid_token, default_value=None) + if specific_team_id: + all_team_ids.add(specific_team_id) - if not team_id: + header_team_id = JWTAuthManager.get_team_id_from_header( + request_headers=request_headers, + allowed_team_ids=all_team_ids, + ) + if header_team_id: + team_id = header_team_id + team_object = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + ) + elif not team_id: + ## SPECIFIC TEAM ID ( team_id, team_object, @@ -1233,7 +1294,6 @@ class JWTAuthManager: if not team_object and not team_id: ## CHECK USER GROUP ACCESS - all_team_ids = JWTAuthManager.get_all_team_ids(jwt_handler, jwt_valid_token) team_id, team_object = await JWTAuthManager.find_team_with_model_access( team_ids=all_team_ids, requested_model=request_data.get("model"), diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a3c78af20f9..d0c284e921c 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -517,6 +517,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, parent_otel_span=parent_otel_span, + request_headers=dict(request.headers), ) is_proxy_admin = result["is_proxy_admin"] diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 603a6928f88..8ecbaced21e 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -1071,4 +1071,79 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): # Verify the result assert result["user_id"] == "test_user_1" - assert result["user_object"] == user_object \ No newline at end of file + assert result["user_object"] == user_object + + +def test_get_team_id_from_header(): + """Test get_team_id_from_header returns team when valid, None when missing, raises on invalid.""" + from fastapi import HTTPException + + # Valid team in allowed list + result = JWTAuthManager.get_team_id_from_header( + request_headers={"x-litellm-team-id": "team-1"}, + allowed_team_ids={"team-1", "team-2"}, + ) + assert result == "team-1" + + # No header returns None + result = JWTAuthManager.get_team_id_from_header( + request_headers={"authorization": "Bearer token"}, + allowed_team_ids={"team-1"}, + ) + assert result is None + + # Invalid team raises 403 + with pytest.raises(HTTPException) as exc_info: + JWTAuthManager.get_team_id_from_header( + request_headers={"x-litellm-team-id": "invalid-team"}, + allowed_team_ids={"team-1", "team-2"}, + ) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_auth_builder_uses_team_from_header_e2e(): + """Test auth_builder e2e flow: selects team from x-litellm-team-id header.""" + from litellm.caching import DualCache + from litellm.proxy.utils import ProxyLogging + + jwt_handler = JWTHandler() + user_api_key_cache = DualCache() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=user_api_key_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field="groups", + user_id_jwt_field="sub", + ), + ) + + team_object = LiteLLM_TeamTable(team_id="team-2") + user_object = LiteLLM_UserTable(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER) + + with patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, \ + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), \ + patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None), \ + patch("litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock) as mock_get_team, \ + patch.object(JWTAuthManager, "get_objects", new_callable=AsyncMock, return_value=(user_object, None, None, None)), \ + patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), \ + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock): + + mock_auth_jwt.return_value = {"sub": "user-1", "scope": "", "groups": ["team-1", "team-2"]} + mock_get_team.return_value = team_object + + result = await JWTAuthManager.auth_builder( + api_key="jwt-token", + jwt_handler=jwt_handler, + request_data={"model": "gpt-4"}, + general_settings={}, + route="/chat/completions", + prisma_client=None, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), + request_headers={"x-litellm-team-id": "team-2"}, + ) + + assert result["team_id"] == "team-2" + assert result["team_object"] == team_object \ No newline at end of file From f4db4b6f0e30e34b563bc5fed306a5fa27096f01 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 12 Dec 2025 11:19:17 -0800 Subject: [PATCH 44/66] feat(otel): add latency metrics (TTFT, TPOT, Total Generation Time) to OTEL logging (#17888) - Add time_to_first_token_histogram using api_call_start_time for accurate measurement - Add time_per_output_token_histogram for average time per output token - Add response_duration_histogram for total LLM API generation time - Extract latency metric recording into dedicated helper methods - Fix parent span double-ending bug when reused as primary span - Use api_call_start_time for TTFT to exclude LiteLLM overhead (matches Prometheus) - Support both streaming and non-streaming requests - Handle both datetime and float timestamp formats --- litellm/integrations/opentelemetry.py | 171 +++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 9f9d45d0e7d..c82a715ce4d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -248,6 +248,9 @@ class OpenTelemetry(CustomLogger): self._operation_duration_histogram = None self._token_usage_histogram = None self._cost_histogram = None + self._time_to_first_token_histogram = None + self._time_per_output_token_histogram = None + self._response_duration_histogram = None return from opentelemetry import metrics @@ -300,6 +303,21 @@ class OpenTelemetry(CustomLogger): description="GenAI request cost", unit="USD", ) + self._time_to_first_token_histogram = meter.create_histogram( + name="gen_ai.client.response.time_to_first_token", + description="Time to first token for streaming requests", + unit="s", + ) + self._time_per_output_token_histogram = meter.create_histogram( + name="gen_ai.client.response.time_per_output_token", + description="Average time per output token (generation time / completion tokens)", + unit="s", + ) + self._response_duration_histogram = meter.create_histogram( + name="gen_ai.client.response.duration", + description="Total LLM API generation time (excludes LiteLLM overhead)", + unit="s", + ) def _init_logs(self, logger_provider): # nothing to do if events disabled @@ -612,8 +630,9 @@ class OpenTelemetry(CustomLogger): if self.config.enable_events: self._emit_semantic_logs(kwargs, response_obj, span) - # 6. End parent span - if parent_span is not None: + # 6. End parent span (only if it wasn't reused as the primary span) + # If parent_span was reused as the primary span, it was already ended in _start_primary_span + if parent_span is not None and parent_span is not span: parent_span.end(end_time=self._to_ns(datetime.now())) def _start_primary_span( @@ -727,6 +746,152 @@ class OpenTelemetry(CustomLogger): if self._cost_histogram and cost: self._cost_histogram.record(cost, attributes=common_attrs) + # Record latency metrics (TTFT, TPOT, and Total Generation Time) + self._record_time_to_first_token_metric(kwargs, common_attrs) + self._record_time_per_output_token_metric( + kwargs, response_obj, end_time, duration_s, common_attrs + ) + self._record_response_duration_metric(kwargs, end_time, common_attrs) + + def _record_time_to_first_token_metric(self, kwargs: dict, common_attrs: dict): + """Record Time to First Token (TTFT) metric for streaming requests.""" + optional_params = kwargs.get("optional_params", {}) + is_streaming = optional_params.get("stream", False) + + if not (self._time_to_first_token_histogram and is_streaming): + return + + # Use api_call_start_time for precision (matches Prometheus implementation) + # This excludes LiteLLM overhead and measures pure LLM API latency + api_call_start_time = kwargs.get("api_call_start_time", None) + completion_start_time = kwargs.get("completion_start_time", None) + + if api_call_start_time is not None and completion_start_time is not None: + # Convert to timestamps if needed (handles both datetime and float) + if isinstance(api_call_start_time, datetime): + api_call_start_ts = api_call_start_time.timestamp() + else: + api_call_start_ts = api_call_start_time + + if isinstance(completion_start_time, datetime): + completion_start_ts = completion_start_time.timestamp() + else: + completion_start_ts = completion_start_time + + time_to_first_token_seconds = completion_start_ts - api_call_start_ts + self._time_to_first_token_histogram.record( + time_to_first_token_seconds, attributes=common_attrs + ) + + def _record_time_per_output_token_metric( + self, + kwargs: dict, + response_obj: Optional[Any], + end_time: datetime, + duration_s: float, + common_attrs: dict, + ): + """Record Time Per Output Token (TPOT) metric. + + Calculated as: generation_time / completion_tokens + - For streaming: uses end_time - completion_start_time (time to generate all tokens after first) + - For non-streaming: uses end_time - api_call_start_time (total generation time) + """ + if not self._time_per_output_token_histogram: + return + + # Get completion tokens from response_obj + completion_tokens = None + if response_obj and (usage := response_obj.get("usage")): + completion_tokens = usage.get("completion_tokens") + + if completion_tokens is None or completion_tokens <= 0: + return + + # Calculate generation time + completion_start_time = kwargs.get("completion_start_time", None) + api_call_start_time = kwargs.get("api_call_start_time", None) + + # Convert end_time to timestamp + if isinstance(end_time, datetime): + end_time_ts = end_time.timestamp() + else: + end_time_ts = end_time + + if completion_start_time is not None: + # Streaming: use completion_start_time (when first token arrived) + # This measures time to generate all tokens after the first one + if isinstance(completion_start_time, datetime): + completion_start_ts = completion_start_time.timestamp() + else: + completion_start_ts = completion_start_time + + generation_time_seconds = end_time_ts - completion_start_ts + elif api_call_start_time is not None: + # Non-streaming: use api_call_start_time (total generation time) + if isinstance(api_call_start_time, datetime): + api_call_start_ts = api_call_start_time.timestamp() + else: + api_call_start_ts = api_call_start_time + + generation_time_seconds = end_time_ts - api_call_start_ts + else: + # Fallback: use duration_s (already calculated as (end_time - start_time).total_seconds()) + generation_time_seconds = duration_s + + if generation_time_seconds > 0: + time_per_output_token_seconds = generation_time_seconds / completion_tokens + self._time_per_output_token_histogram.record( + time_per_output_token_seconds, attributes=common_attrs + ) + + def _record_response_duration_metric( + self, + kwargs: dict, + end_time: Union[datetime, float], + common_attrs: dict, + ): + """Record Total Generation Time (response duration) metric. + + Measures pure LLM API generation time: end_time - api_call_start_time + This excludes LiteLLM overhead and measures only the LLM provider's response time. + Works for both streaming and non-streaming requests. + + Mirrors Prometheus's litellm_llm_api_latency_metric. + Uses kwargs.get("end_time") with fallback to parameter for consistency with Prometheus. + """ + if not self._response_duration_histogram: + return + + api_call_start_time = kwargs.get("api_call_start_time", None) + if api_call_start_time is None: + return + + # Use end_time from kwargs if available (matches Prometheus), otherwise use parameter + # For streaming: end_time is when the stream completes (final chunk received) + # For non-streaming: end_time is when the response is received + _end_time = kwargs.get("end_time") or end_time + if _end_time is None: + _end_time = datetime.now() + + # Convert to timestamps if needed (handles both datetime and float) + if isinstance(api_call_start_time, datetime): + api_call_start_ts = api_call_start_time.timestamp() + else: + api_call_start_ts = api_call_start_time + + if isinstance(_end_time, datetime): + end_time_ts = _end_time.timestamp() + else: + end_time_ts = _end_time + + response_duration_seconds = end_time_ts - api_call_start_ts + + if response_duration_seconds > 0: + self._response_duration_histogram.record( + response_duration_seconds, attributes=common_attrs + ) + def _emit_semantic_logs(self, kwargs, response_obj, span: Span): if not self.config.enable_events: return @@ -1226,7 +1391,7 @@ class OpenTelemetry(CustomLogger): value=usage.get("prompt_tokens"), ) - ######################################################################## + ######################################################################## ########## LLM Request Medssages / tools / content Attributes ########### ######################################################################### From b635f92d90448aa72712c35132e647500452701a Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 12 Dec 2025 11:26:34 -0800 Subject: [PATCH 45/66] Add benchmark_proxy_vs_provider.py script to scripts directory with usage examples (#17889) --- scripts/benchmark_proxy_vs_provider.py | 774 +++++++++++++++++++++++++ 1 file changed, 774 insertions(+) create mode 100755 scripts/benchmark_proxy_vs_provider.py diff --git a/scripts/benchmark_proxy_vs_provider.py b/scripts/benchmark_proxy_vs_provider.py new file mode 100755 index 00000000000..94fd0ed00c7 --- /dev/null +++ b/scripts/benchmark_proxy_vs_provider.py @@ -0,0 +1,774 @@ +#!/usr/bin/env python3 +""" +Benchmark script comparing LiteLLM proxy vs direct provider endpoint. +Makes parallel calls to each endpoint and compares statistics including latency, throughput, and success rates. + +USAGE EXAMPLES: + +1. Basic Usage (Sequential, Recommended): + # Set required environment variables + export LITELLM_PROXY_URL='http://localhost:4000/chat/completions' + export PROVIDER_URL='https://api.openai.com/v1/chat/completions' + export LITELLM_PROXY_API_KEY='sk-1234' + export PROVIDER_API_KEY='sk-openai-key' + + # Run from scripts directory + cd scripts + python benchmark_proxy_vs_provider.py + +2. Multiple Runs for Statistical Accuracy: + python benchmark_proxy_vs_provider.py --runs 5 + # Averages results across 5 runs for more reliable metrics + +3. Realistic Load Testing with Concurrency Limit: + python benchmark_proxy_vs_provider.py --max-concurrent 100 --requests 2000 + # Limits to 100 concurrent requests (prevents overwhelming the server) + +4. Quick Test with Fewer Requests: + python benchmark_proxy_vs_provider.py --requests 100 + # Faster test with 100 requests instead of default 1000 + +5. Parallel Execution (Not Recommended): + python benchmark_proxy_vs_provider.py --parallel + # Runs both benchmarks simultaneously (may affect accuracy) + +6. Custom Timeout: + python benchmark_proxy_vs_provider.py --timeout 120 + # Sets request timeout to 120 seconds + +7. Combined Options: + python benchmark_proxy_vs_provider.py --runs 3 --requests 500 --max-concurrent 50 + # 3 runs, 500 requests each, max 50 concurrent + +REQUIRED ENVIRONMENT VARIABLES: + - LITELLM_PROXY_URL: Full URL to LiteLLM proxy chat completions endpoint + - PROVIDER_URL: Full URL to direct provider chat completions endpoint + +OPTIONAL ENVIRONMENT VARIABLES: + - LITELLM_PROXY_API_KEY: API key for LiteLLM proxy (if auth required) + - PROVIDER_API_KEY: API key for direct provider (if auth required) + +OUTPUT: + The script provides detailed statistics including: + - Success/error rates + - Latency metrics (mean, median, p95, p99) + - Throughput (requests per second) + - Comparison between proxy and provider performance + - Run-to-run variance (when using --runs > 1) +""" + +import asyncio +import aiohttp +import time +import json +import argparse +import os +from typing import List, Dict, Any, Optional +from dataclasses import dataclass, field +from statistics import mean, median, stdev +import sys +from aiohttp import TCPConnector + + +@dataclass +class RequestStats: + """Statistics for a single request""" + success: bool + latency: float + error: str = "" + status_code: int = 0 + + +@dataclass +class BenchmarkResults: + """Aggregated benchmark results""" + total_requests: int = 0 + successful_requests: int = 0 + failed_requests: int = 0 + latencies: List[float] = field(default_factory=list) + errors: List[str] = field(default_factory=list) + status_codes: Dict[int, int] = field(default_factory=dict) + total_time: float = 0.0 + + def calculate_stats(self) -> Dict[str, Any]: + """Calculate statistics from the results""" + if not self.latencies: + return { + "total_requests": self.total_requests, + "successful_requests": self.successful_requests, + "failed_requests": self.failed_requests, + "success_rate": 0.0, + "error_rate": 1.0, + "total_time": self.total_time, + "requests_per_second": 0.0, + "status_codes": self.status_codes, + "unique_errors": len(set(self.errors)) if self.errors else 0, + } + + return { + "total_requests": self.total_requests, + "successful_requests": self.successful_requests, + "failed_requests": self.failed_requests, + "success_rate": (self.successful_requests / self.total_requests) * 100, + "error_rate": (self.failed_requests / self.total_requests) * 100, + "total_time": self.total_time, + "requests_per_second": self.total_requests / self.total_time if self.total_time > 0 else 0, + "latency_stats": { + "mean": mean(self.latencies), + "median": median(self.latencies), + "min": min(self.latencies), + "max": max(self.latencies), + "std_dev": stdev(self.latencies) if len(self.latencies) > 1 else 0.0, + "p50": median(self.latencies), + "p95": self._percentile(self.latencies, 95), + "p99": self._percentile(self.latencies, 99), + }, + "status_codes": self.status_codes, + "unique_errors": len(set(self.errors)) if self.errors else 0, + } + + @staticmethod + def _percentile(data: List[float], percentile: int) -> float: + """Calculate percentile""" + sorted_data = sorted(data) + index = int(len(sorted_data) * (percentile / 100)) + if index >= len(sorted_data): + index = len(sorted_data) - 1 + return sorted_data[index] + + +async def make_request( + session: aiohttp.ClientSession, + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + timeout: aiohttp.ClientTimeout, +) -> RequestStats: + """Make a single async request and return stats""" + # Use time.perf_counter() for higher precision timing + start_time = time.perf_counter() + try: + async with session.post(url, json=payload, headers=headers, timeout=timeout) as response: + # Read response body to ensure complete transfer + response_body = await response.read() + latency = time.perf_counter() - start_time + status_code = response.status + + if response.status == 200: + # Validate response is valid JSON + try: + json.loads(response_body) + except json.JSONDecodeError: + return RequestStats( + success=False, + latency=latency, + error="Invalid JSON response", + status_code=status_code, + ) + + return RequestStats( + success=True, + latency=latency, + status_code=status_code, + ) + else: + error_text = response_body.decode('utf-8', errors='ignore')[:100] + return RequestStats( + success=False, + latency=latency, + error=f"HTTP {status_code}: {error_text}", + status_code=status_code, + ) + except asyncio.TimeoutError: + latency = time.perf_counter() - start_time + return RequestStats( + success=False, + latency=latency, + error="Timeout", + status_code=0, + ) + except Exception as e: + latency = time.perf_counter() - start_time + return RequestStats( + success=False, + latency=latency, + error=str(e)[:100], + status_code=0, + ) + + +async def warmup_endpoint( + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + num_warmup: int = 5, + timeout_seconds: int = 60, +) -> None: + """Perform warm-up requests to avoid cold start penalties""" + timeout = aiohttp.ClientTimeout(total=timeout_seconds) + connector = TCPConnector( + limit=100, # Max connections + limit_per_host=50, # Max connections per host + ttl_dns_cache=300, # DNS cache TTL + force_close=False, # Reuse connections + ) + + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [ + make_request(session, url, headers, payload, timeout) + for _ in range(num_warmup) + ] + await asyncio.gather(*tasks, return_exceptions=True) + + # Brief pause after warmup to let connections stabilize + await asyncio.sleep(0.5) + + +async def make_request_with_semaphore( + session: aiohttp.ClientSession, + semaphore: asyncio.Semaphore, + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + timeout: aiohttp.ClientTimeout, +) -> RequestStats: + """Make a request with semaphore-based concurrency control""" + async with semaphore: + return await make_request(session, url, headers, payload, timeout) + + +async def benchmark_endpoint( + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + num_requests: int = 1000, + timeout_seconds: int = 60, + warmup: bool = True, + max_concurrent: Optional[int] = None, +) -> BenchmarkResults: + """Benchmark an endpoint with parallel requests + + Args: + url: Endpoint URL to benchmark + headers: HTTP headers + payload: Request payload + num_requests: Total number of requests to make + timeout_seconds: Request timeout + warmup: Whether to perform warm-up requests + max_concurrent: Maximum concurrent requests (None = unlimited, all at once) + """ + print(f"\nStarting benchmark for {url}") + + if warmup: + print(f" Warming up with 5 requests...") + await warmup_endpoint(url, headers, payload, num_warmup=5, timeout_seconds=timeout_seconds) + + if max_concurrent: + print(f" Making {num_requests} requests with max {max_concurrent} concurrent...") + else: + print(f" Making {num_requests} requests in parallel (unlimited concurrency)...") + + results = BenchmarkResults(total_requests=num_requests) + timeout = aiohttp.ClientTimeout(total=timeout_seconds) + + # Set connector limits based on concurrency + if max_concurrent: + connector_limit = min(max_concurrent * 2, 200) # Allow some headroom + connector_limit_per_host = max_concurrent + else: + connector_limit = 200 + connector_limit_per_host = 100 + + # Use optimized connector for connection pooling and reuse + connector = TCPConnector( + limit=connector_limit, + limit_per_host=connector_limit_per_host, + ttl_dns_cache=300, # DNS cache TTL (5 minutes) + force_close=False, # Reuse connections for better performance + enable_cleanup_closed=True, # Clean up closed connections + ) + + # Use time.perf_counter() for higher precision + start_time = time.perf_counter() + + async with aiohttp.ClientSession(connector=connector) as session: + if max_concurrent: + # Use semaphore to limit concurrency + semaphore = asyncio.Semaphore(max_concurrent) + tasks = [ + make_request_with_semaphore(session, semaphore, url, headers, payload, timeout) + for _ in range(num_requests) + ] + else: + # Create all tasks at once for maximum parallelism + tasks = [ + make_request(session, url, headers, payload, timeout) + for _ in range(num_requests) + ] + + # Execute all requests (with concurrency limit if specified) + request_stats = await asyncio.gather(*tasks) + + results.total_time = time.perf_counter() - start_time + + # Aggregate results + for stats in request_stats: + if stats.success: + results.successful_requests += 1 + results.latencies.append(stats.latency) + else: + results.failed_requests += 1 + results.errors.append(stats.error) + + if stats.status_code > 0: + results.status_codes[stats.status_code] = results.status_codes.get(stats.status_code, 0) + 1 + + return results + + +def print_results(name: str, results: BenchmarkResults): + """Print formatted benchmark results""" + stats = results.calculate_stats() + + print(f"\n{'='*60}") + print(f"Results for {name}") + print(f"{'='*60}") + print(f"Total Requests: {stats['total_requests']}") + print(f"Successful Requests: {stats['successful_requests']}") + print(f"Failed Requests: {stats['failed_requests']}") + print(f"Success Rate: {stats['success_rate']:.2f}%") + print(f"Error Rate: {stats['error_rate']:.2f}%") + print(f"Total Time: {stats['total_time']:.2f}s") + print(f"Requests/Second: {stats['requests_per_second']:.2f}") + + if 'latency_stats' in stats: + latency = stats['latency_stats'] + print(f"\nLatency Statistics (seconds):") + print(f" Mean: {latency['mean']:.4f}s") + print(f" Median (p50): {latency['median']:.4f}s") + print(f" Min: {latency['min']:.4f}s") + print(f" Max: {latency['max']:.4f}s") + print(f" Std Dev: {latency['std_dev']:.4f}s") + print(f" p95: {latency['p95']:.4f}s") + print(f" p99: {latency['p99']:.4f}s") + + if stats['status_codes']: + print(f"\nStatus Codes:") + for code, count in sorted(stats['status_codes'].items()): + print(f" {code}: {count}") + + if results.errors: + print(f"\nErrors (showing first 5 unique):") + unique_errors = list(set(results.errors))[:5] + for error in unique_errors: + count = results.errors.count(error) + print(f" [{count}x] {error}") + + +def aggregate_results(results_list: List[BenchmarkResults]) -> BenchmarkResults: + """Aggregate results from multiple runs""" + if not results_list: + return BenchmarkResults() + + aggregated = BenchmarkResults() + + # Aggregate all latencies + all_latencies = [] + all_errors = [] + total_requests = 0 + total_successful = 0 + total_failed = 0 + total_time_sum = 0.0 + status_codes_combined = {} + + for result in results_list: + all_latencies.extend(result.latencies) + all_errors.extend(result.errors) + total_requests += result.total_requests + total_successful += result.successful_requests + total_failed += result.failed_requests + total_time_sum += result.total_time + + for code, count in result.status_codes.items(): + status_codes_combined[code] = status_codes_combined.get(code, 0) + count + + aggregated.latencies = all_latencies + aggregated.errors = all_errors + aggregated.total_requests = total_requests + aggregated.successful_requests = total_successful + aggregated.failed_requests = total_failed + aggregated.total_time = total_time_sum / len(results_list) # Average time + aggregated.status_codes = status_codes_combined + + return aggregated + + +def print_run_variance(name: str, results_list: List[BenchmarkResults]): + """Print variance statistics across multiple runs""" + if len(results_list) <= 1: + return + + print(f"\n{'='*60}") + print(f"Run-to-Run Variance: {name}") + print(f"{'='*60}") + + # Collect mean latencies from each run + mean_latencies = [] + throughputs = [] + + for result in results_list: + stats = result.calculate_stats() + if 'latency_stats' in stats: + mean_latencies.append(stats['latency_stats']['mean']) + throughputs.append(stats['requests_per_second']) + + if mean_latencies: + print(f"\nMean Latency Variance:") + print(f" Runs: {len(mean_latencies)}") + print(f" Mean: {mean(mean_latencies):.4f}s") + print(f" Min: {min(mean_latencies):.4f}s") + print(f" Max: {max(mean_latencies):.4f}s") + print(f" Std Dev: {stdev(mean_latencies):.4f}s" if len(mean_latencies) > 1 else " Std Dev: N/A") + print(f" Coefficient of Variation: {(stdev(mean_latencies) / mean(mean_latencies) * 100):.2f}%" if len(mean_latencies) > 1 else " Coefficient of Variation: N/A") + + if throughputs: + print(f"\nThroughput Variance:") + print(f" Mean: {mean(throughputs):.2f} req/s") + print(f" Min: {min(throughputs):.2f} req/s") + print(f" Max: {max(throughputs):.2f} req/s") + print(f" Std Dev: {stdev(throughputs):.2f} req/s" if len(throughputs) > 1 else " Std Dev: N/A") + + +def compare_results(proxy_results: BenchmarkResults, provider_results: BenchmarkResults): + """Compare and print differences between proxy and provider results""" + proxy_stats = proxy_results.calculate_stats() + provider_stats = provider_results.calculate_stats() + + print(f"\n{'='*60}") + print(f"Comparison: LiteLLM Proxy vs Direct Provider") + print(f"{'='*60}") + + # Success Rate Comparison + print(f"\nSuccess Rate:") + print(f" Proxy: {proxy_stats['success_rate']:.2f}%") + print(f" Provider: {provider_stats['success_rate']:.2f}%") + diff = proxy_stats['success_rate'] - provider_stats['success_rate'] + print(f" Difference: {diff:+.2f}%") + + # Throughput Comparison + print(f"\nThroughput (requests/second):") + print(f" Proxy: {proxy_stats['requests_per_second']:.2f}") + print(f" Provider: {provider_stats['requests_per_second']:.2f}") + diff = proxy_stats['requests_per_second'] - provider_stats['requests_per_second'] + print(f" Difference: {diff:+.2f} req/s") + + # Latency Comparison + if 'latency_stats' in proxy_stats and 'latency_stats' in provider_stats: + print(f"\nLatency Comparison (seconds):") + proxy_latency = proxy_stats['latency_stats'] + provider_latency = provider_stats['latency_stats'] + + metrics = ['mean', 'median', 'p95', 'p99'] + for metric in metrics: + proxy_val = proxy_latency[metric] + provider_val = provider_latency[metric] + diff = proxy_val - provider_val + diff_pct = (diff / provider_val * 100) if provider_val > 0 else 0 + print(f" {metric.upper():8s}: Proxy={proxy_val:.4f}s, Provider={provider_val:.4f}s, Diff={diff:+.4f}s ({diff_pct:+.2f}%)") + + # Total Time Comparison + print(f"\nTotal Time:") + print(f" Proxy: {proxy_stats['total_time']:.2f}s") + print(f" Provider: {provider_stats['total_time']:.2f}s") + diff = proxy_stats['total_time'] - provider_stats['total_time'] + diff_pct = (diff / provider_stats['total_time'] * 100) if provider_stats['total_time'] > 0 else 0 + print(f" Difference: {diff:+.2f}s ({diff_pct:+.2f}%)") + + +async def main(): + """Main benchmark function""" + parser = argparse.ArgumentParser( + description="Benchmark LiteLLM proxy vs direct provider endpoint", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Environment Variables (required): + LITELLM_PROXY_URL - URL of the LiteLLM proxy endpoint (e.g., http://localhost:4000/chat/completions) + PROVIDER_URL - URL of the direct provider endpoint (e.g., https://api.openai.com/v1/chat/completions) + LITELLM_PROXY_API_KEY - API key for LiteLLM proxy (optional, but may be required) + PROVIDER_API_KEY - API key for direct provider (optional, but may be required) + +Examples: + # 1. Basic usage (recommended - sequential execution) + export LITELLM_PROXY_URL='http://localhost:4000/chat/completions' + export PROVIDER_URL='https://api.openai.com/v1/chat/completions' + export LITELLM_PROXY_API_KEY='sk-1234' + export PROVIDER_API_KEY='sk-openai-key' + python scripts/benchmark_proxy_vs_provider.py + + # 2. Multiple runs for statistical accuracy (recommended) + python scripts/benchmark_proxy_vs_provider.py --runs 5 + + # 3. Realistic load testing with concurrency limit + python scripts/benchmark_proxy_vs_provider.py --max-concurrent 100 --requests 2000 + + # 4. Quick test with fewer requests + python scripts/benchmark_proxy_vs_provider.py --requests 100 + + # 5. Parallel execution (not recommended - may affect accuracy) + python scripts/benchmark_proxy_vs_provider.py --parallel + + # 6. Custom timeout for slower endpoints + python scripts/benchmark_proxy_vs_provider.py --timeout 120 + + # 7. Combined options for comprehensive testing + python scripts/benchmark_proxy_vs_provider.py --runs 3 --requests 500 --max-concurrent 50 + + # 8. Skip warmup (not recommended - may affect first request accuracy) + python scripts/benchmark_proxy_vs_provider.py --no-warmup + """ + ) + parser.add_argument( + "--parallel", + action="store_true", + help="Run both benchmarks in parallel (default: sequential to avoid interference)", + ) + parser.add_argument( + "--requests", + type=int, + default=1000, + help="Number of requests per endpoint (default: 1000)", + ) + parser.add_argument( + "--timeout", + type=int, + default=60, + help="Request timeout in seconds (default: 60)", + ) + parser.add_argument( + "--runs", + type=int, + default=1, + help="Number of benchmark runs to average (default: 1, recommended: 3-5 for accuracy)", + ) + parser.add_argument( + "--no-warmup", + action="store_true", + help="Skip warm-up requests (not recommended)", + ) + parser.add_argument( + "--max-concurrent", + type=int, + default=None, + help="Maximum concurrent requests (default: unlimited - all at once). " + "Useful for realistic load testing (e.g., --max-concurrent 100)", + ) + + args = parser.parse_args() + + # Configuration from environment variables + LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL") + PROVIDER_URL = os.getenv("PROVIDER_URL") + LITELLM_PROXY_API_KEY = os.getenv("LITELLM_PROXY_API_KEY", "") + PROVIDER_API_KEY = os.getenv("PROVIDER_API_KEY", "") + + # Validate required environment variables + if not LITELLM_PROXY_URL: + print("Error: LITELLM_PROXY_URL environment variable is required") + print(" Example: export LITELLM_PROXY_URL='https://your-proxy.com/chat/completions'") + sys.exit(1) + + if not PROVIDER_URL: + print("Error: PROVIDER_URL environment variable is required") + print(" Example: export PROVIDER_URL='https://your-provider.com/v1/chat/completions'") + sys.exit(1) + + # Headers for LiteLLM proxy + proxy_headers = { + "Content-Type": "application/json", + } + if LITELLM_PROXY_API_KEY: + proxy_headers["Authorization"] = f"Bearer {LITELLM_PROXY_API_KEY}" + else: + print("Warning: LITELLM_PROXY_API_KEY not set, requests may fail if authentication is required") + + # Headers for direct provider + provider_headers = { + "Content-Type": "application/json", + } + if PROVIDER_API_KEY: + provider_headers["Authorization"] = f"Bearer {PROVIDER_API_KEY}" + else: + print("Warning: PROVIDER_API_KEY not set, requests may fail if authentication is required") + + # Payload (same for both) + payload = { + "model": "db-openai-endpoint", # For proxy + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "max_tokens": 100, + "user": "new_user" + } + + # For direct provider, might need different model name + provider_payload = payload.copy() + # provider_payload["model"] = "gpt-3.5-turbo" # Uncomment if needed + + num_requests = args.requests + timeout_seconds = args.timeout + + print("="*60) + print("LiteLLM Proxy vs Provider Benchmark") + print("="*60) + print(f"Configuration (from environment variables):") + print(f" Proxy URL: {LITELLM_PROXY_URL}") + print(f" Provider URL: {PROVIDER_URL}") + print(f" Proxy API Key: {'Set' if LITELLM_PROXY_API_KEY else 'Not set (may cause auth errors)'}") + print(f" Provider API Key: {'Set' if PROVIDER_API_KEY else 'Not set (may cause auth errors)'}") + print(f" Requests: {num_requests}") + print(f" Runs: {args.runs}") + print(f" Max Concurrent: {args.max_concurrent if args.max_concurrent else 'Unlimited (all at once)'}") + print(f" Timeout: {timeout_seconds}s") + print(f" Warmup: {'Enabled' if not args.no_warmup else 'Disabled (not recommended)'}") + print(f" Mode: {'Parallel (may affect results)' if args.parallel else 'Sequential (recommended)'}") + + if not args.max_concurrent: + print(f"\nTip: Use --max-concurrent 100 for more realistic load testing") + print(f" (prevents overwhelming the server with all requests at once)") + + if args.parallel: + print(f"\nWARNING: Running benchmarks in parallel may affect results due to:") + print(f" - Shared network bandwidth") + print(f" - Provider endpoint receiving double load (via proxy + direct)") + print(f" - Potential rate limiting issues") + print(f" - Resource contention") + + # Run benchmarks multiple times if requested + all_proxy_results = [] + all_provider_results = [] + + warmup_enabled = not args.no_warmup + + if args.runs > 1: + print(f"\nRunning {args.runs} benchmark runs for statistical accuracy...") + print(f" Results will be averaged across all runs.\n") + + overall_start_time = time.perf_counter() + + # Initialize to satisfy type checker (will always be set in loop) + proxy_results: Optional[BenchmarkResults] = None + provider_results: Optional[BenchmarkResults] = None + + for run_num in range(1, args.runs + 1): + if args.runs > 1: + print(f"\n{'='*60}") + print(f"Run {run_num}/{args.runs}") + print(f"{'='*60}") + + if args.parallel: + print(f"\nRunning both benchmarks in parallel...") + proxy_results, provider_results = await asyncio.gather( + benchmark_endpoint( + LITELLM_PROXY_URL, + proxy_headers, + payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ), + benchmark_endpoint( + PROVIDER_URL, + provider_headers, + provider_payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ), + ) + else: + print(f"\nRunning benchmarks sequentially (proxy first, then provider)...") + if run_num == 1: + print(f" This ensures accurate results without interference.\n") + + proxy_results = await benchmark_endpoint( + LITELLM_PROXY_URL, + proxy_headers, + payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ) + + if run_num < args.runs or args.runs == 1: + print(f"\nWaiting 3 seconds before starting provider benchmark...") + await asyncio.sleep(3) # Longer pause to ensure clean separation + + provider_results = await benchmark_endpoint( + PROVIDER_URL, + provider_headers, + provider_payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ) + + all_proxy_results.append(proxy_results) + all_provider_results.append(provider_results) + + # Brief pause between runs + if run_num < args.runs: + print(f"\nWaiting 5 seconds before next run...") + await asyncio.sleep(5) + + overall_benchmark_time = time.perf_counter() - overall_start_time + print(f"\nAll benchmark runs completed in {overall_benchmark_time:.2f}s") + + # Aggregate results across multiple runs + if args.runs > 1: + final_proxy_results = aggregate_results(all_proxy_results) + final_provider_results = aggregate_results(all_provider_results) + print(f"\nAggregated results across {args.runs} runs:") + else: + # Use results from single run + if proxy_results is None or provider_results is None: + raise RuntimeError("Benchmark results not initialized") + final_proxy_results = proxy_results + final_provider_results = provider_results + print(f"\nResults:") + + # Print individual results + print_results("LiteLLM Proxy", final_proxy_results) + print_results("Direct Provider", final_provider_results) + + # Print comparison + compare_results(final_proxy_results, final_provider_results) + + # Show run-to-run variance if multiple runs + if args.runs > 1: + print_run_variance("LiteLLM Proxy", all_proxy_results) + print_run_variance("Direct Provider", all_provider_results) + + print(f"\n{'='*60}") + print("Benchmark complete!") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\nBenchmark interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nError running benchmark: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + From 700a1bb574f2d47ee69dc22955e7e49efd6969b3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 11:39:02 -0800 Subject: [PATCH 46/66] [Feat] UI - show UI version on top left near logo (#17891) * left nav * fix order * fix * add ui version on ui * fix link --- .../src/components/leftnav.tsx | 519 ++++++++++-------- .../src/components/navbar.tsx | 35 +- 2 files changed, 334 insertions(+), 220 deletions(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 2ddea2d1e32..b2e13bd522f 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -21,7 +21,8 @@ import { ToolOutlined, UserOutlined, } from "@ant-design/icons"; -import { Badge, ConfigProvider, Layout, Menu } from "antd"; +import { ConfigProvider, Layout, Menu } from "antd"; +import type { MenuProps } from "antd"; import { all_admin_roles, internalUserRoles, isAdminRole, rolesWithWriteAccess } from "../utils/roles"; import UsageIndicator from "./usage_indicator"; const { Sider } = Layout; @@ -35,237 +36,336 @@ interface SidebarProps { collapsed?: boolean; } -// Create a more comprehensive menu item configuration +// Menu item configuration interface MenuItem { key: string; page: string; label: string | React.ReactNode; roles?: string[]; - children?: MenuItem[]; // Add children property for submenus + children?: MenuItem[]; icon?: React.ReactNode; } +// Group configuration +interface MenuGroup { + groupLabel: string; + items: MenuItem[]; + roles?: string[]; +} + const Sidebar: React.FC = ({ accessToken, setPage, userRole, defaultSelectedKey, collapsed = false }) => { - // Note: If a menu item does not have a role, it is visible to all roles. - const menuItems: MenuItem[] = [ + // Navigate to page helper + const navigateToPage = (page: string) => { + const newSearchParams = new URLSearchParams(window.location.search); + newSearchParams.set("page", page); + window.history.pushState(null, "", `?${newSearchParams.toString()}`); + setPage(page); + }; + + // Menu groups organized by category + const menuGroups: MenuGroup[] = [ { - key: "api-keys", - page: "api-keys", - label: "Virtual Keys", - icon: , - }, - { - key: "llm-playground", - page: "llm-playground", - label: "Playground", - icon: , - roles: rolesWithWriteAccess, - }, - { - key: "models", - page: "models", - label: "Models + Endpoints", - icon: , - roles: rolesWithWriteAccess, - }, - { - key: "new_usage", - page: "new_usage", - label: ( - - Usage - - ), - icon: , - roles: [...all_admin_roles, ...internalUserRoles], - }, - { key: "teams", page: "teams", label: "Teams", icon: }, - { - key: "organizations", - page: "organizations", - label: "Organizations", - icon: , - roles: all_admin_roles, - }, - { - key: "users", - page: "users", - label: "Internal Users", - icon: , - roles: all_admin_roles, - }, - { - key: "budgets", - page: "budgets", - label: "Budgets", - icon: , - roles: all_admin_roles, - }, - { key: "api_ref", page: "api_ref", label: "API Reference", icon: }, - { - key: "model-hub-table", - page: "model-hub-table", - label: "AI Hub", - icon: , - }, - { key: "logs", page: "logs", label: "Logs", icon: }, - { - key: "guardrails", - page: "guardrails", - label: "Guardrails", - icon: , - roles: all_admin_roles, - }, - { - key: "mcp-servers", - page: "mcp-servers", - label: "MCP Servers", - icon: , - }, - { - key: "tools", - page: "tools", - label: "Tools", - icon: , - children: [ + groupLabel: "AI GATEWAY", + items: [ { - key: "search-tools", - page: "search-tools", - label: "Search Tools", - icon: , + key: "api-keys", + page: "api-keys", + label: "Virtual Keys", + icon: , }, { - key: "vector-stores", - page: "vector-stores", - label: "Vector Stores", - icon: , - roles: all_admin_roles, + key: "llm-playground", + page: "llm-playground", + label: "Playground", + icon: , + roles: rolesWithWriteAccess, }, - ], - }, - { - key: "experimental", - page: "experimental", - label: "Experimental", - icon: , - children: [ { - key: "caching", - page: "caching", - label: "Caching", - icon: , - roles: all_admin_roles, + key: "models", + page: "models", + label: "Models + Endpoints", + icon: , + roles: rolesWithWriteAccess, }, { key: "agents", page: "agents", label: "Agents", - icon: , + icon: , roles: rolesWithWriteAccess, }, { - key: "prompts", - page: "prompts", - label: "Prompts", - icon: , + key: "mcp-servers", + page: "mcp-servers", + label: "MCP Servers", + icon: , + }, + { + key: "guardrails", + page: "guardrails", + label: "Guardrails", + icon: , roles: all_admin_roles, }, { - key: "transform-request", - page: "transform-request", - label: "API Playground", - icon: , - roles: [...all_admin_roles, ...internalUserRoles], + key: "tools", + page: "tools", + label: "Tools", + icon: , + children: [ + { + key: "search-tools", + page: "search-tools", + label: "Search Tools", + icon: , + }, + { + key: "vector-stores", + page: "vector-stores", + label: "Vector Stores", + icon: , + roles: all_admin_roles, + }, + ], }, - { - key: "tag-management", - page: "tag-management", - label: "Tag Management", - icon: , - roles: all_admin_roles, - }, - { key: "4", page: "usage", label: "Old Usage", icon: }, ], }, { - key: "settings", - page: "settings", - label: "Settings", - icon: , + groupLabel: "OBSERVABILITY", + items: [ + { + key: "new_usage", + page: "new_usage", + label: "Usage", + icon: , + roles: [...all_admin_roles, ...internalUserRoles], + }, + { + key: "logs", + page: "logs", + label: "Logs", + icon: , + }, + ], + }, + { + groupLabel: "ACCESS CONTROL", + items: [ + { + key: "users", + page: "users", + label: "Internal Users", + icon: , + roles: all_admin_roles, + }, + { + key: "teams", + page: "teams", + label: "Teams", + icon: , + }, + { + key: "organizations", + page: "organizations", + label: "Organizations", + icon: , + roles: all_admin_roles, + }, + { + key: "budgets", + page: "budgets", + label: "Budgets", + icon: , + roles: all_admin_roles, + }, + ], + }, + { + groupLabel: "DEVELOPER TOOLS", + items: [ + { + key: "api_ref", + page: "api_ref", + label: "API Reference", + icon: , + }, + { + key: "model-hub-table", + page: "model-hub-table", + label: "AI Hub", + icon: , + }, + { + key: "experimental", + page: "experimental", + label: "Experimental", + icon: , + children: [ + { + key: "caching", + page: "caching", + label: "Caching", + icon: , + roles: all_admin_roles, + }, + { + key: "prompts", + page: "prompts", + label: "Prompts", + icon: , + roles: all_admin_roles, + }, + { + key: "transform-request", + page: "transform-request", + label: "API Playground", + icon: , + roles: [...all_admin_roles, ...internalUserRoles], + }, + { + key: "tag-management", + page: "tag-management", + label: "Tag Management", + icon: , + roles: all_admin_roles, + }, + { + key: "4", + page: "usage", + label: "Old Usage", + icon: , + }, + ], + }, + ], + }, + { + groupLabel: "SETTINGS", roles: all_admin_roles, - children: [ + items: [ { - key: "router-settings", - page: "router-settings", - label: "Router Settings", - icon: , - roles: all_admin_roles, - }, - { - key: "logging-and-alerts", - page: "logging-and-alerts", - label: "Logging & Alerts", - icon: , - roles: all_admin_roles, - }, - { - key: "admin-panel", - page: "admin-panel", - label: "Admin Settings", - icon: , - roles: all_admin_roles, - }, - { - key: "cost-tracking", - page: "cost-tracking", - label: "Cost Tracking", - icon: , - roles: all_admin_roles, - }, - { - key: "ui-theme", - page: "ui-theme", - label: "UI Theme", - icon: , + key: "settings", + page: "settings", + label: "Settings", + icon: , roles: all_admin_roles, + children: [ + { + key: "router-settings", + page: "router-settings", + label: "Router Settings", + icon: , + roles: all_admin_roles, + }, + { + key: "logging-and-alerts", + page: "logging-and-alerts", + label: "Logging & Alerts", + icon: , + roles: all_admin_roles, + }, + { + key: "admin-panel", + page: "admin-panel", + label: "Admin Settings", + icon: , + roles: all_admin_roles, + }, + { + key: "cost-tracking", + page: "cost-tracking", + label: "Cost Tracking", + icon: , + roles: all_admin_roles, + }, + { + key: "ui-theme", + page: "ui-theme", + label: "UI Theme", + icon: , + roles: all_admin_roles, + }, + ], }, ], }, ]; - // Find the menu item that matches the default page, including in submenus - const findMenuItemKey = (page: string): string => { - // Check top-level items - const topLevelItem = menuItems.find((item) => item.page === page); - if (topLevelItem) return topLevelItem.key; - // Check submenu items - for (const item of menuItems) { - if (item.children) { - const childItem = item.children.find((child) => child.page === page); - if (childItem) return childItem.key; + // Filter items based on user role + const filterItemsByRole = (items: MenuItem[]): MenuItem[] => { + return items + .filter((item) => !item.roles || item.roles.includes(userRole)) + .map((item) => ({ + ...item, + children: item.children ? filterItemsByRole(item.children) : undefined, + })); + }; + + // Build menu items with groups + const buildMenuItems = (): MenuProps["items"] => { + const items: MenuProps["items"] = []; + + menuGroups.forEach((group) => { + // Check if group has role restriction + if (group.roles && !group.roles.includes(userRole)) { + return; + } + + const filteredItems = filterItemsByRole(group.items); + if (filteredItems.length === 0) return; + + // Add group with items + items.push({ + type: "group", + label: collapsed ? null : ( + + {group.groupLabel} + + ), + children: filteredItems.map((item) => ({ + key: item.key, + icon: item.icon, + label: item.label, + children: item.children?.map((child) => ({ + key: child.key, + icon: child.icon, + label: child.label, + onClick: () => navigateToPage(child.page), + })), + onClick: !item.children ? () => navigateToPage(item.page) : undefined, + })), + }); + }); + + return items; + }; + + // Find selected menu key + const findMenuItemKey = (page: string): string => { + for (const group of menuGroups) { + for (const item of group.items) { + if (item.page === page) return item.key; + if (item.children) { + const child = item.children.find((c) => c.page === page); + if (child) return child.key; + } } } - return "1"; // Default to first item if not found + return "api-keys"; }; const selectedMenuKey = findMenuItemKey(defaultSelectedKey); - const filteredMenuItems = menuItems.filter((item) => { - // Check if parent item has roles and user has access - const hasParentAccess = !item.roles || item.roles.includes(userRole); - - console.log(`Menu item ${item.label}: roles=${item.roles}, userRole=${userRole}, hasAccess=${hasParentAccess}`); - - if (!hasParentAccess) return false; - - // Filter children if they exist - if (item.children) { - item.children = item.children.filter((child) => !child.roles || child.roles.includes(userRole)); - } - - return true; - }); - return ( = ({ accessToken, setPage, userRole, defau collapsible trigger={null} style={{ - transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)", // Material Design easing + transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)", position: "relative", }} > @@ -284,8 +384,15 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau theme={{ components: { Menu: { - iconSize: 18, - fontSize: 14, + iconSize: 15, + fontSize: 13, + itemMarginInline: 4, + itemPaddingInline: 8, + itemHeight: 30, + itemBorderRadius: 6, + subMenuItemBorderRadius: 6, + groupTitleFontSize: 10, + groupTitleLineHeight: 1.5, }, }, }} @@ -293,38 +400,16 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau ({ - key: item.key, - icon: item.icon, - label: item.label, - children: item.children?.map((child) => ({ - key: child.key, - icon: child.icon, - label: child.label, - onClick: () => { - const newSearchParams = new URLSearchParams(window.location.search); - newSearchParams.set("page", child.page); - window.history.pushState(null, "", `?${newSearchParams.toString()}`); - setPage(child.page); - }, - })), - onClick: !item.children - ? () => { - const newSearchParams = new URLSearchParams(window.location.search); - newSearchParams.set("page", item.page); - window.history.pushState(null, "", `?${newSearchParams.toString()}`); - setPage(item.page); - } - : undefined, - }))} + items={buildMenuItems()} /> {isAdminRole(userRole) && !collapsed && } diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 9113a0f54c0..2e0edcfcd86 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -43,11 +43,28 @@ const Navbar: React.FC = ({ }) => { const baseUrl = getProxyBaseUrl(); const [logoutUrl, setLogoutUrl] = useState(""); + const [version, setVersion] = useState(""); const { logoUrl } = useTheme(); // Simple logo URL: use custom logo if available, otherwise default const imageUrl = logoUrl || `${baseUrl}/get_image`; + useEffect(() => { + const fetchVersion = async () => { + try { + const response = await fetch(`${baseUrl}/health/readiness`); + const data = await response.json(); + if (data.litellm_version) { + setVersion(data.litellm_version); + } + } catch (error) { + console.error("Failed to fetch version:", error); + } + }; + + fetchVersion(); + }, [baseUrl]); + useEffect(() => { const initializeProxySettings = async () => { if (accessToken) { @@ -146,9 +163,21 @@ const Navbar: React.FC = ({ )} - - LiteLLM Brand - +
+ + LiteLLM Brand + + {version && ( + + v{version} + + )} +
{/* Right side nav items */}
From 1531b58493fdb13045f51c8e57f09363b280bb6b Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:40:35 -0300 Subject: [PATCH 47/66] feat(openai): add reasoning_effort='xhigh' support for gpt-5.2 models (#17875) Add support for the 'xhigh' reasoning effort level on all gpt-5.2 model variants, not just gpt-5.2-pro. This enables deeper reasoning capabilities for the base gpt-5.2 model. Changes: - Add is_model_gpt_5_2_model() method to detect gpt-5.2 variants - Update xhigh validation to allow gpt-5.2 models - Update documentation with gpt-5.2 reasoning_effort support - Update tests to reflect new behavior --- docs/my-website/docs/providers/openai.md | 6 ++++-- .../llms/openai/chat/gpt_5_transformation.py | 10 ++++++++-- .../llms/openai/test_gpt5_transformation.py | 17 +++++++++-------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index b170c6aba22..509a106d8a4 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -433,7 +433,7 @@ Expected Response: ### Advanced: Using `reasoning_effort` with `summary` field -By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`—`"xhigh"` is only supported on `gpt-5.1-codex-max`) and only sets the effort level without including a reasoning summary. +By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`—`"xhigh"` is only supported on `gpt-5.1-codex-max` and `gpt-5.2` models) and only sets the effort level without including a reasoning summary. To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI. @@ -501,11 +501,13 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ | `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5.1-codex-mini` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5.1-codex-max` | `adaptive` | `low`, `medium`, `high`, `xhigh` (no `minimal`) | +| `gpt-5.2` | `medium` | `none`, `low`, `medium`, `high`, `xhigh` | +| `gpt-5.2-pro` | `high` | `low`, `medium`, `high`, `xhigh` | | `gpt-5-pro` | `high` | `high` only | **Note:** - GPT-5.1 introduced a new `reasoning_effort="none"` setting for faster, lower-latency responses. This replaces the `"minimal"` setting from GPT-5. -- `gpt-5.1-codex-max` is the only model that supports `reasoning_effort="xhigh"`. All other models will reject this value. +- `gpt-5.1-codex-max` and `gpt-5.2` models support `reasoning_effort="xhigh"`. All other models will reject this value. - `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. - When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column. diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 1b3abb20d63..3fffa335fdc 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -51,6 +51,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.2-pro") + @classmethod + def is_model_gpt_5_2_model(cls, model: str) -> bool: + """Check if the model is a gpt-5.2 variant (including pro).""" + model_name = model.split("/")[-1] + return model_name.startswith("gpt-5.2") + def get_supported_openai_params(self, model: str) -> list: from litellm.utils import supports_tool_choice @@ -89,14 +95,14 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if reasoning_effort is not None and reasoning_effort == "xhigh": if not ( self.is_model_gpt_5_1_codex_max_model(model) - or self.is_model_gpt_5_2_pro_model(model) + or self.is_model_gpt_5_2_model(model) ): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) else: raise litellm.utils.UnsupportedParamsError( message=( - "reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max." + "reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max and gpt-5.2 models." ), status_code=400, ) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 4cb3132f737..fd25d302d07 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -388,11 +388,12 @@ def test_gpt5_2_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" -def test_gpt5_2_rejects_reasoning_effort_xhigh_for_base_model(config: OpenAIConfig): - with pytest.raises(litellm.utils.UnsupportedParamsError): - config.map_openai_params( - non_default_params={"reasoning_effort": "xhigh"}, - optional_params={}, - model="gpt-5.2", - drop_params=False, - ) +def test_gpt5_2_allows_reasoning_effort_xhigh(config: OpenAIConfig): + """Test that gpt-5.2 (base model) also supports reasoning_effort='xhigh'.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="gpt-5.2", + drop_params=False, + ) + assert params["reasoning_effort"] == "xhigh" From b651012fdc993f60095e69d3fe849e351edb488e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 12:00:23 -0800 Subject: [PATCH 48/66] [fix] UI playground - allow custom model name as option[0] (#17892) * fix: move custom model to top * fix test --- .../playground/chat_ui/ChatUI.test.tsx | 36 +++++++++++++++++++ .../components/playground/chat_ui/ChatUI.tsx | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx index 7f970128465..2de324499e1 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx @@ -182,4 +182,40 @@ describe("ChatUI", () => { expect(screen.queryByText("ResponsesModel")).toBeNull(); }); }); + + /** + * Tests that the 'Enter custom model' option is available in the model selector dropdown. + * This ensures users can manually enter a model name if it's not in the list. + */ + it("should show 'Enter custom model' option in model selector", async () => { + const { getByText } = render( + , + ); + + // Wait for the component to render + await waitFor(() => { + expect(getByText("Test Key")).toBeInTheDocument(); + }); + + // Open the "Select Model" dropdown + const selectModelLabel = getByText("Select Model"); + const modelSelectContainer = selectModelLabel.closest("div"); + const modelSelect = modelSelectContainer?.querySelector(".ant-select-selector"); + + fireEvent.mouseDown(modelSelect!); + + await waitFor(() => { + // Get all options in the dropdown (Ant Design renders these in a portal) + const options = document.querySelectorAll(".ant-select-item-option-content"); + expect(options.length).toBeGreaterThan(0); + // Check if the first option is 'Enter custom model' + expect(options[0]).toHaveTextContent("Enter custom model"); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 31cb87611ad..bc3af2cfc73 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -1208,6 +1208,7 @@ const ChatUI: React.FC = ({ placeholder="Select a Model" onChange={onModelChange} options={[ + { value: "custom", label: "Enter custom model", key: "custom" }, ...Array.from( new Set( modelInfo @@ -1237,7 +1238,6 @@ const ChatUI: React.FC = ({ label: model_group, key: index, })), - { value: "custom", label: "Enter custom model", key: "custom" }, ]} style={{ width: "100%" }} showSearch={true} From 92d71a3ed4f21fc0b63f69f706e8ddb734e97312 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 12:10:13 -0800 Subject: [PATCH 49/66] [UI] - Re organize leftnav to have correct categories + get agents on root (#17890) * left nav * fix order * fix * fix bad new usage * fix new --- ui/litellm-dashboard/src/components/leftnav.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index b2e13bd522f..ec000f7582e 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -21,7 +21,7 @@ import { ToolOutlined, UserOutlined, } from "@ant-design/icons"; -import { ConfigProvider, Layout, Menu } from "antd"; +import { Badge, ConfigProvider, Layout, Menu } from "antd"; import type { MenuProps } from "antd"; import { all_admin_roles, internalUserRoles, isAdminRole, rolesWithWriteAccess } from "../utils/roles"; import UsageIndicator from "./usage_indicator"; @@ -90,7 +90,11 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau { key: "agents", page: "agents", - label: "Agents", + label: ( + + Agents + + ), icon: , roles: rolesWithWriteAccess, }, @@ -136,9 +140,13 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau { key: "new_usage", page: "new_usage", - label: "Usage", icon: , roles: [...all_admin_roles, ...internalUserRoles], + label: ( + + Usage + + ), }, { key: "logs", From 5b0bf86ee50de8b87d5c81585bbbf040db8f33c6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 12:13:51 -0800 Subject: [PATCH 50/66] Usage Entity labels --- .../EntityUsage/EntityUsage.test.tsx | 2 +- .../components/EntityUsage/EntityUsage.tsx | 18 ++++++++++++++++-- .../src/utils/dataUtils.test.ts | 16 ++++++++++++++-- ui/litellm-dashboard/src/utils/dataUtils.ts | 5 +++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx index 96d8d25407e..2ace90e0122 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx @@ -265,7 +265,7 @@ describe("EntityUsage", () => { }); expect(await screen.findByText("Tag Spend Overview")).toBeInTheDocument(); - expect(await screen.findByText("$-")).toBeInTheDocument(); + expect(await screen.findByText("$0.00")).toBeInTheDocument(); expect(screen.getByText("Total Spend")).toBeInTheDocument(); expect(screen.getAllByText("0")[0]).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx index 3179ce25f80..1725925fe76 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx @@ -305,6 +305,20 @@ const EntityUsage: React.FC = ({ } }; + const getEntityLabel = (entity: string, metadata?: Record): string => { + if (entityList) { + const entityItem = entityList.find((item) => item.value === entity); + if (entityItem) { + return entityItem.label; + } + } + // Fallback to team_alias for backward compatibility + if (metadata?.team_alias) { + return metadata.team_alias; + } + return entity; + }; + const filterDataByTags = (data: EntityMetricWithMetadata[]) => { if (selectedTags.length === 0) return data; return data.filter((item) => selectedTags.includes(item.metadata.id)); @@ -328,7 +342,7 @@ const EntityUsage: React.FC = ({ cache_creation_input_tokens: 0, }, metadata: { - alias: (data.metadata as any).team_alias || entity, + alias: getEntityLabel(entity, data.metadata as any), id: entity, }, }; @@ -472,7 +486,7 @@ const EntityUsage: React.FC = ({ const metrics = entityData as EntityMetrics; return (

- {metrics.metadata.team_alias || entity}: $ + {getEntityLabel(entity, metrics.metadata)}: $ {formatNumberWithCommas(metrics.metrics.spend, 2)}

); diff --git a/ui/litellm-dashboard/src/utils/dataUtils.test.ts b/ui/litellm-dashboard/src/utils/dataUtils.test.ts index 24d9d7f1d8d..14eba8a7edd 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.test.ts @@ -72,8 +72,8 @@ describe("dataUtils", () => { }); it("should handle zero and non-finite values", () => { - expect(formatNumberWithCommas(0)).toBe("-"); - expect(formatNumberWithCommas(0, 2)).toBe("-"); + expect(formatNumberWithCommas(0)).toBe("0"); + expect(formatNumberWithCommas(0, 2)).toBe("0.00"); expect(formatNumberWithCommas(Infinity)).toBe("-"); expect(formatNumberWithCommas(Number.NaN)).toBe("-"); }); @@ -83,6 +83,18 @@ describe("dataUtils", () => { expect(formatNumberWithCommas(12_345, 2, true)).toBe("12.35K"); expect(formatNumberWithCommas(-1_200, 2, true)).toBe("-1.20K"); }); + + it("should show zero when showZero is true", () => { + expect(formatNumberWithCommas(0, 0, false, true)).toBe("0"); + expect(formatNumberWithCommas(0, 2, false, true)).toBe("0.00"); + expect(formatNumberWithCommas(0, 0, true, true)).toBe("0"); + }); + + it("should return '-' for zero when showZero is false", () => { + expect(formatNumberWithCommas(0, 0, false, false)).toBe("-"); + expect(formatNumberWithCommas(0, 2, false, false)).toBe("-"); + expect(formatNumberWithCommas(0, 0, true, false)).toBe("-"); + }); }); describe("getSpendString", () => { diff --git a/ui/litellm-dashboard/src/utils/dataUtils.ts b/ui/litellm-dashboard/src/utils/dataUtils.ts index f58ad074444..0f8d11a178e 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.ts @@ -16,8 +16,9 @@ export const formatNumberWithCommas = ( value: number | null | undefined, decimals: number = 0, abbreviate: boolean = false, + showZero: boolean = true, ): string => { - if (value === null || value === undefined || !Number.isFinite(value) || value === 0) { + if (value === null || value === undefined || !Number.isFinite(value) || (value === 0 && !showZero)) { return "-"; } @@ -51,7 +52,7 @@ export const getSpendString = (value: number | null | undefined, decimals: numbe return "-"; } - const formatted = formatNumberWithCommas(value, decimals); + const formatted = formatNumberWithCommas(value, decimals, false, false); const numericFormatted = Number(formatted.replace(/,/g, "")); if (numericFormatted === 0) { From 1037dc18b9f33221b4652093e661dbe59786e91a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 12:23:11 -0800 Subject: [PATCH 51/66] Adding test --- .../EntityUsage/EntityUsage.test.tsx | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx index 2ace90e0122..afd5c5789b8 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx @@ -269,4 +269,79 @@ describe("EntityUsage", () => { expect(screen.getByText("Total Spend")).toBeInTheDocument(); expect(screen.getAllByText("0")[0]).toBeInTheDocument(); }); + + it("should use entityList label when entityList is provided and entity exists", async () => { + const customEntityList = [ + { label: "Custom Tag Label", value: "tag-1" }, + { label: "Tag 2", value: "tag-2" }, + ]; + + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + await waitFor(() => { + expect(screen.getByText("Custom Tag Label")).toBeInTheDocument(); + }); + }); + + it("should fallback to team_alias when entityList is provided but entity does not exist", async () => { + const customEntityList = [{ label: "Tag 2", value: "tag-2" }]; + + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + await waitFor(() => { + expect(screen.getByText("Tag 1")).toBeInTheDocument(); + }); + }); + + it("should fallback to team_alias when entityList is null", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + await waitFor(() => { + expect(screen.getByText("Tag 1")).toBeInTheDocument(); + }); + }); + + it("should fallback to entity value when no entityList and no team_alias", async () => { + const spendDataWithoutAlias = { + ...mockSpendData, + results: [ + { + ...mockSpendData.results[0], + breakdown: { + ...mockSpendData.results[0].breakdown, + entities: { + "tag-1": { + ...mockSpendData.results[0].breakdown.entities["tag-1"], + metadata: {}, + }, + }, + }, + }, + ], + }; + + mockTagDailyActivityCall.mockResolvedValue(spendDataWithoutAlias); + + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + await waitFor(() => { + expect(screen.getByText("tag-1")).toBeInTheDocument(); + }); + }); }); From 19504adb9d8b939c5f7bfcac4ee5743e6936d75b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 12:52:51 -0800 Subject: [PATCH 52/66] Agent Usage small issues --- .../EntityUsage/EntityUsage.test.tsx | 43 +++++++- .../components/EntityUsage/EntityUsage.tsx | 48 ++++----- .../src/components/activity_metrics.test.tsx | 101 ++++++++++++++++++ .../src/components/activity_metrics.tsx | 62 +++++++---- .../shared/advanced_date_picker.tsx | 6 +- 5 files changed, 209 insertions(+), 51 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/activity_metrics.test.tsx diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx index 96d8d25407e..14f53dd28a0 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.test.tsx @@ -1,7 +1,7 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import EntityUsage from "./EntityUsage"; import * as networking from "../../../networking"; +import EntityUsage from "./EntityUsage"; beforeAll(() => { if (typeof window !== "undefined" && !window.ResizeObserver) { @@ -269,4 +269,45 @@ describe("EntityUsage", () => { expect(screen.getByText("Total Spend")).toBeInTheDocument(); expect(screen.getAllByText("0")[0]).toBeInTheDocument(); }); + + it("should display Model Activity tab for non-agent entity types", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Model Activity")).toBeInTheDocument(); + }); + + it("should display Request / Token Consumption tab for agent entity type", async () => { + render(); + + await waitFor(() => { + expect(mockAgentDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Request / Token Consumption")).toBeInTheDocument(); + }); + + it("should display Top Models title for non-agent entity types", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + const topModelsElements = screen.getAllByText("Top Models"); + expect(topModelsElements.length).toBeGreaterThan(0); + }); + + it("should display Top Agents title for agent entity type", async () => { + render(); + + await waitFor(() => { + expect(mockAgentDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Top Agents")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx index 3179ce25f80..3ccda5aa047 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx @@ -1,41 +1,41 @@ -import React, { useState, useEffect } from "react"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; import { BarChart, Card, - Title, - Text, - Grid, Col, DateRangePickerValue, + DonutChart, + Grid, + Subtitle, + Tab, + TabGroup, Table, - TableHead, - TableRow, - TableHeaderCell, TableBody, TableCell, - DonutChart, - TabPanel, - TabGroup, + TableHead, + TableHeaderCell, + TableRow, TabList, - Tab, + TabPanel, TabPanels, - Subtitle, + Text, + Title, } from "@tremor/react"; +import React, { useEffect, useState } from "react"; import { ActivityMetrics, processActivityData } from "../../../activity_metrics"; -import { DailyData, BreakdownMetrics, KeyMetricWithMetadata, EntityMetricWithMetadata, TagUsage } from "../../types"; +import { UsageExportHeader } from "../../../EntityUsageExport"; +import type { EntityType } from "../../../EntityUsageExport/types"; import { + agentDailyActivityCall, + customerDailyActivityCall, organizationDailyActivityCall, tagDailyActivityCall, teamDailyActivityCall, - customerDailyActivityCall, - agentDailyActivityCall, } from "../../../networking"; -import TopKeyView from "./TopKeyView"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { valueFormatterSpend } from "../../utils/value_formatters"; import { getProviderLogoAndName } from "../../../provider_info_helpers"; -import { UsageExportHeader } from "../../../EntityUsageExport"; -import type { EntityType } from "../../../EntityUsageExport/types"; +import { BreakdownMetrics, DailyData, EntityMetricWithMetadata, KeyMetricWithMetadata, TagUsage } from "../../types"; +import { valueFormatterSpend } from "../../utils/value_formatters"; +import TopKeyView from "./TopKeyView"; import TopModelView from "./TopModelView"; interface EntityMetrics { @@ -385,7 +385,7 @@ const EntityUsage: React.FC = ({ Cost - Model Activity + {entityType === "agent" ? "Request / Token Consumption" : "Model Activity"} Key Activity @@ -583,7 +583,7 @@ const EntityUsage: React.FC = ({ {/* Top Models */} - Top Models + {entityType === "agent" ? "Top Agents" : "Top Models"} @@ -661,10 +661,10 @@ const EntityUsage: React.FC = ({ - + - + diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx new file mode 100644 index 00000000000..04c2e4ec597 --- /dev/null +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -0,0 +1,101 @@ +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { beforeAll, describe, expect, it, vi } from "vitest"; +import { ActivityMetrics } from "./activity_metrics"; +import { ModelActivityData } from "./UsagePage/types"; + +beforeAll(() => { + if (typeof window !== "undefined" && !window.ResizeObserver) { + window.ResizeObserver = class ResizeObserver { + observe() {} + unobserve() {} + disconnect() {} + } as any; + } +}); + +vi.mock("@tremor/react", () => ({ + Card: ({ children }: { children: React.ReactNode }) =>
{children}
, + Grid: ({ children }: { children: React.ReactNode }) =>
{children}
, + Text: ({ children }: { children: React.ReactNode }) => {children}, + Title: ({ children }: { children: React.ReactNode }) =>

{children}

, + AreaChart: () =>
AreaChart
, + BarChart: () =>
BarChart
, +})); + +vi.mock("antd", () => { + const CollapseComponent = ({ children }: { children: React.ReactNode }) =>
{children}
; + CollapseComponent.Panel = ({ children, header }: { children: React.ReactNode; header: React.ReactNode }) => ( +
+
{header}
+
{children}
+
+ ); + return { + Collapse: CollapseComponent, + }; +}); + +vi.mock("./UsagePage/utils/value_formatters", () => ({ + valueFormatter: (value: number) => value.toString(), +})); + +vi.mock("./common_components/chartUtils", () => ({ + CustomTooltip: () => null, + CustomLegend: () =>
Legend
, +})); + +vi.mock("@/utils/dataUtils", () => ({ + formatNumberWithCommas: (value: number, decimals?: number) => { + return value.toFixed(decimals || 0); + }, +})); + +describe("ActivityMetrics", () => { + const mockModelMetrics: Record = { + "gpt-4": { + label: "GPT-4", + total_requests: 100, + total_successful_requests: 95, + total_failed_requests: 5, + total_tokens: 50000, + prompt_tokens: 30000, + completion_tokens: 20000, + total_spend: 100.5, + total_cache_read_input_tokens: 1000, + total_cache_creation_input_tokens: 500, + top_api_keys: [], + daily_data: [ + { + date: "2025-01-01", + metrics: { + prompt_tokens: 30000, + completion_tokens: 20000, + total_tokens: 50000, + api_requests: 100, + spend: 100.5, + successful_requests: 95, + failed_requests: 5, + cache_read_input_tokens: 1000, + cache_creation_input_tokens: 500, + }, + }, + ], + }, + }; + + it("should render", () => { + render(); + expect(screen.getByText("Overall Usage")).toBeInTheDocument(); + }); + + it("should display prompt caching metrics when hidePromptCachingMetrics is false", () => { + render(); + expect(screen.getByText("Prompt Caching Metrics")).toBeInTheDocument(); + }); + + it("should hide prompt caching metrics when hidePromptCachingMetrics is true", () => { + render(); + expect(screen.queryByText("Prompt Caching Metrics")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 8b1c03e92db..f8535b8a7e2 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -6,12 +6,22 @@ import { Collapse } from "antd"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { valueFormatter } from "./UsagePage/utils/value_formatters"; import { CustomTooltip, CustomLegend } from "./common_components/chartUtils"; +import { EntityType } from "./EntityUsageExport/types"; interface ActivityMetricsProps { modelMetrics: Record; + hidePromptCachingMetrics?: boolean; } -const ModelSection = ({ modelName, metrics }: { modelName: string; metrics: ModelActivityData }) => { +const ModelSection = ({ + modelName, + metrics, + hidePromptCachingMetrics = false, +}: { + modelName: string; + metrics: ModelActivityData; + hidePromptCachingMetrics?: boolean; +}) => { return (
{/* Summary Cards */} @@ -138,35 +148,37 @@ const ModelSection = ({ modelName, metrics }: { modelName: string; metrics: Mode /> - -
- Prompt Caching Metrics - +
+ Prompt Caching Metrics + +
+
+ Cache Read: {metrics.total_cache_read_input_tokens?.toLocaleString() || 0} tokens + Cache Creation: {metrics.total_cache_creation_input_tokens?.toLocaleString() || 0} tokens +
+ -
-
- Cache Read: {metrics.total_cache_read_input_tokens?.toLocaleString() || 0} tokens - Cache Creation: {metrics.total_cache_creation_input_tokens?.toLocaleString() || 0} tokens -
- - + + )}
); }; -export const ActivityMetrics: React.FC = ({ modelMetrics }) => { +export const ActivityMetrics: React.FC = ({ modelMetrics, hidePromptCachingMetrics = false }) => { const modelNames = Object.keys(modelMetrics).sort((a, b) => { if (a === "") return 1; if (b === "") return -1; @@ -314,7 +326,11 @@ export const ActivityMetrics: React.FC = ({ modelMetrics }
} > - + ))} diff --git a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx index d9e70ef3336..4293657b71a 100644 --- a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx +++ b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx @@ -1,7 +1,7 @@ -import React, { useCallback, useState, useRef, useEffect } from "react"; -import { DateRangePickerValue, Text, Button } from "@tremor/react"; import { CalendarOutlined, ClockCircleOutlined } from "@ant-design/icons"; +import { Button, DateRangePickerValue, Text } from "@tremor/react"; import moment from "moment"; +import React, { useCallback, useEffect, useRef, useState } from "react"; interface AdvancedDatePickerProps { value: DateRangePickerValue; @@ -299,7 +299,7 @@ const AdvancedDatePicker: React.FC = ({ {/* Dropdown panel */} {isOpen && ( -
+
{/* Left side - Relative time options */}
From 5d01f14f94ed2255c5135ac3f0ada78a1efd70ea Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 12:55:15 -0800 Subject: [PATCH 53/66] Fixing build --- .../src/components/activity_metrics.test.tsx | 4 +++- .../src/components/activity_metrics.tsx | 12 +++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index 04c2e4ec597..85bcabcfda3 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -25,12 +25,14 @@ vi.mock("@tremor/react", () => ({ vi.mock("antd", () => { const CollapseComponent = ({ children }: { children: React.ReactNode }) =>
{children}
; - CollapseComponent.Panel = ({ children, header }: { children: React.ReactNode; header: React.ReactNode }) => ( + const PanelComponent = ({ children, header }: { children: React.ReactNode; header: React.ReactNode }) => (
{header}
{children}
); + PanelComponent.displayName = "Collapse.Panel"; + CollapseComponent.Panel = PanelComponent; return { Collapse: CollapseComponent, }; diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index f8535b8a7e2..7a387c26600 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,12 +1,10 @@ -import React from "react"; -import { Card, Grid, Text, Title } from "@tremor/react"; -import { AreaChart, BarChart } from "@tremor/react"; -import { DailyData, ModelActivityData, KeyMetricWithMetadata, TopApiKeyData } from "./UsagePage/types"; -import { Collapse } from "antd"; import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { AreaChart, BarChart, Card, Grid, Text, Title } from "@tremor/react"; +import { Collapse } from "antd"; +import React from "react"; +import { CustomLegend, CustomTooltip } from "./common_components/chartUtils"; +import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopApiKeyData } from "./UsagePage/types"; import { valueFormatter } from "./UsagePage/utils/value_formatters"; -import { CustomTooltip, CustomLegend } from "./common_components/chartUtils"; -import { EntityType } from "./EntityUsageExport/types"; interface ActivityMetricsProps { modelMetrics: Record; From ed28818f76357cfe285e4e3e14d4d833649c7e9a Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 12 Dec 2025 17:56:17 -0300 Subject: [PATCH 54/66] feat(bedrock): add EU Claude Opus 4.5 model (#17897) Add eu.anthropic.claude-opus-4-5-20251101-v1:0 to support AWS Bedrock cross-region inference in EU regions. Fixes #17867 --- model_prices_and_context_window.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 696eeb51a34..1ca26164431 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24834,6 +24834,32 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "eu.anthropic.claude-opus-4-5-20251101-v1:0": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, From d090b4ad3e55ef69d44a90b07fab88e1c4816716 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 13:15:25 -0800 Subject: [PATCH 55/66] Add All Proxy Models To Default User Settings --- .../components/DefaultUserSettings.test.tsx | 153 ++++++++++++++++++ ...SOSettings.tsx => DefaultUserSettings.tsx} | 14 +- .../src/components/view_users.tsx | 4 +- 3 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx rename ui/litellm-dashboard/src/components/{SSOSettings.tsx => DefaultUserSettings.tsx} (98%) diff --git a/ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx b/ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx new file mode 100644 index 00000000000..78d50fa7f31 --- /dev/null +++ b/ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx @@ -0,0 +1,153 @@ +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import DefaultUserSettings from "./DefaultUserSettings"; +import * as networking from "./networking"; + +vi.mock("./networking", () => ({ + getInternalUserSettings: vi.fn(), + updateInternalUserSettings: vi.fn(), + modelAvailableCall: vi.fn(), +})); + +vi.mock("./common_components/budget_duration_dropdown", () => ({ + default: ({ value, onChange }: { value: string | null; onChange: (value: string | null) => void }) => ( + + ), + getBudgetDurationLabel: (value: string) => value, +})); + +vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ + getModelDisplayName: (model: string) => model, +})); + +describe("DefaultUserSettings", () => { + const mockGetInternalUserSettings = vi.mocked(networking.getInternalUserSettings); + const mockUpdateInternalUserSettings = vi.mocked(networking.updateInternalUserSettings); + const mockModelAvailableCall = vi.mocked(networking.modelAvailableCall); + + const defaultProps = { + accessToken: "test-token", + userID: "user-123", + userRole: "Admin", + possibleUIRoles: { + internal_user_admin: { + ui_label: "Admin", + description: "Full access", + }, + internal_user_viewer: { + ui_label: "Viewer", + description: "Read-only access", + }, + }, + }; + + const mockSettings = { + values: { + user_role: "internal_user_admin", + budget_duration: "monthly", + max_budget: 1000, + teams: [], + }, + field_schema: { + description: "Default user settings", + properties: { + user_role: { + type: "string", + description: "User role", + }, + budget_duration: { + type: "string", + description: "Budget duration", + }, + max_budget: { + type: "number", + description: "Maximum budget", + }, + teams: { + type: "array", + description: "Teams", + }, + }, + }, + }; + + beforeEach(() => { + mockGetInternalUserSettings.mockClear(); + mockUpdateInternalUserSettings.mockClear(); + mockModelAvailableCall.mockClear(); + mockModelAvailableCall.mockResolvedValue({ + data: [{ id: "gpt-4" }, { id: "gpt-3.5-turbo" }], + }); + }); + + it("should render", async () => { + mockGetInternalUserSettings.mockResolvedValue(mockSettings); + + render(); + + await waitFor(() => { + expect(mockGetInternalUserSettings).toHaveBeenCalled(); + }); + + expect(screen.getByText("Default User Settings")).toBeInTheDocument(); + }); + + it("should toggle edit mode when edit button is clicked", async () => { + mockGetInternalUserSettings.mockResolvedValue(mockSettings); + + render(); + + await waitFor(() => { + expect(screen.getByText("Edit Settings")).toBeInTheDocument(); + }); + + const editButton = screen.getByText("Edit Settings"); + act(() => { + fireEvent.click(editButton); + }); + + expect(screen.getByText("Cancel")).toBeInTheDocument(); + expect(screen.getByText("Save Changes")).toBeInTheDocument(); + expect(screen.queryByText("Edit Settings")).not.toBeInTheDocument(); + }); + + it("should save settings when save button is clicked", async () => { + mockGetInternalUserSettings.mockResolvedValue(mockSettings); + mockUpdateInternalUserSettings.mockResolvedValue({ + settings: { + ...mockSettings.values, + max_budget: 2000, + }, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Edit Settings")).toBeInTheDocument(); + }); + + const editButton = screen.getByText("Edit Settings"); + act(() => { + fireEvent.click(editButton); + }); + + await waitFor(() => { + expect(screen.getByText("Save Changes")).toBeInTheDocument(); + }); + + const saveButton = screen.getByText("Save Changes"); + act(() => { + fireEvent.click(saveButton); + }); + + await waitFor(() => { + expect(mockUpdateInternalUserSettings).toHaveBeenCalled(); + }); + + expect(screen.getByText("Edit Settings")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/SSOSettings.tsx b/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx similarity index 98% rename from ui/litellm-dashboard/src/components/SSOSettings.tsx rename to ui/litellm-dashboard/src/components/DefaultUserSettings.tsx index 6402220f374..988a3bcec92 100644 --- a/ui/litellm-dashboard/src/components/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/DefaultUserSettings.tsx @@ -8,7 +8,7 @@ import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_t import { formatNumberWithCommas } from "@/utils/dataUtils"; import NotificationManager from "./molecules/notifications_manager"; -interface SSOSettingsProps { +interface DefaultUserSettingsProps { accessToken: string | null; possibleUIRoles?: Record> | null; userID: string; @@ -21,7 +21,12 @@ interface TeamEntry { user_role: "user" | "admin"; } -const SSOSettings: React.FC = ({ accessToken, possibleUIRoles, userID, userRole }) => { +const DefaultUserSettings: React.FC = ({ + accessToken, + possibleUIRoles, + userID, + userRole, +}) => { const [loading, setLoading] = useState(true); const [settings, setSettings] = useState(null); const [isEditing, setIsEditing] = useState(false); @@ -277,6 +282,9 @@ const SSOSettings: React.FC = ({ accessToken, possibleUIRoles, + {availableModels.map((model: string) => (
) : ( - Date: Sat, 13 Dec 2025 07:17:50 +0900 Subject: [PATCH 56/66] fix: strip mcp server prefixes in responses --- .../mcp/litellm_proxy_mcp_handler.py | 14 +++- .../mcp/test_litellm_proxy_mcp_handler.py | 84 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 4dda665f70c..40f185a27b8 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -494,9 +494,21 @@ class LiteLLM_Proxy_MCP_Handler: server_name = tool_server_map[tool_name] + # Remove the server name prefix if the tool name includes it. + sanitized_tool_name = tool_name + unprefixed_name, prefixed_server_name = split_server_prefix_from_name( + tool_name + ) + if ( + prefixed_server_name + and prefixed_server_name == server_name + and unprefixed_name + ): + sanitized_tool_name = unprefixed_name + result = await global_mcp_server_manager.call_tool( server_name=server_name, - name=tool_name, + name=sanitized_tool_name, arguments=parsed_arguments, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 9d4e0aeded2..ab84294b35a 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,3 +1,7 @@ +import sys +import types +from unittest.mock import AsyncMock + import pytest from litellm.responses.mcp.litellm_proxy_mcp_handler import ( @@ -6,6 +10,26 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( from litellm.types.utils import ModelResponse +class _DummyMCPResult: + def __init__(self): + self.content = [] + + +def _setup_mcp_call_environment(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: + """Patch MCP globals so _execute_tool_calls can run in tests.""" + proxy_module = types.SimpleNamespace(proxy_logging_obj=object()) + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_module) + + fake_manager = types.SimpleNamespace( + call_tool=AsyncMock(return_value=_DummyMCPResult()) + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + return fake_manager.call_tool + + def test_deduplicate_mcp_tools_single_allowed_server(): tools = [{"name": "search"}, {"name": "search"}] # duplicate on purpose @@ -142,3 +166,63 @@ def test_transform_mcp_tools_to_openai_uses_chat_format(monkeypatch): assert resp_tools == [{"responses": True}] assert captured["chat"] == ["tool"] assert captured["responses"] == ["tool"] + + +@pytest.mark.asyncio +async def test_execute_tool_calls_strips_server_prefix(monkeypatch): + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + tool_name = "deepwiki-read_wiki_structure" + tool_calls = [ + { + "id": "call-1", + "function": {"name": tool_name, "arguments": "{}"}, + } + ] + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=tool_calls, + user_api_key_auth=None, + ) + + assert call_tool_mock.await_args.kwargs["name"] == "read_wiki_structure" + + +@pytest.mark.asyncio +async def test_execute_tool_calls_keeps_tool_name_without_prefix(monkeypatch): + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + tool_name = "read_wiki_structure" + tool_calls = [ + { + "id": "call-2", + "function": {"name": tool_name, "arguments": "{}"}, + } + ] + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=tool_calls, + user_api_key_auth=None, + ) + + assert call_tool_mock.await_args.kwargs["name"] == tool_name + + +@pytest.mark.asyncio +async def test_execute_tool_calls_keeps_tool_name_when_equal_to_server(monkeypatch): + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + tool_name = "echo" + tool_calls = [ + { + "id": "call-3", + "function": {"name": tool_name, "arguments": "{}"}, + } + ] + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "echo"}, + tool_calls=tool_calls, + user_api_key_auth=None, + ) + + assert call_tool_mock.await_args.kwargs["name"] == tool_name From e3f1ce0138b7489311d38d87ab2d3bc6d258dc37 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Dec 2025 14:42:31 -0800 Subject: [PATCH 57/66] bumping docusaurus/theme-mermaid to 3.9.0 --- ui/litellm-dashboard/package-lock.json | 16 ++++++++-------- ui/litellm-dashboard/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 49aac75a4e6..b65c35e78d8 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@anthropic-ai/sdk": "^0.54.0", - "@docusaurus/theme-mermaid": "^3.8.1", + "@docusaurus/theme-mermaid": "^3.9.0", "@headlessui/react": "^1.7.18", "@headlessui/tailwindcss": "^0.2.0", "@heroicons/react": "^1.0.6", @@ -4723,9 +4723,9 @@ } }, "node_modules/@next/env": { - "version": "14.2.33", - "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.33.tgz", - "integrity": "sha512-CgVHNZ1fRIlxkLhIX22flAZI/HmpDaZ8vwyJ/B0SDPTBuLZ1PJ+DWMjCHhqnExfmSQzA/PbZi8OAc7PAq2w9IA==", + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.35.tgz", + "integrity": "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -18170,12 +18170,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "14.2.33", - "resolved": "https://registry.npmjs.org/next/-/next-14.2.33.tgz", - "integrity": "sha512-GiKHLsD00t4ACm1p00VgrI0rUFAC9cRDGReKyERlM57aeEZkOQGcZTpIbsGn0b562FTPJWmYfKwplfO9EaT6ng==", + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.35.tgz", + "integrity": "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==", "license": "MIT", "dependencies": { - "@next/env": "14.2.33", + "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index e366b4febff..ce42d0ba41a 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@anthropic-ai/sdk": "^0.54.0", - "@docusaurus/theme-mermaid": "^3.8.1", + "@docusaurus/theme-mermaid": "^3.9.0", "@headlessui/react": "^1.7.18", "@headlessui/tailwindcss": "^0.2.0", "@heroicons/react": "^1.0.6", From 864b61b43327cc3ab9e8fcc1274f0615178c058b Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 13 Dec 2025 07:55:13 +0900 Subject: [PATCH 58/66] fix: support ResponseFunctionToolCall in follow-up input --- .../mcp/litellm_proxy_mcp_handler.py | 5 +++ .../mcp/test_litellm_proxy_mcp_handler.py | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 40f185a27b8..2a637893836 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -638,6 +638,11 @@ class LiteLLM_Proxy_MCP_Handler: function_calls: List[Dict[str, Any]] = [] for output_item in response.output: + if not isinstance(output_item, dict) and hasattr( + output_item, "model_dump" + ): + output_item = output_item.model_dump() + if isinstance(output_item, dict): if output_item.get("type") == "function_call": call_id = output_item.get("call_id") or output_item.get("id") diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index ab84294b35a..b632e72f567 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -8,6 +8,7 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.types.utils import ModelResponse +from litellm.types.responses.main import OutputFunctionToolCall class _DummyMCPResult: @@ -168,6 +169,36 @@ def test_transform_mcp_tools_to_openai_uses_chat_format(monkeypatch): assert captured["responses"] == ["tool"] +def test_create_follow_up_input_handles_response_function_tool_call(): + response = types.SimpleNamespace( + output=[ + OutputFunctionToolCall( + id="id", + type="function_call", + call_id="call-1", + name="foo", + arguments="{}", + status="completed", + ) + ] + ) + + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=response, + tool_results=[], + original_input=None, + ) + + assert follow_up == [ + { + "type": "function_call", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + } + ] + + @pytest.mark.asyncio async def test_execute_tool_calls_strips_server_prefix(monkeypatch): call_tool_mock = _setup_mcp_call_environment(monkeypatch) From bcb26cc55aee599cbd50c2d49fefd67177809040 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 13 Dec 2025 07:56:05 +0900 Subject: [PATCH 59/66] fix: format --- litellm/responses/mcp/litellm_proxy_mcp_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 2a637893836..2eea28f6cc1 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -638,9 +638,7 @@ class LiteLLM_Proxy_MCP_Handler: function_calls: List[Dict[str, Any]] = [] for output_item in response.output: - if not isinstance(output_item, dict) and hasattr( - output_item, "model_dump" - ): + if not isinstance(output_item, dict) and hasattr(output_item, "model_dump"): output_item = output_item.model_dump() if isinstance(output_item, dict): From bede40a90dd7aa1d73906bdceb2d71cb6324cf79 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 16:37:31 -0800 Subject: [PATCH 60/66] [A2a gateway] Add agent cost tracking on UI (#17899) * add CostConfigFields * add CostConfigFields * add output_cost_per_token * refactor table * add agent cost view --- .../src/components/agents/agent_config.ts | 47 +++- .../src/components/agents/agent_cost_view.tsx | 46 ++++ .../components/agents/agent_form_fields.tsx | 7 + .../src/components/agents/agent_info.tsx | 3 + .../src/components/agents/agent_table.tsx | 234 +++++++++++++----- .../components/agents/cost_config_fields.tsx | 23 ++ .../agents/dynamic_agent_form_fields.tsx | 23 +- 7 files changed, 320 insertions(+), 63 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/agents/agent_cost_view.tsx create mode 100644 ui/litellm-dashboard/src/components/agents/cost_config_fields.tsx diff --git a/ui/litellm-dashboard/src/components/agents/agent_config.ts b/ui/litellm-dashboard/src/components/agents/agent_config.ts index e930c2e07d7..9dd41eed8ac 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_config.ts +++ b/ui/litellm-dashboard/src/components/agents/agent_config.ts @@ -28,6 +28,7 @@ export const AGENT_FORM_CONFIG: { capabilities: SectionConfig; optional: SectionConfig; litellm: SectionConfig; + cost: SectionConfig; } = { basic: { key: "basic", @@ -146,6 +147,33 @@ export const AGENT_FORM_CONFIG: { }, ], }, + cost: { + key: "cost", + title: "Cost Configuration", + fields: [ + { + name: "cost_per_query", + label: "Cost Per Query ($)", + type: "text", + placeholder: "0.0", + tooltip: "Fixed cost per query", + }, + { + name: "input_cost_per_token", + label: "Input Cost Per Token ($)", + type: "text", + placeholder: "0.000001", + tooltip: "Cost per input token", + }, + { + name: "output_cost_per_token", + label: "Output Cost Per Token ($)", + type: "text", + placeholder: "0.000002", + tooltip: "Cost per output token", + }, + ], + }, }; export const SKILL_FIELD_CONFIG = { @@ -229,12 +257,16 @@ export const buildAgentDataFromForm = (values: any, existingAgent?: any) => { }, }; - // Only add litellm_params if there are values - if (values.model || values.make_public !== undefined) { - agentData.litellm_params = { - ...(values.model && { model: values.model }), - ...(values.make_public !== undefined && { make_public: values.make_public }), - }; + const params: Record = {}; + + if (values.model) params.model = values.model; + if (values.make_public !== undefined) params.make_public = values.make_public; + if (values.cost_per_query) params.cost_per_query = parseFloat(values.cost_per_query); + if (values.input_cost_per_token) params.input_cost_per_token = parseFloat(values.input_cost_per_token); + if (values.output_cost_per_token) params.output_cost_per_token = parseFloat(values.output_cost_per_token); + + if (Object.keys(params).length > 0) { + agentData.litellm_params = params; } return agentData; @@ -267,5 +299,8 @@ export const parseAgentForForm = (agent: any) => { supportsAuthenticatedExtendedCard: agent.agent_card_params?.supportsAuthenticatedExtendedCard, model: agent.litellm_params?.model, make_public: agent.litellm_params?.make_public, + cost_per_query: agent.litellm_params?.cost_per_query, + input_cost_per_token: agent.litellm_params?.input_cost_per_token, + output_cost_per_token: agent.litellm_params?.output_cost_per_token, }; }; diff --git a/ui/litellm-dashboard/src/components/agents/agent_cost_view.tsx b/ui/litellm-dashboard/src/components/agents/agent_cost_view.tsx new file mode 100644 index 00000000000..38851486dcb --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/agent_cost_view.tsx @@ -0,0 +1,46 @@ +import React from "react"; +import { Title } from "@tremor/react"; +import { Descriptions } from "antd"; +import { Agent } from "./types"; + +interface AgentCostViewProps { + agent: Agent; +} + +const AgentCostView: React.FC = ({ agent }) => { + const params = agent.litellm_params; + + if ( + params?.cost_per_query === undefined && + params?.input_cost_per_token === undefined && + params?.output_cost_per_token === undefined + ) { + return null; + } + + return ( +
+ Cost Configuration + + {params.cost_per_query !== undefined && ( + + ${params.cost_per_query} + + )} + {params.input_cost_per_token !== undefined && ( + + ${params.input_cost_per_token} + + )} + {params.output_cost_per_token !== undefined && ( + + ${params.output_cost_per_token} + + )} + +
+ ); +}; + +export default AgentCostView; + diff --git a/ui/litellm-dashboard/src/components/agents/agent_form_fields.tsx b/ui/litellm-dashboard/src/components/agents/agent_form_fields.tsx index 6d0c0820a4e..4dc4ad6829b 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_form_fields.tsx @@ -4,6 +4,8 @@ import { Button as AntButton } from "antd"; import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; import { AGENT_FORM_CONFIG, SKILL_FIELD_CONFIG } from "./agent_config"; +import CostConfigFields from "./cost_config_fields"; + const { Panel } = Collapse; interface AgentFormFieldsProps { @@ -154,6 +156,11 @@ const AgentFormFields: React.FC = ({ showAgentName = true ))} + {/* Cost Configuration */} + + + + {/* LiteLLM Parameters */} {AGENT_FORM_CONFIG.litellm.fields.map((field) => ( diff --git a/ui/litellm-dashboard/src/components/agents/agent_info.tsx b/ui/litellm-dashboard/src/components/agents/agent_info.tsx index c997ebc1e5b..626e352bd38 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_info.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_info.tsx @@ -6,6 +6,7 @@ import { getAgentInfo, patchAgentCall } from "../networking"; import { Agent } from "./types"; import AgentFormFields from "./agent_form_fields"; import { buildAgentDataFromForm, parseAgentForForm } from "./agent_config"; +import AgentCostView from "./agent_cost_view"; interface AgentInfoViewProps { agentId: string; @@ -147,6 +148,8 @@ const AgentInfoView: React.FC = ({ {formatDate(agent.updated_at)} + + {agent.agent_card_params?.skills && agent.agent_card_params.skills.length > 0 && (
Skills diff --git a/ui/litellm-dashboard/src/components/agents/agent_table.tsx b/ui/litellm-dashboard/src/components/agents/agent_table.tsx index fed79d5abb0..b141122c94e 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_table.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_table.tsx @@ -1,8 +1,17 @@ -import React from "react"; -import { Table, TableHead, TableRow, TableHeaderCell, TableBody, TableCell, Button, Icon } from "@tremor/react"; -import { TrashIcon } from "@heroicons/react/outline"; +import React, { useState } from "react"; +import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Button } from "@tremor/react"; +import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon, TrashIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; +import { CopyOutlined } from "@ant-design/icons"; import { Agent } from "./types"; +import { + ColumnDef, + flexRender, + getCoreRowModel, + getSortedRowModel, + SortingState, + useReactTable, +} from "@tanstack/react-table"; interface AgentTableProps { agentsList: Agent[]; @@ -23,71 +32,184 @@ const AgentTable: React.FC = ({ isAdmin, onAgentClick, }) => { - if (isLoading) { - return
Loading agents...
; - } + const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); - if (!agentsList || agentsList.length === 0) { - return
No agents found. Create one to get started.
; - } + const formatDate = (dateString?: string) => { + if (!dateString) return "-"; + const date = new Date(dateString); + return date.toLocaleString(); + }; - return ( - - - - Agent Name - Description - Created At - {isAdmin && Actions} - - - - {agentsList.map((agent) => ( - - - - - - - - {agent.agent_card_params?.description || "No description"} - - - {agent.created_at - ? new Date(agent.created_at).toLocaleDateString() - : "N/A"} - - {isAdmin && ( - -
+ const copyToClipboard = (text: string) => { + navigator.clipboard.writeText(text); + }; + + const columns: ColumnDef[] = [ + { + header: "Agent Name", + accessorKey: "agent_name", + cell: ({ row }) => { + const agent = row.original; + const name = agent.agent_name || ""; + return ( +
+ + + + + { + e.stopPropagation(); + copyToClipboard(agent.agent_id); + }} + className="cursor-pointer text-gray-500 hover:text-blue-500 text-xs" + /> + +
+ ); + }, + }, + { + header: "Description", + accessorKey: "agent_card_params.description", + cell: ({ row }) => { + const description = row.original.agent_card_params?.description || "No description"; + return ( + + {description} + + ); + }, + }, + { + header: "Created At", + accessorKey: "created_at", + cell: ({ row }) => { + const agent = row.original; + return ( + + {formatDate(agent.created_at)} + + ); + }, + }, + ...(isAdmin + ? [ + { + header: "Actions", + id: "actions", + enableSorting: false, + cell: ({ row }: any) => { + const agent = row.original; + + return ( +
- { e.stopPropagation(); onDeleteClick(agent.agent_id, agent.agent_name); }} - aria-label="Delete agent" + icon={TrashIcon} + className="text-red-500 hover:text-red-700 hover:bg-red-50" />
- + ); + }, + }, + ] + : []), + ]; + + const table = useReactTable({ + data: agentsList, + columns, + state: { + sorting, + }, + onSortingChange: setSorting, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + enableSorting: true, + }); + + return ( +
+
+
+ + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + +
+
+ {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} +
+
+ {header.column.getIsSorted() ? ( + { + asc: , + desc: , + }[header.column.getIsSorted() as string] + ) : ( + + )} +
+
+ + ))} + + ))} + + + {isLoading ? ( + + +
+

Loading...

+
+ + + ) : agentsList && agentsList.length > 0 ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + )) + ) : ( + + +
+

No agents found. Create one to get started.

+
+ + )} - - ))} - -
+ + +
+
); }; export default AgentTable; - diff --git a/ui/litellm-dashboard/src/components/agents/cost_config_fields.tsx b/ui/litellm-dashboard/src/components/agents/cost_config_fields.tsx new file mode 100644 index 00000000000..b07aba8dc7c --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/cost_config_fields.tsx @@ -0,0 +1,23 @@ +import React from "react"; +import { Form, Input } from "antd"; +import { AGENT_FORM_CONFIG } from "./agent_config"; + +const CostConfigFields: React.FC = () => { + return ( + <> + {AGENT_FORM_CONFIG.cost.fields.map((field) => ( + + + + ))} + + ); +}; + +export default CostConfigFields; + diff --git a/ui/litellm-dashboard/src/components/agents/dynamic_agent_form_fields.tsx b/ui/litellm-dashboard/src/components/agents/dynamic_agent_form_fields.tsx index 67f0f470ab2..55a4a62953a 100644 --- a/ui/litellm-dashboard/src/components/agents/dynamic_agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/components/agents/dynamic_agent_form_fields.tsx @@ -1,6 +1,10 @@ import React from "react"; -import { Form, Input, Select } from "antd"; +import { Form, Input, Select, Collapse } from "antd"; import { AgentCreateInfo, AgentCredentialFieldMetadata } from "../networking"; +import { AGENT_FORM_CONFIG } from "./agent_config"; +import CostConfigFields from "./cost_config_fields"; + +const { Panel } = Collapse; interface DynamicAgentFormFieldsProps { agentTypeInfo: AgentCreateInfo; @@ -59,6 +63,12 @@ const DynamicAgentFormFields: React.FC = ({ )} ))} + + + + + + ); }; @@ -84,6 +94,17 @@ export const buildDynamicAgentData = ( } } + // Add cost configuration + if (values.cost_per_query) { + litellmParams.cost_per_query = parseFloat(values.cost_per_query); + } + if (values.input_cost_per_token) { + litellmParams.input_cost_per_token = parseFloat(values.input_cost_per_token); + } + if (values.output_cost_per_token) { + litellmParams.output_cost_per_token = parseFloat(values.output_cost_per_token); + } + // Apply model_template if defined (e.g., "bedrock/agentcore/{agent_runtime_arn}") if (agentTypeInfo.model_template) { let model = agentTypeInfo.model_template; From 3054b6ea60fe533c00652b3637e0cab9b9a2fee7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 12 Dec 2025 16:38:04 -0800 Subject: [PATCH 61/66] [Feat] A2A Gateway - allow adding Azure Foundry Agents on UI (#17909) * add CostConfigFields * add CostConfigFields * add output_cost_per_token * refactor table * add agent cost view * add azure foundry fields * add foundry logo * fix: clean error * fix utils * fix agent edi * add easter egg * fix order * test_handle_streaming_forwards_api_key * fix forward api key down * fix a2a send msg * add A2a comparison on compare playground * fix chat ui * fix bedrock agentcore stream --- .../litellm_completion_bridge/handler.py | 4 + .../bedrock/chat/agentcore/sse_iterator.py | 412 ++++++++---------- .../public_endpoints/agent_create_fields.json | 43 ++ .../llm_translation/test_bedrock_agentcore.py | 8 +- .../test_completion_bridge_streaming.py | 90 ++++ .../public/assets/logos/azure_ai_foundry.png | Bin 0 -> 26316 bytes .../src/components/agents.tsx | 14 +- .../src/components/agents/agent_info.tsx | 67 ++- .../src/components/agents/agent_table.tsx | 24 +- .../src/components/agents/agent_type_utils.ts | 67 +++ .../src/components/navbar.tsx | 11 +- .../components/playground/chat_ui/ChatUI.tsx | 19 + .../playground/compareUI/CompareUI.tsx | 172 ++++++-- .../compareUI/components/ComparisonPanel.tsx | 31 +- .../compareUI/components/UnifiedSelector.tsx | 48 ++ .../playground/compareUI/endpoint_config.ts | 138 ++++++ .../playground/llm_calls/a2a_send_message.tsx | 25 +- 17 files changed, 867 insertions(+), 306 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/azure_ai_foundry.png create mode 100644 ui/litellm-dashboard/src/components/agents/agent_type_utils.ts create mode 100644 ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/compareUI/endpoint_config.ts diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 1f8892c91bf..2eab2551833 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -55,6 +55,7 @@ class A2ACompletionBridgeHandler: # Get completion params custom_llm_provider = litellm_params.get("custom_llm_provider") model = litellm_params.get("model", "agent") + api_key = litellm_params.get("api_key") # Build full model string if provider specified # Skip prepending if model already starts with the provider prefix @@ -72,6 +73,7 @@ class A2ACompletionBridgeHandler: model=full_model, messages=openai_messages, api_base=api_base, + api_key=api_key, stream=False, ) @@ -127,6 +129,7 @@ class A2ACompletionBridgeHandler: # Get completion params custom_llm_provider = litellm_params.get("custom_llm_provider") model = litellm_params.get("model", "agent") + api_key = litellm_params.get("api_key") # Build full model string if provider specified # Skip prepending if model already starts with the provider prefix @@ -157,6 +160,7 @@ class A2ACompletionBridgeHandler: model=full_model, messages=openai_messages, api_base=api_base, + api_key=api_key, stream=True, ) diff --git a/litellm/llms/bedrock/chat/agentcore/sse_iterator.py b/litellm/llms/bedrock/chat/agentcore/sse_iterator.py index e0da4fcd44f..90c5ada769f 100644 --- a/litellm/llms/bedrock/chat/agentcore/sse_iterator.py +++ b/litellm/llms/bedrock/chat/agentcore/sse_iterator.py @@ -5,7 +5,7 @@ Handles Server-Sent Events (SSE) streaming responses from AgentCore. """ import json -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional import httpx @@ -19,262 +19,234 @@ if TYPE_CHECKING: class AgentCoreSSEStreamIterator: - """Iterator for AgentCore SSE streaming responses. Supports both sync and async iteration.""" + """ + Iterator for AgentCore SSE streaming responses. + Supports both sync and async iteration. + + CRITICAL: The line iterators are created lazily on first access and reused. + We must NOT create new iterators in __aiter__/__iter__ because + CustomStreamWrapper calls __aiter__ on every call to its __anext__, + which would create new iterators and cause StreamConsumed errors. + """ def __init__(self, response: httpx.Response, model: str): self.response = response self.model = model self.finished = False - self.line_iterator = None - self.async_line_iterator = None + self._sync_iter: Any = None + self._async_iter: Any = None + self._sync_iter_initialized = False + self._async_iter_initialized = False def __iter__(self): - """Initialize sync iteration.""" - self.line_iterator = self.response.iter_lines() + """Initialize sync iteration - create iterator lazily on first call only.""" + if not self._sync_iter_initialized: + self._sync_iter = iter(self.response.iter_lines()) + self._sync_iter_initialized = True return self def __aiter__(self): - """Initialize async iteration.""" - self.async_line_iterator = self.response.aiter_lines() + """Initialize async iteration - create iterator lazily on first call only.""" + if not self._async_iter_initialized: + self._async_iter = self.response.aiter_lines().__aiter__() + self._async_iter_initialized = True return self - def __next__(self) -> ModelResponse: - """Sync iteration - parse SSE events and yield ModelResponse chunks.""" + def _parse_sse_line(self, line: str) -> Optional[ModelResponse]: + """ + Parse a single SSE line and return a ModelResponse chunk if applicable. + + AgentCore SSE format: + - data: {"event": {"contentBlockDelta": {"delta": {"text": "..."}}}} + - data: {"event": {"metadata": {"usage": {...}}}} + - data: {"message": {...}} + """ + line = line.strip() + if not line or not line.startswith("data:"): + return None + + json_str = line[5:].strip() + if not json_str: + return None + try: - if self.line_iterator is None: + data = json.loads(json_str) + + # Skip non-dict data (some lines contain Python repr strings) + if not isinstance(data, dict): + return None + + # Process content delta events + if "event" in data and isinstance(data["event"], dict): + event_payload = data["event"] + content_block_delta = event_payload.get("contentBlockDelta") + + if content_block_delta: + delta = content_block_delta.get("delta", {}) + text = delta.get("text", "") + + if text: + # Return chunk with text + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ] + + return chunk + + # Check for metadata/usage - this signals the end + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + + usage_data: AgentCoreUsage = metadata["usage"] # type: ignore + setattr( + chunk, + "usage", + Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + ), + ) + + self.finished = True + return chunk + + # Check for final message (alternative finish signal) + if "message" in data and isinstance(data["message"], dict): + if not self.finished: + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + + self.finished = True + return chunk + + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + + return None + + def _create_final_chunk(self) -> ModelResponse: + """Create a final chunk to signal stream completion.""" + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + + return chunk + + def __next__(self) -> ModelResponse: + """ + Sync iteration - parse SSE events and yield ModelResponse chunks. + + Uses next() on the stored iterator to properly resume between calls. + """ + try: + if self._sync_iter is None: raise StopIteration - for line in self.line_iterator: - line = line.strip() - - if not line or not line.startswith('data:'): - continue - - # Extract JSON from SSE line - json_str = line[5:].strip() - if not json_str: - continue - + + # Keep getting lines until we have a result to return + while True: try: - data = json.loads(json_str) - - # Skip non-dict data - if not isinstance(data, dict): - continue - - # Process content delta events - if "event" in data and isinstance(data["event"], dict): - event_payload = data["event"] - content_block_delta = event_payload.get("contentBlockDelta") - - if content_block_delta: - delta = content_block_delta.get("delta", {}) - text = delta.get("text", "") - - if text: - # Yield chunk with text - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta(content=text, role="assistant"), - ) - ] - - return chunk - - # Check for metadata/usage - metadata = event_payload.get("metadata") - if metadata and "usage" in metadata: - # This is the final chunk with usage - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason="stop", - index=0, - delta=Delta(), - ) - ] - - usage_data: AgentCoreUsage = metadata["usage"] # type: ignore - setattr(chunk, "usage", Usage( - prompt_tokens=usage_data.get("inputTokens", 0), - completion_tokens=usage_data.get("outputTokens", 0), - total_tokens=usage_data.get("totalTokens", 0), - )) - - self.finished = True - return chunk - - # Check for final message (alternative finish signal) - if "message" in data and isinstance(data["message"], dict): - if not self.finished: - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason="stop", - index=0, - delta=Delta(), - ) - ] - - self.finished = True - return chunk - - except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") - continue - - # Stream ended naturally - raise StopIteration + line = next(self._sync_iter) + except StopIteration: + # Stream ended - send final chunk if not already finished + if not self.finished: + self.finished = True + return self._create_final_chunk() + raise + + result = self._parse_sse_line(line) + if result is not None: + return result except StopIteration: raise except httpx.StreamConsumed: - # This is expected when the stream has been fully consumed raise StopIteration except httpx.StreamClosed: - # This is expected when the stream is closed raise StopIteration except Exception as e: verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}") raise StopIteration async def __anext__(self) -> ModelResponse: - """Async iteration - parse SSE events and yield ModelResponse chunks.""" + """ + Async iteration - parse SSE events and yield ModelResponse chunks. + + Uses __anext__() on the stored iterator to properly resume between calls. + """ try: - if self.async_line_iterator is None: + if self._async_iter is None: raise StopAsyncIteration - async for line in self.async_line_iterator: - line = line.strip() - - if not line or not line.startswith('data:'): - continue - - # Extract JSON from SSE line - json_str = line[5:].strip() - if not json_str: - continue - + + # Keep getting lines until we have a result to return + while True: try: - data = json.loads(json_str) - - # Skip non-dict data - if not isinstance(data, dict): - continue - - # Process content delta events - if "event" in data and isinstance(data["event"], dict): - event_payload = data["event"] - content_block_delta = event_payload.get("contentBlockDelta") - - if content_block_delta: - delta = content_block_delta.get("delta", {}) - text = delta.get("text", "") - - if text: - # Yield chunk with text - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta(content=text, role="assistant"), - ) - ] - - return chunk - - # Check for metadata/usage - metadata = event_payload.get("metadata") - if metadata and "usage" in metadata: - # This is the final chunk with usage - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason="stop", - index=0, - delta=Delta(), - ) - ] - - usage_data: AgentCoreUsage = metadata["usage"] # type: ignore - setattr(chunk, "usage", Usage( - prompt_tokens=usage_data.get("inputTokens", 0), - completion_tokens=usage_data.get("outputTokens", 0), - total_tokens=usage_data.get("totalTokens", 0), - )) - - self.finished = True - return chunk - - # Check for final message (alternative finish signal) - if "message" in data and isinstance(data["message"], dict): - if not self.finished: - chunk = ModelResponse( - id=f"chatcmpl-{uuid.uuid4()}", - created=0, - model=self.model, - object="chat.completion.chunk", - ) - - chunk.choices = [ - StreamingChoices( - finish_reason="stop", - index=0, - delta=Delta(), - ) - ] - - self.finished = True - return chunk - - except json.JSONDecodeError: - verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") - continue - - # Stream ended naturally - raise StopAsyncIteration + line = await self._async_iter.__anext__() + except StopAsyncIteration: + # Stream ended - send final chunk if not already finished + if not self.finished: + self.finished = True + return self._create_final_chunk() + raise + + result = self._parse_sse_line(line) + if result is not None: + return result except StopAsyncIteration: raise except httpx.StreamConsumed: - # This is expected when the stream has been fully consumed raise StopAsyncIteration except httpx.StreamClosed: - # This is expected when the stream is closed raise StopAsyncIteration except Exception as e: verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}") raise StopAsyncIteration - diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index ab2838d050c..232f6a300f9 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -71,6 +71,49 @@ "litellm_params_template": { "custom_llm_provider": "bedrock" } + }, + { + "agent_type": "azure_ai_foundry", + "agent_type_display_name": "Azure AI Foundry", + "description": "Connect to Microsoft Azure AI Foundry agents", + "logo_url": "/assets/logos/azure_ai_foundry.png", + "inherit_credentials_from_provider": "Azure AI", + "model_template": "azure_ai/agents/{agent_id}", + "credential_fields": [ + { + "key": "agent_id", + "label": "Agent ID", + "placeholder": "asst_abc123", + "tooltip": "The agent/assistant ID from your Azure AI Foundry project (e.g., asst_abc123)", + "required": true, + "field_type": "text", + "default_value": null, + "include_in_litellm_params": false + }, + { + "key": "api_base", + "label": "Azure AI API Base", + "placeholder": "https://your-project.services.ai.azure.com", + "tooltip": "The base URL for your Azure AI Foundry project endpoint", + "required": true, + "field_type": "text", + "default_value": null, + "include_in_litellm_params": true + }, + { + "key": "api_key", + "label": "Azure AI API Key", + "placeholder": null, + "tooltip": "API key for authenticating with your Azure AI Foundry project", + "required": true, + "field_type": "password", + "default_value": null, + "include_in_litellm_params": true + } + ], + "litellm_params_template": { + "custom_llm_provider": "azure_ai" + } } ] diff --git a/tests/llm_translation/test_bedrock_agentcore.py b/tests/llm_translation/test_bedrock_agentcore.py index 6dc6215a5e2..029bdf4e37b 100644 --- a/tests/llm_translation/test_bedrock_agentcore.py +++ b/tests/llm_translation/test_bedrock_agentcore.py @@ -41,16 +41,16 @@ def test_bedrock_agentcore_basic(model): @pytest.mark.asyncio @pytest.mark.parametrize( "model", [ - "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/non_stream_agent-mdfwS2DlAu", # non-streaming invocation - "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", # streaming invocation + "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_13sf6-cALnp38iZD", # streaming invocation ] ) async def test_bedrock_agentcore_with_streaming(model): """ Test AgentCore with streaming """ + print("running streming test for model=", model) #litellm._turn_on_debug() - response = litellm.completion( + response = await litellm.acompletion( model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", messages=[ { @@ -61,7 +61,7 @@ async def test_bedrock_agentcore_with_streaming(model): stream=True, ) - for chunk in response: + async for chunk in response: print("chunk=", chunk) diff --git a/tests/test_litellm/a2a_protocol/test_completion_bridge_streaming.py b/tests/test_litellm/a2a_protocol/test_completion_bridge_streaming.py index c088b3460a2..6f21029cd13 100644 --- a/tests/test_litellm/a2a_protocol/test_completion_bridge_streaming.py +++ b/tests/test_litellm/a2a_protocol/test_completion_bridge_streaming.py @@ -157,3 +157,93 @@ async def test_handle_streaming_emits_proper_events(): assert events[3]["result"]["status"]["state"] == "completed" assert events[3]["result"]["final"] is True + +@pytest.mark.asyncio +async def test_handle_streaming_forwards_api_key(): + """Test that handle_streaming forwards api_key from litellm_params to acompletion.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_chunk = MagicMock() + mock_chunk.choices = [MagicMock()] + mock_chunk.choices[0].delta = MagicMock() + mock_chunk.choices[0].delta.content = "Response" + + async def mock_streaming_response(): + yield mock_chunk + + with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: + mock_acompletion.return_value = mock_streaming_response() + + params = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hi"}], + "messageId": "msg-123", + } + } + + events = [] + async for event in A2ACompletionBridgeHandler.handle_streaming( + request_id="req-456", + params=params, + litellm_params={ + "custom_llm_provider": "azure_ai", + "model": "agents/asst_123", + "api_key": "test-api-key-12345", + }, + api_base="https://example.azure.com/", + ): + events.append(event) + + # Verify acompletion was called with api_key + mock_acompletion.assert_called_once() + call_kwargs = mock_acompletion.call_args.kwargs + assert call_kwargs["api_key"] == "test-api-key-12345" + assert call_kwargs["api_base"] == "https://example.azure.com/" + assert call_kwargs["model"] == "azure_ai/agents/asst_123" + + +@pytest.mark.asyncio +async def test_handle_non_streaming_forwards_api_key(): + """Test that handle_non_streaming forwards api_key from litellm_params to acompletion.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message = MagicMock() + mock_response.choices[0].message.content = "Hello!" + mock_response.id = "resp-123" + + with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: + mock_acompletion.return_value = mock_response + + params = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hi"}], + "messageId": "msg-123", + } + } + + await A2ACompletionBridgeHandler.handle_non_streaming( + request_id="req-456", + params=params, + litellm_params={ + "custom_llm_provider": "azure_ai", + "model": "agents/asst_456", + "api_key": "my-secret-api-key", + }, + api_base="https://my-azure.com/", + ) + + # Verify acompletion was called with api_key + mock_acompletion.assert_called_once() + call_kwargs = mock_acompletion.call_args.kwargs + assert call_kwargs["api_key"] == "my-secret-api-key" + assert call_kwargs["api_base"] == "https://my-azure.com/" + assert call_kwargs["model"] == "azure_ai/agents/asst_456" + diff --git a/ui/litellm-dashboard/public/assets/logos/azure_ai_foundry.png b/ui/litellm-dashboard/public/assets/logos/azure_ai_foundry.png new file mode 100644 index 0000000000000000000000000000000000000000..9f19b52e0bca454a75a12cd7fdc82d1efe5ee92b GIT binary patch literal 26316 zcmbrlcUY58w>BD@6p`M0lOoc4uTqpI5b2$O2ubL@iy&R;QbiCH1O%jpCRIU-(jgFf z3mqYJ&cpA0_q+H0uJ4a?o%2U7c%ICxnOW;z_gb@N5^JERd5?&J2m}J%)7DZq1c9*r z-hK$~0B;5+P;tN?LU%1QFA#{N`}TvC&QHP!0^!!V7@K;V>gvckz@dV6j&OU3pg+_d zXbl3%EBU+IIk-Z++3X?CE-(e2Jw!VXn~S3YkBNk?u&%or}HKSx7`iMn*_jR7g}*0B9lL6#(h|4(22uMqa zI|(@1N!g1zNQ+8{IywAD(7(F;Z!sG7fD{R!ouq_}h`5N9jD)oCe>(j0<^OJP4EJ$( ze!CP!(f>62?|1*Sl^43zuA9rhl=9ERzqRt89{*C=zZ(7zv;4m|m7~Ld%;fIl>Glr< z936xpZV)I0=6!1(LvVR#oG5Ko9FADbiGK|x(b^SQf+mW_jkG@FkX#8UyRq9J3b;-Moc>C9&D zX$NyuP`!Qg*w5EnLWRv60(Ex-oIpWd=>HV|e_hhQRtYczz!5_KVG`isKU@UNMuEo@ zU>fm)yfzTXu2oxI#n?Z0J0#!~@+e@_7Wcx9GJm`x-M>Uc3osQo` zrdSKMcn@*E8eqk5)mXoN+A7O#fGVoto}ml-_>O6;f#mPcWHstUJJh!W{?$H|A4=$} z-(LRjb)KMVII68P?gDEY&s`{-6=w>^7C+pAdOwfU4YA8ZMo>b`?j7;I?{t7a%%35A zl*+vfJCu==!iFS)KtgW>SLM!?#>&8Kt|WB<$Y5FPxT^PqAz_}48UvvG>1mf%BBh z>Y%D+*#_vAuygh9_@{hre-9M*JF50uE#TjWnElvsqdVy3LO+Lxk$!4#gw579;8Whk zbxRsE{eg129YW_Fr7Q-NSYn7F9wa0ardf@RibJ7ca z4REG|*3t00*V4-*pNi7E@0ZTeXLM-sD@XU+t;0=IL^ z5~)OZ&XR+yi^2!6n0m-R&q`*L4K>#@bAx9m3Us|l754crtExiDFIa-fB)C-<(suel z)<^Tu$zUvOl_zAXv;JSABVhPP32c=!!F{Vl z{$geKU^CpfF;iW*RXCE@7LtSGD5WMp+UvbRDS81hPWS#c6RhFT$%*m?HMZyfUW(#*Zqn^0n!-SegBKTNYa`394{eqY!;B>K)T>7aAXur+Rt@Q;CZY1%o zw>vB_=Y)B4g=6>@IAEzXS(39FLS0fx&P$(S)hV=3f=7|VeyEqT*BD(ZZukDs-?I(Y z$Mgiav8KXkECSq+o>{4z4Q7TZyoC=bPW^Z`JHZRV7xm_-7a()mf-k2np+1F>E8nG} z@r~3a*XJwVVX%a5>Bzp7^>I+B?{H<@1~uqo+LZ`&qVFo^Xy8D1C}24#DeOXojDUiD zas)3@rv-YHoL?EP?_&EgGHASh%=Gx!8HJB!5fWl)r@pmXckCbh8S@QNbRIK46hL;} zY$>bNn~X<3KEmC&7i8Ib-<0W`ClQ#|$7SUD56-`Kar{GRM)_n2xX^u=BmN^Vr+tt~y`Oc^9JPvok zHK++J1T&srq`aV`(%D|V81Q7W|?zf-#d8gP`sUZ zxL0&w|C>^Ki+-YV9X$ok0YM-&VdD;w@m5OQ9d=)i2@W~F_ zLgQ8iPWJ~l?a|6);q5G3WWPivpv_)yv0byB7BcQwW0?@Vhs8xGW zVH=(;BsWUsq0i{RhFc57NT55WmnP06RDQHdzbrO2JnFvxmg&+pU|I!k_sN@k|LVTR z0-yGt3}85w4S!i9nd~CXIC|dZl4LS!Y-w{lSa!!;)=MS-PC20{Wo2aoE8$81qY^~r zNCz!efk8juab#91`Apx{m-E`@Ag9$!KB=cvqEfSnHW$6xC_?Z24!#=+G6Di%qH8qp z8ShCj!zr%5_uo4Ea1PctFBJG$pnbHyJ4pgGD9egb$)%#MvYBD#d`pA2k4oYrUf6zp zH~6fwi&pyK09aWba%MfbEZ6m$`xG${@jKRt5yfq+LN*rEJtzGtTY5mu+gE;C>B{=h zzvaP_RQtC*@w^>M0$OlVQ6n3DkoqsZl)W~-(?MDhkz_Ct8?jO2n@MgVmav)tH^o=m=2 zkhKAy%hfIh_och9r7mgJ<&*253{Q6w$zrs)1mi!+DV_2CMGua+^svQi z_N+)TtnGfe$tJ!nQnw0jlq?WmUvJ!<`*w|6w$a*e zTAQmm@%8rJx?Ql0532yo{z<~q;=I_m=#5CC4WYh|AT8$Pa{5R=*6mhWu)}SC%+YU`|%dWlkbh6lWa>OK$%C%QcB|`4tfjSRRp3FZwok&e|u^7mZ}Lvy-Q{A<^YHgd8~9d zvMI(bj`A-<)qJBv^&Q0`14&XB;?v^wKW{XGzj3_j)>qXKOEx5ryUlPHSIY}w8-4|}SN_Fv? zfV0W_prW(sTe^rMP3^r5T{9W7II&c0Ud&ytz97jPj8?MWwdDNNpX8jwLfhTsPcz=y zk!eccDvMJ8d%5&?B_4D-8Tt0dx3gt)nqvc$A?Yf&EbVOu3Tqu5wzJgF*m!qb7XPbtJ?Z5tEAjm&W-6iXlsN!{N3tb|R}0x#tUa z!VUfFIEr_g`dbz1to5WltxvK4I0ZF&q?r!~EIx@dU+>E(nb@p8Aj(Z}KF4rkJ;$s* zYy~e|pI*K30$p;xeJqR*u-68)b)DV%fPuXFOOx4eDIY@|PJY6(@3P+g=oo`=8A>c# zJ`4Rkyzv^pu%q>W?CM9&1=GuHDk9vx?njKif`%P=Ad*)#H_PB5KdT9HIh&H5zva>u zHE~nraOUbGXuUva!gzy}KXy3clbAL>IM(HnLZT5@{tXnXB@;x6CP;ofxfO|i^|3g3R( z!GO{78Wn(T>xFb+LVr2}ledW|zRtY%)Mnyyq;FUU7K6E;v-Xe`?pAnuS|ZbN|ND3H z##vt$TrWa)Zxx~U5tFV9Z$7xyC%5?!(ND&Gm3fEgW{H)0Pbp2lGi~Us^2d=aMn! zWiuPJZT<%gD;4I(mAMv3q~xn?q%u>HjM*o*-l3Y>Q zC$2nrP~GM!HF)sY^sS3;A*|SsR@$d=nHAH_0|0 znAETL6uo(%@sp8}*`_|jC?$e#f4?zp+q*y$u(me?!<2DwP??+Y8ub^m-R%+?PHkSL zk7*7Lh0wWqam|o^4~v8ODeS8)l?U##`Z9f>g+0B%f|d~I5U>f{cxc^hH|YXON~Yv( zj~#fx$VGNi@$=;%HM;zq<+?;BKk75`FFZ$4#eJ!@`V19*^3x+6SMM)}eqxT;j}*lF z+K~?G@FrVVq0)26w0Iw8uVGrV>x}N8ck0&{7!xCWdaEs`kMaJRAC7R~Y_@2!>A2*n z`>bqD)KIsfg2@{QM&>1;-AoTcvS%INvW>_uHf)YsT!scIF?@U@jnMP1##Q}cw<)RyufKuq_%M(^&()YAopbN8^Rg}?44RiP`N-_dH|R@$e-8`) z@92Ifo1)T?1poD9HLGQw<;W`pJXAjsAVmg)nPOYI`?xdaEq5wk*DDJhgn~b?f)*@0 zb%hOXtxm$w*fPda!hmy0DGynueo*mMp{}&+q*?y#7W}gIQl6{F`h1J0@%j&QN|vsq zCFg4n+BE}%r=66}sWk#XM7gmQ*D|N^JR$qy1Z&s;0|+h=rRIj8J6dHl1u{hZc66?D zDVW;4Jm$h8M`g|Ba^WU_{M4NcmK?+)8u5Cxpb@>r`5;8^)(%wG-xhblpl4~v*<&ts zdgLEniwM8iJZ1|5B#dbO3;7{&O&YO6bySgNisi0-Q_#VX7;gy%F~`v86e$zFV<*#HP?POku!j!L4Ql6POoQ12I-k?E)<(u*TtQ`=D3?`D{fu7Qbj1S z!sxo3u?S_&nY!{})k$-PXpB;FQKyVB9#imSG0(%=;DOI4(4*hc+XP4_vdpbI*-`lGsHnkcHJbbG>YDr^usiEBH2C0s_U!P>i;1vs}bAUr-k z`k>6`{+Q6~#T@1s`i&v)x}?d#X7`}KM5Y}+4IG${UO|_p^m48`C$wnuxUU*kF z;%EXDQh6sMpsZQ`fZw7zt$*;ag5YCG!2D$WQz9QnsYwv_iw55A^!9@Cdvz@G8J%Rq zkIH>u580sgn!J^6?&EiU6n2nv6C>#gU1lpY%kJ1ICHbsQaKiQ(kiu_-s-NEReG$PFJP-h)Hh3 ze6J*FcyhgNE4d8n;C7<*h6dms8*>+%Ruba~{6H)!cPtk&;ybsK>xjqlvDJEH;bS#f z)J9!hdRkJpii_v^N$(GA^)Uh^pMUq6pk&*p?jzD=ZLi}hd4D|ETbb1hCH?FDcVxsg z0QTOg(dI1n8<9S0bVuXz(GTi9EXYLj=-Jk|2xg$ZNlc4712xLyVrhkTfGar>bNIL0 zI316Z^B@vjcXr$SEHUFcmB;Sep>DyQ9n=2UM_x6!)HV2?uP#WZDbw4Nrphh9nGbW$ z{Niafb8tN=mOiSyC?sn{)3UiYu(JQfx=mcU5v!d2keFIiY3y5jYB*VEdS&QMHuzTcLfvm~bGc0}wk!GwnPA(V%P)w`xU zx47@48+ui2xp@L?NKEu??c3}k3x-nH)^a;OhdMS~DBt#^A2JUc_Pm#?O$oDNWmf~s z6;u5k{K?2S2_+<&Fvv5$@vRJnWf%2!uEt)BQ6_Bz-diuP;E&yK@$^u3yigTpLHUcg zIe3hJi}I@(2D%z+QQHP?mEUCKv29~jga_;RAm*OsD@!;C_2YSDH@H4+E9>u`-Vy2b zDrC4H^bn7{Ir+`UWC>3M6FGc_?6g(62N9P0gYKaTXAvs;Gg3q2O!o`xY8_!ADHUMp z+HM9+@mzD6;Em{u0FG+oYIxhtRlmB~q#_SQMiz%ur>NZ1HH@?t>tYV1-*YC3fjKnJOzA z5Us=_?g)zY6XG%+4H}FS^Yy$|fatj3&H%-VCT_1iSw)8`I3d+oV6B9S^Fx_ z-!Bv-pIyj>jpm+tX2|gD_gC#xtFqFq)JuGS=x!F!DWLDx1b^fea%HM0G@Y!mBmjfN z>#?L-_#UNm7SBI|^<)-T(jn4a#Z_lm;-z*DERf-m(pY;mQtefTvUF!K`B` z)ALlX@W#Js9Et{4GOXBX!y@HljefSbn>H`^tKPoiTwjD+QpO3G_lFVQ_WWBTU&VZu^01J+ zUmu?;%w+5hvN>mltN$ZQz_<;(YNK+sZA|*X=TMCIBv;fE#eY z3*s#t?BxG;ugi9+Y051L*xDvl7jWJ^_-`U269s!5u7s zU%$DFkg;n+XFJaByq?3XqID0S9e;@>m}=o$F4cxN=d2z6XtT&!dtpwEUA2^U9Iz%( z8tl!7+ZqFuewO@qo970^_fxn;2e`DkPS;Y~s>+;Y$USDQYKo}pHQiuk)&{q8SDtcsRo~}`t!JXcumq7ngr)HO zHM5V1lX`JVu$_Q7T)Gbsk#jh;)3^5&F*NwnCE^ zE*f5FtZ{$V_Z`hOJ3-ZR;|?VlvEa;l0nEZpWYzj0ofC*?dD@=2Ig8Z7&8hKg4j$0_z9Ss=NS51bRZ8QxhdSzDxqbnpestMM zFVz;08kqLgw@^_N!K|8!XRa^qmRThZ$-4|4L9W-vs-G#&Ts~SikB}=mw5LV+_8$+w zeBW?O5yT9qYH?pabZ083fEAQekWuWJTnI^W!k!ZHPxd<*ReFQtJGx#)_x&hU7a~lX zLMr-hd*KdSH(ZE=_(26<7RI_9GgMM!>k5d$%%9d*j_pgO$qTjNp+KD+K?QRdB18g# z5JLxWS*$W+&SHXO6${TN;*^I(GF+QU<$NwP$3RE66NC9&k)|ENY9pdqPA%qYe9w=` zJxcSEl7oa1rE6dydNg09B@0EozumX%5M?$;-8qeVsntsUu1+T<_#ccrbEM4c&#QAK zDucTf&nLO^#cBIrE%_HUy>1UMdp1J9BI$<qIu34WJ)rEOY%V= zhHK7WQZk1NGHN+16s(=2Q#^GfHJHBB?@oiQ-~88I5w|r8S->fEg3P4BM(V`s-#*|(=`9QCkHA>vwTVX& z1K{hp52WJ~UppFh}(BO+Xya-{H7+|w5OlbM*^WoTuH#o0w z;uzQ1=lw1wwXiJlpyYTIFvw2Z{(bl;^0^@okv!*mYN&hvJ^h#O{Q%v3i*v4=y|M@4 zV+Y)>EAhRkRt@jUn_{`j5+GVaJ@5A3d99Ep<*WlkJlTfnr6~JMS!b5EyXX~cjht5@uo`(FUW-t8)ggo8S{h)l&CpIsUPbeIuIs4(x%ip$x|sY^+?3``_DPA1 zS4zh;wKou+6VU__2oGnq5jnt<%)7)UcB}~&Iz0$;4+q(`rScO8xqn{jdO>yQJh?f1 z6ued=`DSsaTXnsB1p`MdvydhF3xcw$3!GJff4t5o#-$!|NAqt81K9?22?CJtNbaFu4!Xs|>L%me2SDsGJhXU~EH(pmBR zQDKGDu*b5*?2c%NMYh>9<-ary%|ffIk6GhqOUfjZp38g+*W?v+g9kIx+x?eo-~GTn z2@D3*Q|sN|nu$>7&h0u_Qlpy7NIs!*Tb9fHro}V=gh|uB{{XkhvA`P|d%)z^x1}%t zO4?CH6QH~2c-gI(T<&8)J^!uezpmqpB6=_m!^s?7b!3h1`#GdI4{pBB!kFpGNgWGd zUtvi-i-vVA&@hhlVP2&<{N?{vY_16T@6>m%i^-nTN&zmop3qbCvnD8+4LHJuF4))F z=d16W9F{r73C;DMHSajJxK^R>A8a(jKl6egEBRPUnFQ;!!L8PhQmPp$ln(2^ARz67 zLt7`#*Od$sGQXP)m_zX%=f90j%;u6 zVEJ*@jGhbWS>1;+%JIPZGVY{8YB2QEvA+^MKds(7lYwuYz*&KhOc=d2mH#gq%O=yn z$t}hLLKPt(l+)eSw`g$23pa|eJY)s}c=OQuAdlyyiR55YpZ2Ogk2JeGw_e6v5H=7w zO-DK)dI~vHqaj|0(O0Tmo4vKw(_AtDV(mop2LY3G|53WyjsLW|TSoq77k{rb>`Wly>~$o^A$)46QxSFF|$odd2#s)wWK z`w;teuiuzTp^11&wpFp-(?>PaV7ms$D|DsX%F*W48q{-s~a95@j?+YS*x@^jT zA@!(4XaI+5PqRbafXjJl<-o5;aa;;KGtw&vu)J{ z9CQuEte?L-o@wW@NsXgNrHoOVq5$P3o=_eS2CWDfzfdY;@aWG{U2WE-FZ#Is3Fh(A ze*0PFZvtwKvR51$mG`xxv5Fp7_Vg~zJWn>TY_s`$y4tj>5Drw?ICGnQhu@8ul0;E^ z=ur2~B1$K2sn?>y&u^6JV^r-kK_fJ(-*(^@v&P!eyzio&e%m=r^8?U;*CD zppr?NDl+sTm2Kv!MVRDZK(!fH?BAB(B8Cn_S+6(7LPfcWlM;Cxlz3$5N63TTJ|O^Z zxo`nT)Mtddk@Ud5kr66dGoBaA2?~TCnCsR*AJduj1X;WF`cYE^qCUM*qV@a^kQB(Q!GTA0yB^~_j z{kwW^lp)+@xMAnoQmM>3S)=rf29NoGd7L`jt~EbB4m@PKQ**{Y(zd#1u9z`dr?gvb zV-hlaAh9oUgafk9dS@;S0dizVH8I4j#M7T(^k$iPzPMCdp#DI4;`?u*y+-J7GtbE7 zOzQ-RTC_v|gl$%i7X$~YzP8l>py0TeRtIeF_m{X*#IMLp2EWPV0nz4Rot+H-MAt42d6_6Z8$mHF<1L5@Amy<4(28|ls&Pth`* z-BGbARhW~~uSg!Xk<|w3K*qq9Rf}9)UM_Dm4k&rN>TtLYrs+FGB1~?^RS4$@oPhfW zDx%LS=4y$1h+E0E8Ax`>q{q2|#Q*+bTCA3M{c0=$H9Bcruy4GX?9>!jZxN4hj_99W zB!ZIsiQ0eIPu|0e^DsNZt>}C>w)Vua4tQ{?F;>fS8<&Tp&~vWD6hY=M{p96kDNiGz z(`}~vhS+Kw=66$CYWyZZ6cQ5u_=n%+HC2g6e9r7z@3;>h+rhJMDO}(e^G}_;EL|Lk z&L#q6P-U@@DaDfJIhfHSvmjr(P$uZdH)hQR9FL)E$!9U8G|MYt{_R)R=Mze2 z*Qjf^gAbe_rn?KmpaIZBfwsZnlwZ5&2?f}Vv>gH<#fxeI&6`OVQZ@{nKg%e$#dr8E z$T!_*SBa@i;irknPQ}T(>uzp~%+=@c)=Kp2IbMELFDCV^R5{DA>o{0XDRg)2zzE{i zG8wvB{+u&@~O19+Jo8ga6$FGxze@Qn?;Vj^QD%(Q` zez#|7!~ivl&yC$*Rz7WU*{H?U6{fkM>-i`9rjW>i$>^??j2<*(dQIj}@MvZu;;dMvVg7mN$f0^v+(&{oLH@=t z9@?YF3(4CJtiFPB1hbg}JtsVZJql0q!~1A{)Rz%1xPzqHotBd(=f19fYJcabDSAbr z?(t`)l?SVF(2ddWO!lV>--vzi^10P$bjqi67{6&Z)% zEi_kMVr)qmx;XbKk_Ak|D`MoyQ3^{2ZVD^5><62RDj?^1UnHrG7P4XP2=m=CG!%+rRH8dLnGO zy3c*^<}`a;*8AU6hxUjxs=aP;ow~cQ!PvlbY%m-9sPJy&`^?!RV?nK(J7s~`FXo_` zx?Z0h4$*)HZ0Xyyt;<+YG|oj;xYkR%AMqDm`ICB|u^(10Nnzz;Ej)qp(5TXOySr?8 z9)8N>=`QJBKa=fNB#Ah>(p{9!s&{$g2&4?XeN;z-S;CI`>KgBs1Dft*7(#g5!7a2W z*H@Un%w#zll+4n|FXBl0g^#y;0ZRVKU4^?vr3zR-?u!98f{uJ@*-~iF!ta3O-ys?T0hRha}GmiRIX8m@32;t8QS^AU)-Tt}hHkjAA+9U?@xMvqsNiX-kc6_Y97dQ#GJ&Ne3?jJZS zN&bDDw~ID2%0v$8f0A(f`IB|ur2pJt3e(uPc*LLFWh0LkXFm#E+8>bIeSS5s@^k~6 zLf;C3Sp}7e^>BeP=y9B*k2xhF5j=k zp@E!mZIH2?j%Ad}|M4|s_r@Z;XNjDtX&j#)I`kQtWIw6T?9qf8Nt2X7Q@A$!oCNp8 zBxj@?I^QUe^NFIbDH}JhNHtlKgmU*S{2i5=o(b*)spr0X=S(X;wts+Jci&lmNmEwU z>^xBCeBBBLwt`YWEyioPIh=e)w~7V;I4oYBbd{P^bl+H97yp{aJHiDf@yaACUHc(V zG7xB*U_n|HrO5()exiLUt6COIL&oE>$y|MA(fUuvp?d1?ui8~v36R}$fd9A4ta6#z z)2ahgc<(MUE*-nN^7Mkr+VzFB&Y`r%j_h~8mh0>kE=ER!c8AYf8+2BvmMc(o?CZ#z zpk85DY$*>kT(IwUE=7mf)GF9O)S^kULHIS@8CuhmO|{Tl1H%}vbRSoY?u-J5@UvWm z@4we=?ZUI;3!mV^&co2JyY}Bv7ML^VD3iS;YwqZI1k!(SpSwC&{)MaanQYla{Nt+R zmoAJJV0^7V$*cbbXm?8Ti}eP3BM|{?se@JOz6E~%q@iD2S$;rDWBEmdpZ%lQ=YJL= znM2P-b__)pxMjP3n4isl&1;}fO5i{PC~@)y*7ay?-!dzSQbKM50WSMx81N&Ro15)C zs{}5SfN0b_Ywd|3_ke$(@8rvBdPV{^HAg&h_BWh&m%^C@rlx8f+?BN}J>OdM^@A%Z z9>u;vF3CQQ`oS8@ffe~iJudtW0VO$S8bOTG%S;IkgcvW)6ycP=o?vr1qU?O^^U+aW z!B!}%Ym3o(@Nr(*JaDt?Y76~>NY-q>|8AxqR?BAv*z$zW(IU*aca>o>-XMC|lZDyi zoT8zq{8>G$m4WY*%aAIS7mgx-&B%M@av>#L);d{4;q}oD-)9o?R;xoAYkzK;iP}B4j->h z%-;4(_953U6}{w*ESJnxn|k&7fnmq&fHYNSnyU(FWaJ-|;&Qm8i+@V835Jt2+p?YWA4$$+H!Ks*d)Zd@p$- z)n|)2jM_a*t?NX2p7iBhIac zM{9;fgOX(C6z=+O@>0L{Iar8^HL!emzQoPWY@r2iDiQ8XSl+iQTkv`b-%YC^S zS@^DDpj1fv{(_I7^@V-#iceK9drvMwXNnw_`JTe0-2=Pk?jC*~G|J;)opliBM8L0J z%G?<4N<{aLX>>zkMf$*8^3QKWqE&i=4jCiRrJKDek;~1Bs;;}7*XQ?ZMhYoEon3hm z(po2rttLFx%TN57R%L|@lNK>Rwuy>{d1pyuE0` zoIDT4pJ!I;q#e%dDUx%DZzb@EA<6raHfv?sm}W6K7=w69pXq8@)mL%xW!;lyhpBKJ zixDGNT+(D#_dZ~vn)y`gh#3@ov!WgRcf7&WhM01Ri23}E*U>9}9#D&>ROU>D-{6xb zWc?_070H;SVZP><<8CXT!3vDr{&maTMJ24%d!Z;vp+GK>D(vulasdFlfGEWhKJ&Sg z7p-tM>euK%McUnl#8l6w^8;M{=Q(iVYk6@Gj>_hf^C=`wpM34b)zMl3cQI2V-!-Ka zDymd3-`g^lOp14dF(Es17TumExAZjuJ;?VgnwCFa3PA6E@6?P&Ld~h$!|ws!37b8Q z2x8O_HE`bDreIjEG|GqDGSuU0lQ7(}eV!C%74_qL%0+aI-(o)>Xg#7iAtye;3SxMGd!GhtcR;Uf9Pd5FEsign0c{)G68Y?3@dv<|n5eT?H=V2|PoBCy{~e zID12g=DjU?ZCQ89VJ4-dlpM@iA}#i@$>&NMDAe0q zA)D*UD_a$>NP+`U)_ylFGdv|YQU?uH$S$$IGyaU?mr78^*Du7o=$F4HFpg{JcwEWc z7X+yfEG3G(6KxTxFC|vW0;pj${=hI8VNW{?zTZ_yj53=yX%=(kxW8~5oLSYxU?UyG zt?H}VT=*W&jlqE}y+BW(pOyqmdJNX>9Q|ng?JwvjVm&JoJu=kWrP$ajG0_KRoRf9x zoNrIMj#SYk#)2Px&VfbQb24cY?-j!7sv6xPq#caqJ0~H1g4rHxh*-(9%gOqJJ%-xk zx|7)5lw`FJU)Av0JkH(+A=IjCWiO|*AMP`b?t8L- z2a9Uyhgzxf+lb+D@tdB`%bWfCFBOuLq*03f3zD&zHPskMF9Lr-bYrR}1VRC^&lb z+czG_d;0d39jd<;f7k>kld@_&yaNgf@wkSZehmhHf#$l>&m#aKPxdFi{sP{$_(D#{ zo6N9gsSs27_|s{N&^Hf%hMo6jj^?GhjTFLZNV^4IVGuq!b@!7+oX)a!y>_eXD`)C} zDJYqDf4P3x`*xN*KlC^Fbo+RHV@O+)cO$3g1CphIsx2SlpBeOLhiMn$an#d(m;RgYcEl|X|+2`9Dbgp@@$Yy8b`e&Kbb%?W$0FT+C!r#Hg z>&n>5pyDk?zfU=LGZF=n&T`eGK`` zNQkqQ)KOP?Wv7Wv;Eb1ms^fb#6w@yH?rWQ9ee{H`%p(Aj)qi1g{F-n#^9E~HFCR4j z#RRfBq^T7Zk#pMlP9(J7Njim|==_fR@=Cv4dZ!C$<06l;foeVsKIa7e>5yR7<}{ic z^hNuV^XwD5y*ZP2k9X~`=hu$bhiZ^|HC-tePG5N&W)+>)nu$>f7Q>>KTXewfUBE$AAP7X>M7?!ihPaOg!MjI3WMz5<8}FP%H~*0SZiF~@4bm- zFwyop#M0*-U}kjs@&wwjGDj}b9e=*hp> vp<|9dc z&=WbXv%~A|P1fUB<4k^L!-|sI@|F{|e&~S#;wDU?5Pf2X4eI=z>%k@YHN$;~a!lJi z5@$-z$(DPo9yZ5a+HO&mT-n=OPBk^$=iOwFhWMiYpf{0$K|ulHFIKaHg|C86`zL7T zWccn_n^#IDdLrt&>b%ilz0JlFhlbcxi&-D*{?(G}4%*C?lko^_rTrc?D4L{YZ|Q{J zLdeG^{EgN~VHKZHfh9doqm`8Nj=NaX@q>wpiYJxKOF+USSY3M{s+2z-(kjk6Ji{`I zI1YV;31RtF+n2Ps^x4RJqnK`EWHJ+9H??W5d3+{O`UpPPn2vpM<}THr>@1OpX!e=Y zPmBksFfp?w%3d!x_OM7Trmci`0qb|={#i0>f%v3SLDy=1UxmFZzVTK-rI6VdOSB0U_7cqk`<2kQnu(>;*9eyo(_Dl?HPkfwo z?du7?n8D~-Q;Nv3?*hx>*{gszO52%sA8LD@92pAwNw#fy_e_*S8`q%8&I-xt)zU11 zxC=qRDujVmq^4i2>j@$y1ye1m?4AE2j{Sj!1k5!)*>;nU7^rhcFQR+ol1;JYUW8x) zUF3N;Tzl`O@;6{jp|fderB}JP>pemt{_pCb%a8fH5GxYamX3nJ-yMr#%y&^)tg2zf zpl#{=wyg*4D^co^_c0P`+EQK9y=EDU%Nd>1;*;r~+>E>~;=%O`%19kFwH{U3e{nwSCEtem>D6 zp!zFe=T{R(8*X%NCe=WqPA2`tE;$Ie-;VA_*I5g^Upad7KJRGSp>K0U+hd>Q{Cq$B zjP}>z_50&i_eJTBo%z6)+pFDEfpwtIEjtHqj`pcBZHTSm1F;e9i2YK}dsBJa1t52p zujKDazRt1>4bkj1P?Km2N+bguO)CWCK~hyNm1rLlfq`1Sq%;D&5QRn?sVO1R zvz#@Mv`-gM{Y-GGJ@I$=Qvb9wV8b)|@K?3N$wzusito+8JEnn2-PzYOO0c3=w0<}J zV5o+e^aNUKL|1F3?dn_^)Kx9V#HgD6jHXy^ z`tH4fqltyVqS&q8oPc_+`J$Ixg_h0-_W8=`4Fww%V4h7B=pIk>xkm`SqWUjgS_?s1 zXjE||Nyw=@NUK>>a!|lwvfQOouJ0T;rU;gf2TrbjQ5a$aV%N@pUJHgdjP+R}fr}i! zs@<~V7s}S}uinF%h1y zn)cFgsbc`x1bo>b`&X^0i2lx zK#+kqfubb2LwRW=cp;4~?dRMX{#hRUU79i)|G){7z6il>v6xz3ff6F1UwQ&LKk_a( zI@taU3D3>hZYS@61z9%#Lb6Ui>?(aICS{)v=xbi2KYj6j($kiQZFFeCu9)^r%j;IO z;GUXPT>V=ATaJHblJy)h%l{|y)?>Z5?OdO*IPk&dTw7s3i&nOOx-c#}I)NSff4dL;)+fG)2BNiCoz zuDRx$8Q%a$OCR90#kXY>8N%#e8U)mOcsII=Ci5D8va`5lm)nk7`p;*4m0K}QL%zbx z1!e{ychRG2FUe!zm+X{y12CXvQc2TW)za{b_Y5e+HzYX)tL$Ga58s+FYphb$Ih9SA zmG4eK?pJ{Apv&1w3U0TtdvijU)%8J*HMA%wY@EPHpwMqeGv*&VKBQUoowU(`{)yK- znh$)jXx?phr&po(TC@eK80$cF>mJMLi?fW)Q&0P@t+HOB)>FdYcz<0{BpcE@0e)Vr zAB3#Ooqcp}*l0s<{g-Y%0+P0fc(N~fR%ZHHw8hk{MTFZZTB_GKZSg`;aa^8R^odsB z#z;&TqM6$eh)K`5o#|l;&h(+cZw=MUamr-o=ey=3axg_%sT753tR&C&6i9v0Uw^Q# zvrGf42ROHK9hNPYi41W<+7oCKGM|H_<))SHE$HQ#5@HOREi$cwSTd(8q?m(d>a5m3 zA_L<}nw);l%}4mD_~|{4j#MQ6(1wwB43l|lH)?4#29*AqSq#9~op>$urC)AH0Fh62 zyE;y{eJLp*{*r_DEfM5>B~2qGO%I%>UKMC`Q0K|=krD+i=H_khVf%Y&k!0Fj7hVhy zB6WVP2vTKN*JCCkLhX`|&!^@s8q6%V=m50x`xa9lzi(0KbQM^rFpZPpBPR+xljlv> z>a6JEh{^4!c7f3YgZPr<#K%|4?eQ)SlUoU?j%KQG>nao)h{dc^=7q!D-6IZeEy;uLenl#=QK#Q zajkAvhv7A%TCIT8)V-=oj9LX6@z9t~qCO(j?u^~Cx@;+1MjnqK9{8@3Qo1zrws{3< zMp_~bdvLqo=hnF<1N`;Vei1EeAm{k=^uMiOfKGwr_5rx??Ymin!1bE9vb3(P;# z>|lNJ$%Sk}w2rK3lWe_52&&zp4&8Jz_1zR0t#-S>JimaYn?ZdXtk^IFOJ zWX*OdpvMI~rNZkvemF>|q8Nj31mE7SNX|~VXE=Sbv_>{ZVaUD)W3Kn9l8vvLaMm9e z`s7LC%`)_^sxsOJA)$60)k90-mChCvk&54`w9Fz9Q*H;`g}=~`UoL`SMzVriL#lDb zY_;>f3)zJqbrp$DVWg{d*3VXD!{RQ00ItxD2pFP!^8)}ip1$ueh+LuKl&%&9_SRjI$c}Sz{cP^KSshz*ZtSI}`6-_D-AMKM(tE&L z^(%OQ^!&llC)z8XTJ5&9w>@2Pqgku|Gt-W>wZcvop!4G|aC2ExD2JqH+r3ao#R-7* zc_F^us%~(w>WMMEuZ*5KtDjy&!V@Ja2VThajEO~^rXtlFc8bKew*DZ}Jys^~FeLm- zMIjO6U1i!}JLgzGfoD+A?v!c!&gqQZmY_r^OIRdKfB5#eMnv` zFd`V9tLMye;&9%t04H%TXtWF&)%C#aNiY!((1Wr?U9E1As!yIRxI1kmD#+d^uhjCo z)~RTzJrGp6a4r+c*@dq@9`5vY{w|IY)4hdc@oD0ayaZu`3p-uVS7n4Mbb~LbvhvYc zmuj9}ISo<#j^p;GQy%-okTN~_YuXu$IuD{G6R>p7=^Y5Tqa8;Adk9Bh^orjrwv=hC zFL#`OcFMG{b+~LT=qmgo4wtLMa51GvzIYj${%Un{ln1jWq7r^1T$WH(`@#t2E>+J- zaB|J*9`EYSfu_DOrl;O|4vu}VlO+-(jZhxG;Iw-i(_AU9%0$KQjmFe%*7A#ON5XXGiJND_=;)cpB3p7k&n6OIPfaQ)++mO6BVvP%UO1`<&avCsOst&1A;(iW?t^kAwiSYa4VW8AouoDAA?&P*&2w2Zv(bcDh{d|ERKzN_{+H(b07$V0o*fd4-mLc8rq` zx^9P;n(_YL9=3hF#L`tlu4e?$z)T?;yizJqbpg(P#_!S~46d(Jj8J8W_K8D-AeUz~ z=#5z0D(Cd5m()?7KPrefl%_~ zI?KmNp}S}DGV{JHRL_9sGX>vF%vHfFJ6r>`T&S({{AjM6Hh*SjG2M+RF9EQ7F|8X$ zcEPSa=hNyU7ibo@`!G{?(*t%`Ji}~OjzVI;-Nu9pz6Pq{bkB9x*l=w2pZ;X6t0M8f z4IkR9c)6Nv8RaGbt(rKw!s9XC#9O<+(q`f)qjfnpb|u>`Vx~9bA}Fc=e-~1E9$ zU6;Gtm1;SRMqo>tZfbXtxzXiR*!o6lQ0wCN#5*xu3_`{qnW5@ONJbpS5Fc;<03Sz1 zX|O!XtSYoE^>!86Mzw%rSt6rEWV6a+N1?irJe$Z+!Pqs3@$8xKvkyQQIRie_EDfxc z1Awa$#V`B8TUy06({l|byf=0B=To{?*Ulb)li64gx*P`}ygOIr$`5c>si%+M2o;DR z2}vIn9nD2*`%%{{51Vy}xerZ_%3~^BCcqV#sqe62*xQ;0Ld(j+uslhI5OJ95JOdLs zv_A#pX6GNz&TJeLBncRx+5wZ-_EAzT%XMaU%R0>GqvkKK@TUN|{QORQwAQwA!sg^8s)&g>2-PF;EXG1Dl zmwE3aJU--M1~I?*?dPjX{`E?wJ)ktV+>~y4;7rc%8&X|`_wRwZh8n^fkM^ea7|x|AsaTLD|RX0<-hi+B{0wWD{v^+mUvkL~!4Wlh-}S zv?Y9v2ZOgwbv+Q?40#2{C*+Wab@OA|U-EK$aF(*!k0VlRoQiIpx9dtZuN!MFuf!J zs3I8e0Pl*u9x{_719XV<=Vp3@dBKEXns<_#-N~Q?y9u-t*1{lbUvZrad_6O3RSFG+ z)GG7HF-gwW+&hn7kfeF6_NU~@Esv&5cbg~aFa~U?zYWmw;jfbo1hRc7439%XDL;BA z9l%b9htO4CfGds4{qhot-sMK9Ts)3&Sa~WyjLO;CytqT){054G8t0S-gwNn_Y{5E3 ztYZG5{tfhT92H?0S;NiWsrQIfCdlS(tm}arg<%zFXxl*vVb1@buWwsc}MC(7*D zVNv>;`T4uM$b{>=oET1^DHygH5~h9GJG=aWGTgu`U~Y}01!qk#Y)t{%aI$$n)6`hX z=TpU=Oq)TLE98f2K)6pU$L~N6>6y7%uZi?l*Je$nO~p)n3g5VOZL)M|#0$4y&YcvE z+VkbX=sexJT2RnW8n5lIe`q;@*s|}rL5dN3qDMy+#@O5N$}!D$;QF!rWIBh`9pxO! zp4HdCM76<6ssI`kZ?I#0iI=imSK0DAfMF0rU>hZ6q`C&5+YGs;a~c2e!_DMz9@kucNtppt?=!%06iGjM~gnLtpM( zHrUQKx0kRF)6yJfC-BBa%{1U2DJl`Sf}UFp%PXm`jQnQUZ7%i|7|OpY@6R`&K2w=i zJZ*PfhH-oKYwc!$g&m*$JfWnYEU(l}=20S^4`R19WTK9ZJZu?yI&*rjG{-w**=t{M z(-9`z!K>VFtQQ|7y{QUYAybbN6Td7zLhfz#3(njcmfu7l?3C&hXCam#n$)y(=L%iJ z_Iu~%bmE7_=7tXswY_a!c>Gk${yjEjENP0cdLAo)!x=snyj!e9MCxNXj*UhnKC@!gAF8PW@z)>lj^4E8SOM_ZQNDMBAbxG~`ejaZ{aS~8^VImW zX$G+fK{;n1ytS+mSTlGM>0=|zrop-Qh&)syQh1_dfbuz|QoN+OLdxy!!|MylujSOK+2oLM?%9Y`7m4Ed&nf-_;;REx z^Y%1i45_zTqroshon636q@%dOzn$I3H6UtQZy>yQRZe4ifyLAEsaEYNH|tB2wD2J> zk&1AnjT7x#er05{WAuR;2}_fu@#RvAbj8uKd$(P&gPPu>WyU&Jt(e!Jlvk>Yw1dZF zo57XzTJN1+y}!@fChfGM|Al*^6kQy<`Z7GGnQJyV%|+tjY2=LERUAyIMi(W9thc6~ zI6C`|H6#c+xWqv-?ZIH{R4!q!gjAQLR-m0}cnY7el<%oSL65TLMa$NcUnvn)e4jhf z<^WbSmECbbbMV!VTMj0jcNdx{D`P$%$*`TWCS&mov8Xe+kw(Aj$m9ld1(w;w#?l8;K(6RqT@zNej z_Q9#%fM*cPUaZJXJ4F>(3d~M;eUh3>Y}pSSsQ5dnY%KOwOV=^0eAN!|p_(?{$B8TA zc7-tnb~gpmirj-10jqrVj*)T-Z)fk`v%b^7eI}O$n66o(GeaQR*iL3{Jr~97W$#$# zTuA5{k#Dp(YL&0pp|BF`H`P35San075ZC@-@WjeT2bw6Aj&^qSxH5`i>kXs=z8UG0 z9`nv1F+{^vqB!X@LLs9tQg-#0!r%%6aVkHu!1Cm*!Ag_!XLK;XQWF8yA3pL(22Ocr zfC~vSEMVYXtWieBH@20%h6liB@*@f?d4YQ#(K%-G6CU?hffu>B<&4Tnms6?*=C+wq zchoeW|EKv!%Ct{K);TLE4c|D=yA-s0B5lS4zMv)otj9F5fZn#VG~Rg9F6;_?Q z{;NMBDTVr#Ep7Pz=knN~s>ylFw^%J9d;(IRG-cNayX_nBj8x#X0Ji&(oOI=uMwy1H zDgJbgS-71eC01}EyKfgi%?qOt*Zul?;o2W|jucy!u-Cdtn?|3b2cR13Z;z1|Y>rur z*@OFw^Y}4eR$wY)1%2sS{iB7NZ!SGIP(yC@B~vj4T(g(C=4-wodgSn9Yq5}VNsW%i zrOD4Z){}OPT>p)=Ekn~F2*@nNcm4Awm8WsLh!36bP0r zxGw_JHBEd&i*1cX=Nx(A1Mpw`L17s-C6F)uytn1sSuN|3!KJnTRjYI$rKMuPxjyJ2 zD=hvmtH{#Gv^<6abC=B6!6wdYs)_yrDqM>-R}ejBgWKE)8mrDKc3A!N+JS{tfQX~P zHDDjs%yZ&Q+eQT5Q99&M@+;hcbJ3O}3>1O07imsOoR0P=u0Q&h)su>_JoN(edzn#7 z8)83u1E2L5=pgG+71`&${~J<}eLi{sS{mX4^4B@7 z&4SFO+Ajx8_V?X}XZKJFW)ClwI ze>oxk2DhUPAJN?Gb2t#n)?tuIN5r#*M4p}e zgD|=i=|c~I5$zD)TTiriedd7I{nuCZ6|zK!S)w`p9=WGh_Z?iUI}osuIJk{EG(m^| z3JS_nExa=-cHd&4LgBw~;qs#O7-XJudY(sL;?Hu8Hr;$$CZmsK(>$_L7^YQ=Ce-S| z@p^y=&Tt`j#gLeSFpl-dg9mXr7KMSwzPLsMw`~gd5BeE|f$ENsqUfM*sXt=GuN4Y< z$bbo!{If@pLxLc%5S-r!g8eMEovnhaj_?8G{tiMM$$2T-6tDMgXT02sQ(?2@Pz&Vt zVZPZ@yH~3_dvQN|_f~rgl3#;d9#APb3h00g*6uXvzb?^-qg4Ut9x0+m!-QVMAu{X9 z+4#0=s(1XEwTDh+QKFvQP@g0>vT?AKA6Mu&8u;lHatQn-hq_{SUj7_|;J*U8EM1Kc zf-RY`pb>zKKWPPVC-Tc32%0R);p;NSE zDwCPJlkyScmIZQmV?=sx2UUDG{Est7W^+}_2F>1@hRDoKlMgFLmbpfQ3KW+zt)WPk z6pncmlJ5Ig4zP3$N#%6YppTTe&Kfs(8ZZN^I!`38eV@8X-hmwdY5K-GxO1 z%n4D2b)`wWI*&&Sy?x=E-*3VXevBTb3ytKNo$-)$Pzb9~&>rHIP$H^7`pi~W%|(NF z`NjHq0_mgb-=K2<_x-c$87NPgJqgt(U2ig0f+@+@2UilJY}bkpML@M9kIdqMjtk6a zLr{N@;D6U%Qsw6g-2sJe4{T zlsP2Gf&%T!!ojm~JC4-7@?}R7?Ea-wkzddt!S?<12MMG*u@1DzJ8~^JDWB}2Tnr(C9DnHp9f{pEshCFT^D-z;%K$po zO3nGBit9E1D&`)E#pl^WN0TeHl06X==(o#%D0F{hb0R~+cFnZ+JEV5s<=JHN??AypnnXTgczaAq$A zqyG`k4coOZ<+_Twb!MGgm^Ab=%n6|q|mzOH1j}mz4 ze5OS^(2pw9qVBK)mbFb>-ZP*;A;Z2b>z5*epCl#F1AJyyg(Z`!UUf?1+J{GdXmdu2I> zg+T%z6X<{5jb5YL1P^!<`B1X-k)qx@!F#IqSGsew$Tkx&yGJISvXgd2C(`<;lU z73@9MDv)SvK1HZN5-h==TkCXxQ7LLU&GKFBbL$kVO8;q}FpMv10b>iKLo}RsZ1syjW9{d@ zUy#!ty7hEs^*m6N4is4uPHivsGQBIGI(-F>zwy&|x2y(1q0RerJ0yLv`t31Wu1lhQ zxMM+rnjPNk87K}HN#D%af&D@8G~+;WsBCx)*L*K4EH)#~GmxSS5&!YCgQ*te=Tkht z1fV>tYHaQ4IcyDL!$X!)@^XxpX7bo8CLRwtPp~U!a*D`RNq`_-36@Tj5WfCv_HTPdkG#~9#N=G13x(xaIVil zD^48Jdj~)4E;!QTO>CdCV>EqpkLEezbhNpEo4}V_#-^Xqh?w??%I%`9eh@T+#7|kp zdri*yDmB8#!%Xqhi~KO%u z8IXT!yOj0B^bw>wd)XN7J}h?mOlyd|In?hJt#`*g}P$04F zfl!nUo1+ADpXEN03acpr!7Np0#O&2*e35HA1GcH!ud;t}butXAz~#5LjiqcC5l9c@ zF}~)EepDcrO*S&lbLyvfAC7G|w#%+0gSanS|D53iYZvTP>E2oW3_L8Hm(ibQ3=^CUlW3!~?l`!0=ksr#zHofmwC`?B4Z>A0Kn`-lF)-O*Aw*{NI9}c- zDy8&T`$W%siSbfH6sFF{#7qY*A

Q8DihK4X*Wy{OMZSvDb4Y(% z)I-Wsh)s-DW^mz8@|%_6-#k9c+ox3Zy%*6&8U7-Wq=(05?G>8=dF~JZ&TtQMecmXE$f=MQ)i04ls)u9918bY1YKQxoT zb4Bo6=;eBpUP;u|4|%5O49Y|yotnTn>JasM`tIg?0OfHWq zcw}Br(rnqYjUT?JP^$gm0YIh#vpVlqnh5JsRyD>&=42A{81m>;Wd*!IzQ z%8T`)>Z1H@hIkD2)q`*EmV$LU-EInI>w)QGp?hXT?cKd9MUL(!7@r>UKq7wFAu#n6 z_u|=dj*Z}i^N=3RY+>g#@B(6Uu+VLJlRTpg)w=U3) zUi$<)Q3@8qq4RU#{$126NldQ1A zSYjBTKD2<%W3oSjDR{iZ1j&RjszM-8;SM^8WZePXiB>~wF|V2fxJd)i*EPOUr0o#; EKlrEFfdBvi literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/agents.tsx b/ui/litellm-dashboard/src/components/agents.tsx index b8a93069908..d6d8b320ec9 100644 --- a/ui/litellm-dashboard/src/components/agents.tsx +++ b/ui/litellm-dashboard/src/components/agents.tsx @@ -91,14 +91,14 @@ const AgentsPanel: React.FC = ({ accessToken, userRole }) => { return (

-
-
-

Agents

-

List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents public.

+
+

Agents

+

List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents public.

+
+
-
{selectedAgentId ? ( diff --git a/ui/litellm-dashboard/src/components/agents/agent_info.tsx b/ui/litellm-dashboard/src/components/agents/agent_info.tsx index 626e352bd38..8d0febd8417 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_info.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_info.tsx @@ -2,11 +2,13 @@ import React, { useState, useEffect } from "react"; import { Card, Title, Text, Button as TremorButton, Tab, TabGroup, TabList, TabPanel, TabPanels} from "@tremor/react"; import { Form, Input, Button as AntButton, message, Spin, Descriptions } from "antd"; import { ArrowLeftIcon } from "@heroicons/react/outline"; -import { getAgentInfo, patchAgentCall } from "../networking"; +import { getAgentInfo, patchAgentCall, getAgentCreateMetadata, AgentCreateInfo } from "../networking"; import { Agent } from "./types"; import AgentFormFields from "./agent_form_fields"; +import DynamicAgentFormFields, { buildDynamicAgentData } from "./dynamic_agent_form_fields"; import { buildAgentDataFromForm, parseAgentForForm } from "./agent_config"; import AgentCostView from "./agent_cost_view"; +import { detectAgentType, parseDynamicAgentForForm } from "./agent_type_utils"; interface AgentInfoViewProps { agentId: string; @@ -26,6 +28,20 @@ const AgentInfoView: React.FC = ({ const [isEditing, setIsEditing] = useState(false); const [isSaving, setIsSaving] = useState(false); const [form] = Form.useForm(); + const [agentTypeMetadata, setAgentTypeMetadata] = useState([]); + const [detectedAgentType, setDetectedAgentType] = useState("a2a"); + + useEffect(() => { + const fetchMetadata = async () => { + try { + const metadata = await getAgentCreateMetadata(); + setAgentTypeMetadata(metadata); + } catch (error) { + console.error("Error fetching agent metadata:", error); + } + }; + fetchMetadata(); + }, []); useEffect(() => { fetchAgentInfo(); @@ -38,7 +54,22 @@ const AgentInfoView: React.FC = ({ try { const data = await getAgentInfo(accessToken, agentId); setAgent(data); + + // Detect agent type + const agentType = detectAgentType(data); + setDetectedAgentType(agentType); + + // Parse form values based on agent type + if (agentType === "a2a") { + form.setFieldsValue(parseAgentForForm(data)); + } else { + const typeInfo = agentTypeMetadata.find(t => t.agent_type === agentType); + if (typeInfo) { + form.setFieldsValue(parseDynamicAgentForForm(data, typeInfo)); + } else { form.setFieldsValue(parseAgentForForm(data)); + } + } } catch (error) { console.error("Error fetching agent info:", error); message.error("Failed to load agent information"); @@ -47,12 +78,38 @@ const AgentInfoView: React.FC = ({ } }; + // Re-parse form when metadata is loaded + useEffect(() => { + if (agent && agentTypeMetadata.length > 0) { + const agentType = detectAgentType(agent); + if (agentType !== "a2a") { + const typeInfo = agentTypeMetadata.find(t => t.agent_type === agentType); + if (typeInfo) { + form.setFieldsValue(parseDynamicAgentForForm(agent, typeInfo)); + } + } + } + }, [agentTypeMetadata, agent]); + + const selectedAgentTypeInfo = agentTypeMetadata.find(t => t.agent_type === detectedAgentType); + const handleUpdate = async (values: any) => { if (!accessToken || !agent) return; setIsSaving(true); try { - const updateData = buildAgentDataFromForm(values, agent); + let updateData: any; + + if (detectedAgentType === "a2a") { + updateData = buildAgentDataFromForm(values, agent); + } else if (selectedAgentTypeInfo) { + updateData = buildDynamicAgentData(values, selectedAgentTypeInfo); + // Preserve the agent_name from form + updateData.agent_name = values.agent_name; + } else { + updateData = buildAgentDataFromForm(values, agent); + } + await patchAgentCall(accessToken, agentId, updateData); message.success("Agent updated successfully"); setIsEditing(false); @@ -192,7 +249,13 @@ const AgentInfoView: React.FC = ({ + {detectedAgentType === "a2a" ? ( + + ) : selectedAgentTypeInfo ? ( + + ) : ( + )}
{ diff --git a/ui/litellm-dashboard/src/components/agents/agent_table.tsx b/ui/litellm-dashboard/src/components/agents/agent_table.tsx index b141122c94e..7634c4396f1 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_table.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_table.tsx @@ -51,18 +51,18 @@ const AgentTable: React.FC = ({ cell: ({ row }) => { const agent = row.original; const name = agent.agent_name || ""; - return ( + return (
- - + + { @@ -201,12 +201,12 @@ const AgentTable: React.FC = ({

No agents found. Create one to get started.

-
- +
+ )} - - + +
); diff --git a/ui/litellm-dashboard/src/components/agents/agent_type_utils.ts b/ui/litellm-dashboard/src/components/agents/agent_type_utils.ts new file mode 100644 index 00000000000..fd04aa4c26c --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/agent_type_utils.ts @@ -0,0 +1,67 @@ +import { Agent } from "./types"; +import { AgentCreateInfo } from "../networking"; + +/** + * Detects the agent type from an agent's litellm_params. + * Returns the agent_type string (e.g., "langgraph", "azure_ai_foundry", "bedrock_agentcore", or "a2a") + */ +export const detectAgentType = (agent: Agent): string => { + const model = agent.litellm_params?.model || ""; + const customProvider = agent.litellm_params?.custom_llm_provider; + + // Check by custom_llm_provider first + if (customProvider === "langgraph") return "langgraph"; + if (customProvider === "azure_ai") return "azure_ai_foundry"; + if (customProvider === "bedrock") return "bedrock_agentcore"; + + // Check by model prefix + if (model.startsWith("langgraph/")) return "langgraph"; + if (model.startsWith("azure_ai/agents/")) return "azure_ai_foundry"; + if (model.startsWith("bedrock/agentcore/")) return "bedrock_agentcore"; + + // Default to a2a + return "a2a"; +}; + +/** + * Parses agent data for dynamic form fields (non-A2A agents). + * Extracts values from litellm_params based on the agent type metadata. + */ +export const parseDynamicAgentForForm = ( + agent: Agent, + agentTypeInfo: AgentCreateInfo +): Record => { + const values: Record = { + agent_name: agent.agent_name, + description: agent.agent_card_params?.description || "", + }; + + // Extract credential field values from litellm_params + for (const field of agentTypeInfo.credential_fields) { + if (field.include_in_litellm_params !== false) { + values[field.key] = agent.litellm_params?.[field.key] || field.default_value || ""; + } else { + // For fields not in litellm_params (like agent_id), try to extract from model string + if (agentTypeInfo.model_template && agent.litellm_params?.model) { + const model = agent.litellm_params.model; + const templateParts = agentTypeInfo.model_template.split("/"); + const modelParts = model.split("/"); + + // Find the placeholder position and extract the value + templateParts.forEach((part, index) => { + if (part === `{${field.key}}` && modelParts[index]) { + values[field.key] = modelParts[index]; + } + }); + } + } + } + + // Extract cost configuration + values.cost_per_query = agent.litellm_params?.cost_per_query; + values.input_cost_per_token = agent.litellm_params?.input_cost_per_token; + values.output_cost_per_token = agent.litellm_params?.output_cost_per_token; + + return values; +}; + diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 2e0edcfcd86..e032649ae29 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -165,7 +165,16 @@ const Navbar: React.FC = ({
- LiteLLM Brand +
+ LiteLLM Brand + + 🎄 + +
{version && ( = ({
)} + {/* Suggested prompts - show when chat is empty and not loading */} + {chatHistory.length === 0 && !isLoading && ( +
+ {(endpointType === EndpointType.A2A_AGENTS + ? ["What can you help me with?", "Tell me about yourself", "What tasks can you perform?"] + : ["Write me a poem", "Explain quantum computing", "Draft a polite email requesting a meeting"] + ).map((prompt) => ( + + ))} +
+ )} +
{/* Left: attachment and code interpreter icons */} diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx index 9ba09535053..85cfd160a23 100644 --- a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx @@ -11,11 +11,25 @@ import type { TokenUsage } from "../chat_ui/ResponseMetrics"; import type { MessageType, VectorStoreSearchResponse } from "../chat_ui/types"; import { makeOpenAIChatCompletionRequest } from "../llm_calls/chat_completion"; import { fetchAvailableModels } from "../llm_calls/fetch_models"; +import { Agent, fetchAvailableAgents } from "../llm_calls/fetch_agents"; +import { makeA2AStreamMessageRequest } from "../llm_calls/a2a_send_message"; import { ComparisonPanel } from "./components/ComparisonPanel"; import { MessageInput } from "./components/MessageInput"; +import { + EndpointId, + EndpointIdType, + getAvailableEndpoints, + getEndpointConfig, + isAgentEndpoint, + hasValidSelection, + getComparisonSelection, + modelOptionsToSelectorOptions, + agentOptionsToSelectorOptions, +} from "./endpoint_config"; export interface ComparisonInstance { id: string; model: string; + agent: string; messages: MessageType[]; isLoading: boolean; tags: string[]; @@ -38,12 +52,13 @@ const GENERIC_FOLLOW_UPS = [ "What are the next steps?", ]; const SUGGESTED_PROMPTS = ["Write me a poem", "Explain quantum computing", "Draft a polite email requesting a meeting"]; -const DEFAULT_ENDPOINT = "/v1/chat/completions"; +const DEFAULT_ENDPOINT = EndpointId.CHAT_COMPLETIONS; export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: CompareUIProps) { const [comparisons, setComparisons] = useState([ { id: "1", model: "", + agent: "", messages: [], isLoading: false, tags: [], @@ -58,6 +73,7 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: { id: "2", model: "", + agent: "", messages: [], isLoading: false, tags: [], @@ -71,7 +87,18 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: }, ]); const [modelOptions, setModelOptions] = useState([]); + const [agentOptions, setAgentOptions] = useState([]); const [isLoadingModels, setIsLoadingModels] = useState(false); + const [isLoadingAgents, setIsLoadingAgents] = useState(false); + const [selectedEndpoint, setSelectedEndpoint] = useState(DEFAULT_ENDPOINT); + + // Derived state from endpoint config + const endpointConfig = getEndpointConfig(selectedEndpoint); + const isA2AMode = isAgentEndpoint(selectedEndpoint); + const selectorOptions = isA2AMode + ? agentOptionsToSelectorOptions(agentOptions) + : modelOptionsToSelectorOptions(modelOptions); + const isLoadingOptions = isA2AMode ? isLoadingAgents : isLoadingModels; const [inputValue, setInputValue] = useState(""); const [uploadedFile, setUploadedFile] = useState(null); const [uploadedFilePreviewUrl, setUploadedFilePreviewUrl] = useState(null); @@ -134,6 +161,37 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: active = false; }; }, [effectiveApiKey]); + + // Fetch agents when A2A mode is selected + useEffect(() => { + let active = true; + const loadAgents = async () => { + if (!effectiveApiKey || !isA2AMode) { + setAgentOptions([]); + return; + } + setIsLoadingAgents(true); + try { + const agents = await fetchAvailableAgents(effectiveApiKey); + if (!active) return; + setAgentOptions(agents); + } catch (error) { + console.error("CompareUI: failed to fetch agents", error); + if (active) { + setAgentOptions([]); + } + } finally { + if (active) { + setIsLoadingAgents(false); + } + } + }; + loadAgents(); + return () => { + active = false; + }; + }, [effectiveApiKey, isA2AMode]); + useEffect(() => { if (modelOptions.length === 0) { return; @@ -160,10 +218,12 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: if (comparisons.length >= maxComparisons) { return; } - const fallback = modelOptions[comparisons.length % (modelOptions.length || 1)] ?? ""; + const fallbackModel = modelOptions[comparisons.length % (modelOptions.length || 1)] ?? ""; + const fallbackAgent = agentOptions[comparisons.length % (agentOptions.length || 1)]?.agent_name ?? ""; const newComparison: ComparisonInstance = { id: Date.now().toString(), - model: fallback, + model: fallbackModel, + agent: fallbackAgent, messages: [], isLoading: false, tags: [], @@ -430,8 +490,9 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: if (targetComparisons.length === 0) { return; } - if (targetComparisons.some((comparison) => !comparison.model)) { - NotificationsManager.fromBackend("Select a model before sending a message."); + // Validate selection based on endpoint type + if (targetComparisons.some((comparison) => !hasValidSelection(comparison, selectedEndpoint))) { + NotificationsManager.fromBackend(endpointConfig.validationMessage); return; } @@ -450,6 +511,8 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: { id: string; model: string; + agent: string; + inputMessage: string; traceId: string; tags: string[]; vectorStores: string[]; @@ -472,6 +535,8 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: preparedTargets.set(comparison.id, { id: comparison.id, model: comparison.model, + agent: comparison.agent, + inputMessage: trimmed, traceId, tags: comparison.tags, vectorStores: comparison.vectorStores, @@ -508,26 +573,55 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: const guardrails = prepared.guardrails.length > 0 ? prepared.guardrails : undefined; const comparison = comparisons.find((c) => c.id === prepared.id); const useAdvancedParams = comparison?.useAdvancedParams ?? false; - makeOpenAIChatCompletionRequest( - prepared.apiChatHistory, - (chunk, model) => appendAssistantChunk(prepared.id, chunk, model), - prepared.model, - effectiveApiKey, - tags, - undefined, - (content) => appendReasoningContent(prepared.id, content), - (time) => updateTimingDataForComparison(prepared.id, time), - (usage) => updateUsageDataForComparison(prepared.id, usage), - prepared.traceId, - vectorStoreIds, - guardrails, - undefined, - undefined, - (searchResults) => updateSearchResultsForComparison(prepared.id, searchResults), - useAdvancedParams ? prepared.temperature : undefined, - useAdvancedParams ? prepared.maxTokens : undefined, - (latency) => updateTotalLatencyForComparison(prepared.id, latency), - ) + + // Use A2A or chat completion based on endpoint + const requestPromise = isA2AMode + ? makeA2AStreamMessageRequest( + prepared.agent, + prepared.inputMessage, + (text, model) => { + // A2A sends full accumulated text, so replace instead of append + setComparisons((prev) => + prev.map((c) => { + if (c.id !== prepared.id) return c; + const messages = [...c.messages]; + const last = messages[messages.length - 1]; + if (last && last.role === "assistant") { + messages[messages.length - 1] = { ...last, content: text, model: last.model ?? model }; + } else { + messages.push({ role: "assistant", content: text, model }); + } + return { ...c, messages }; + }), + ); + }, + effectiveApiKey, + undefined, + (time) => updateTimingDataForComparison(prepared.id, time), + (latency) => updateTotalLatencyForComparison(prepared.id, latency), + ) + : makeOpenAIChatCompletionRequest( + prepared.apiChatHistory, + (chunk, model) => appendAssistantChunk(prepared.id, chunk, model), + prepared.model, + effectiveApiKey, + tags, + undefined, + (content) => appendReasoningContent(prepared.id, content), + (time) => updateTimingDataForComparison(prepared.id, time), + (usage) => updateUsageDataForComparison(prepared.id, usage), + prepared.traceId, + vectorStoreIds, + guardrails, + undefined, + undefined, + (searchResults) => updateSearchResultsForComparison(prepared.id, searchResults), + useAdvancedParams ? prepared.temperature : undefined, + useAdvancedParams ? prepared.maxTokens : undefined, + (latency) => updateTotalLatencyForComparison(prepared.id, latency), + ); + + requestPromise .catch((error) => { const errorMessage = error instanceof Error ? error.message : String(error); console.error("CompareUI: failed to fetch response", error); @@ -618,11 +712,20 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }:
Endpoint - - - +
{uploadedFile && ( diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/components/ComparisonPanel.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/components/ComparisonPanel.tsx index c9a99b3d5ce..5073d4549cc 100644 --- a/ui/litellm-dashboard/src/components/playground/compareUI/components/ComparisonPanel.tsx +++ b/ui/litellm-dashboard/src/components/playground/compareUI/components/ComparisonPanel.tsx @@ -2,11 +2,13 @@ import { Settings, X } from "lucide-react"; import { useState } from "react"; import { ComparisonInstance } from "../CompareUI"; import { MessageDisplay } from "./MessageDisplay"; -import { ModelSelector } from "./ModelSelector"; +import { UnifiedSelector } from "./UnifiedSelector"; import TagSelector from "../../../tag_management/TagSelector"; import VectorStoreSelector from "../../../vector_store_management/VectorStoreSelector"; import GuardrailSelector from "../../../guardrails/GuardrailSelector"; import { Checkbox, Divider, Popover, Slider } from "antd"; +import { SelectorOption, EndpointConfig, isAgentEndpoint, getComparisonSelection } from "../endpoint_config"; + interface ComparisonPanelProps { comparison: ComparisonInstance; onUpdate: ( @@ -15,8 +17,9 @@ interface ComparisonPanelProps { ) => void; onRemove: () => void; canRemove: boolean; - modelOptions: string[]; - isLoadingModels: boolean; + selectorOptions: SelectorOption[]; + isLoadingOptions: boolean; + endpointConfig: EndpointConfig; apiKey: string; } export function ComparisonPanel({ @@ -24,10 +27,13 @@ export function ComparisonPanel({ onUpdate, onRemove, canRemove, - modelOptions, - isLoadingModels, + selectorOptions, + isLoadingOptions, + endpointConfig, apiKey, }: ComparisonPanelProps) { + const isA2AMode = isAgentEndpoint(endpointConfig.id); + const currentSelection = getComparisonSelection(comparison, endpointConfig.id); const [popoverVisible, setPopoverVisible] = useState(false); const handleSyncChange = (checked: boolean) => { @@ -194,14 +200,13 @@ export function ComparisonPanel({
- - onUpdate({ - model, - }) + + onUpdate(isA2AMode ? { agent: value } : { model: value }) } />
diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.tsx new file mode 100644 index 00000000000..c531eed3737 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.tsx @@ -0,0 +1,48 @@ +/** + * Unified selector component that handles both model and agent selection + * based on the current endpoint configuration. + */ + +import { Select, Spin } from "antd"; +import { SelectorOption, EndpointConfig } from "../endpoint_config"; + +interface UnifiedSelectorProps { + value: string; + options: SelectorOption[]; + loading: boolean; + config: EndpointConfig; + onChange: (value: string) => void; +} + +export function UnifiedSelector({ + value, + options, + loading, + config, + onChange, +}: UnifiedSelectorProps) { + return ( +