diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md new file mode 100644 index 00000000000..fbdeaa191fd --- /dev/null +++ b/docs/my-website/docs/mcp.md @@ -0,0 +1,320 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# /mcp Model Context Protocol [Beta] + +Use Model Context Protocol with LiteLLM. + +## Overview + + +LiteLLM acts as a MCP bridge to utilize **MCP tools** with **all LiteLLM supported models**. LiteLLM offers a client that exposes a tools method for retrieving tools from a MCP server. + +## Usage + + + + +```python +import asyncio +import litellm +from litellm import experimental_create_mcp_client +from litellm.mcp_stdio import experimental_stdio_mcp_transport + +async def main(): + client_one = None + + try: + # Initialize an MCP client to connect to a `stdio` MCP server: + transport = experimental_stdio_mcp_transport( + command='node', + args=['src/stdio/dist/server.js'] + ) + client_one = await experimental_create_mcp_client( + transport=transport + ) + + tools = await client_one.list_tools(format="openai") + response = await litellm.completion( + model="gpt-4o", + tools=tools, + messages=[ + { + "role": "user", + "content": "Find products under $100" + } + ] + ) + + print(response.text) + except Exception as error: + print(error) + finally: + await asyncio.gather( + client_one.close() if client_one else asyncio.sleep(0), + ) + +if __name__ == "__main__": + asyncio.run(main()) +``` + + + + +```python +import asyncio +from openai import OpenAI +from litellm import experimental_create_mcp_client +from litellm.mcp_stdio import experimental_stdio_mcp_transport + +async def main(): + client_one = None + + try: + # Initialize an MCP client to connect to a `stdio` MCP server: + transport = experimental_stdio_mcp_transport( + command='node', + args=['src/stdio/dist/server.js'] + ) + client_one = await experimental_create_mcp_client( + transport=transport + ) + + # Get tools from MCP client + tools = await client_one.list_tools(format="openai") + + # Use OpenAI client connected to LiteLLM Proxy Server + client = OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" + ) + response = client.chat.completions.create( + model="gpt-4", + tools=tools, + messages=[ + { + "role": "user", + "content": "Find products under $100" + } + ] + ) + + print(response.choices[0].message.content) + except Exception as error: + print(error) + finally: + await asyncio.gather( + client_one.close() if client_one else asyncio.sleep(0), + ) + +if __name__ == "__main__": + asyncio.run(main()) +``` + + + + +## Advanced + +### Expose MCP tools on LiteLLM Proxy Server + +This allows you to define tools that can be called by any MCP compatible client. Define your mcp_tools with LiteLLM and all your clients can list and call available tools. + +#### How it works + +LiteLLM exposes the following MCP endpoints: + +- `/mcp/list_tools` - List all available tools +- `/mcp/call_tool` - Call a specific tool with the provided arguments + +When MCP clients connect to LiteLLM they can follow this workflow: + +1. Connect to the LiteLLM MCP server +2. List all available tools on LiteLLM +3. Client makes LLM API request with tool call(s) +4. LLM API returns which tools to call and with what arguments +5. MCP client makes tool calls to LiteLLM +6. LiteLLM makes the tool calls to the appropriate handlers +7. LiteLLM returns the tool call results to the MCP client + +#### Usage + +#### 1. Define your tools on mcp_tools + +LiteLLM allows you to define your tools on the `mcp_tools` section in your config.yaml file. All tools listed here will be available to MCP clients (when they connect to LiteLLM and call `list_tools`). + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: sk-xxxxxxx + + + +mcp_tools: + - name: "get_current_time" + description: "Get the current time" + input_schema: { + "type": "object", + "properties": { + "format": { + "type": "string", + "description": "The format of the time to return", + "enum": ["short"] + } + } + } + handler: "mcp_tools.get_current_time" +``` + +#### 2. Define a handler for your tool + +Create a new file called `mcp_tools.py` and add this code. The key method here is `get_current_time` which gets executed when the `get_current_time` tool is called. + +```python +# mcp_tools.py + +from datetime import datetime + +def get_current_time(format: str = "short"): + """ + Simple handler for the 'get_current_time' tool. + + Args: + format (str): The format of the time to return ('short'). + + Returns: + str: The current time formatted as 'HH:MM'. + """ + # Get the current time + current_time = datetime.now() + + # Format the time as 'HH:MM' + return current_time.strftime('%H:%M') +``` + +#### 3. Start LiteLLM Gateway + + + + +Mount your `mcp_tools.py` on the LiteLLM Docker container. + +```shell +docker run -d \ + -p 4000:4000 \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + --name my-app \ + -v $(pwd)/my_config.yaml:/app/config.yaml \ + -v $(pwd)/mcp_tools.py:/app/mcp_tools.py \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ +``` + + + + + +```shell +litellm --config config.yaml --detailed_debug +``` + + + + + +#### 4. Make an LLM API request + + + +```python +import asyncio +from langchain_mcp_adapters.tools import load_mcp_tools +from langchain_openai import ChatOpenAI +from langgraph.prebuilt import create_react_agent +from mcp import ClientSession +from mcp.client.sse import sse_client + + +async def main(): + # Initialize the model with your API key + model = ChatOpenAI(model="gpt-4o") + + # Connect to the MCP server + async with sse_client(url="http://localhost:4000/mcp/") as (read, write): + async with ClientSession(read, write) as session: + # Initialize the session + print("Initializing session...") + await session.initialize() + print("Session initialized") + + # Load available tools from MCP + print("Loading tools...") + tools = await load_mcp_tools(session) + print(f"Loaded {len(tools)} tools") + + # Create a ReAct agent with the model and tools + agent = create_react_agent(model, tools) + + # Run the agent with a user query + user_query = "What's the weather in Tokyo?" + print(f"Asking: {user_query}") + agent_response = await agent.ainvoke({"messages": user_query}) + print("Agent response:") + print(agent_response) + + +if __name__ == "__main__": + asyncio.run(main()) + +``` + + +### Specification for `mcp_tools` + +The `mcp_tools` section in your LiteLLM config defines tools that can be called by MCP-compatible clients. + +#### Tool Definition Format + +```yaml +mcp_tools: + - name: string # Required: Name of the tool + description: string # Required: Description of what the tool does + input_schema: object # Required: JSON Schema defining the tool's input parameters + handler: string # Required: Path to the function that implements the tool +``` + +#### Field Details + +- `name`: A unique identifier for the tool +- `description`: A clear description of what the tool does, used by LLMs to determine when to call it +- `input_schema`: JSON Schema object defining the expected input parameters +- `handler`: String path to the Python function that implements the tool (e.g., "module.submodule.function_name") + +#### Example Tool Definition + +```yaml +mcp_tools: + - name: "get_current_time" + description: "Get the current time in a specified format" + input_schema: { + "type": "object", + "properties": { + "format": { + "type": "string", + "description": "The format of the time to return", + "enum": ["short", "long", "iso"] + }, + "timezone": { + "type": "string", + "description": "The timezone to use (e.g., 'UTC', 'America/New_York')", + "default": "UTC" + } + }, + "required": ["format"] + } + handler: "mcp_tools.get_current_time" +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 1df988f372d..a1499cc0b01 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -293,6 +293,7 @@ const sidebars = { "text_completion", "embedding/supported_embedding", "anthropic_unified", + "mcp", { type: "category", label: "/images",