+
+ +### Add HTTP MCP Server + +This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE. + + + +
+
+ +### Add SSE MCP Server + +This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE. + + + +
+
+ +### Add STDIO MCP Server For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format: diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index 4801fa8e1b0..b4c9a2bd1ad 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs |----------|----------|-------------|---------| | `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` | | `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` | -| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) | +| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` | ### Endpoint Resolution -The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`: +The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST` - **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel` - **EU Region**: `https://cloud.langfuse.com/api/public/otel` -- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel` +- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel` ## Usage @@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." # Use EU region -os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region -# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region (default) +os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region +# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint # Or use self-hosted instance -# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com" +# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com" litellm.callbacks = ["langfuse_otel"] ``` @@ -98,14 +98,16 @@ import litellm # Get keys for your project from the project settings page: https://cloud.langfuse.com os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." -os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region -# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region +os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region +# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region +# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint LANGFUSE_AUTH = base64.b64encode( f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode() ).decode() -os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel" +host = os.environ.get("LANGFUSE_OTEL_HOST") +os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel" os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}" litellm.callbacks = ["langfuse_otel"] @@ -120,7 +122,8 @@ Add the integration to your proxy configuration: ```bash export LANGFUSE_PUBLIC_KEY="pk-lf-..." export LANGFUSE_SECRET_KEY="sk-lf-..." -export LANGFUSE_HOST="https://us.cloud.langfuse.com" # Default US region +export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region +# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint ``` 2. Setup config.yaml diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index a7a9dc30013..820c2906bf0 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -55,8 +55,29 @@ import os os.environ["ANTHROPIC_API_KEY"] = "your-api-key" # os.environ["ANTHROPIC_API_BASE"] = "" # [OPTIONAL] or 'ANTHROPIC_BASE_URL' +# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending ``` +### Custom API Base + +When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL. + +If your custom endpoint already includes the full path or doesn't follow Anthropic's standard URL structure, you can disable this automatic suffix appending: + +```python +import os + +os.environ["ANTHROPIC_API_BASE"] = "https://my-custom-endpoint.com/custom/path" +os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # Prevents automatic suffix +``` + +Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`: +- Base URL `https://my-proxy.com` β `https://my-proxy.com/v1/messages` +- Base URL `https://my-proxy.com/api` β `https://my-proxy.com/api/v1/messages` + +With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`: +- Base URL `https://my-proxy.com/custom/path` β `https://my-proxy.com/custom/path` (unchanged) + ## Usage ```python diff --git a/docs/my-website/docs/providers/datarobot.md b/docs/my-website/docs/providers/datarobot.md new file mode 100644 index 00000000000..3f4a0f71ac4 --- /dev/null +++ b/docs/my-website/docs/providers/datarobot.md @@ -0,0 +1,43 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataRobot +LiteLLM supports all models from [DataRobot](https://datarobot.com). Select `datarobot` as the provider to route your request through the `datarobot` OpenAI-compatible endpoint using the upstream [official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md). + +## Usage + +### Environment variables +```python +import os +from litellm import completion +os.environ["DATAROBOT_API_KEY"] = "" +os.environ["DATAROBOT_API_BASE"] = "" # [OPTIONAL] defaults to https://app.datarobot.com + +response = completion( + model="datarobot/openai/gpt-4o-mini", + messages=messages, + ) + + +### Completion +```python +import litellm +import os + +response = litellm.completion( + model="datarobot/openai/gpt-4o-mini", # add `datarobot/` prefix to model so litellm knows to route through DataRobot + messages=[ + { + "role": "user", + "content": "Hey, how's it going?", + } + ], +) +print(response) +``` + +## DataRobot completion models + +π¨ LiteLLM supports _all_ DataRobot LLM gateway models. To get a list for your installation and user account, send the following CURL command: +`curl -X GET -H "Authorization: Bearer $DATAROBOT_API_TOKEN" "$DATAROBOT_ENDPOINT/genai/llmgw/catalog/" | jq | grep 'model":'DATAROBOT_ENDPOINT/genai/llmgw/catalog/` + diff --git a/docs/my-website/docs/providers/google_ai_studio/image_gen.md b/docs/my-website/docs/providers/google_ai_studio/image_gen.md index f4e96d5225a..31b1766e450 100644 --- a/docs/my-website/docs/providers/google_ai_studio/image_gen.md +++ b/docs/my-website/docs/providers/google_ai_studio/image_gen.md @@ -42,7 +42,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here" # Generate a single image response = litellm.image_generation( - model="gemini/imagen-4.0-generate-preview-06-06", + model="gemini/imagen-4.0-generate-001", prompt="A cute baby sea otter swimming in crystal clear water" ) @@ -64,7 +64,7 @@ async def generate_image(): # Generate image asynchronously response = await litellm.aimage_generation( - model="gemini/imagen-4.0-generate-preview-06-06", + model="gemini/imagen-4.0-generate-001", prompt="A beautiful sunset over mountains with vibrant colors", n=1, ) @@ -89,7 +89,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here" # Generate image with additional parameters response = litellm.image_generation( - model="gemini/imagen-4.0-generate-preview-06-06", + model="gemini/imagen-4.0-generate-001", prompt="A futuristic cityscape at night with neon lights", n=1, size="1024x1024", @@ -112,7 +112,7 @@ for image in response.data: model_list: - model_name: google-imagen litellm_params: - model: gemini/imagen-4.0-generate-preview-06-06 + model: gemini/imagen-4.0-generate-001 api_key: os.environ/GEMINI_API_KEY model_info: mode: image_generation @@ -198,7 +198,7 @@ Google AI Studio Image Generation supports the following OpenAI-compatible param | Parameter | Type | Description | Default | Example | |-----------|------|-------------|---------|---------| | `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` | -| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-preview-06-06"` | +| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` | | `n` | integer | Number of images to generate (1-4) | `1` | `2` | | `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` | diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index 28beb71094a..6fc1835154a 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -44,7 +44,11 @@ response = completion( oci_user=
+
+ +https://vercel.com/docs/ai-gateway + +**We support ALL models available through Vercel AI Gateway, just set `vercel_ai_gateway/` as a prefix when sending completion requests** + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key +# OR +os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication +``` + +## Optional Variables + +```python showLineNumbers title="Environment Variables" +os.environ["VERCEL_SITE_URL"] = "" # your site url +# OR +os.environ["VERCEL_APP_NAME"] = "" # your app name +``` + +Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key. + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Vercel AI Gateway Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Vercel AI Gateway call +response = completion( + model="vercel_ai_gateway/openai/gpt-4o", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Vercel AI Gateway Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key" + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Vercel AI Gateway call with streaming +response = completion( + model="vercel_ai_gateway/openai/gpt-4o", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Usage - LiteLLM Proxy + +Add the following to your LiteLLM Proxy configuration file: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gpt-4o-gateway + litellm_params: + model: vercel_ai_gateway/openai/gpt-4o + api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY + + - model_name: claude-4-sonnet-gateway + litellm_params: + model: vercel_ai_gateway/anthropic/claude-4-sonnet + api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY +``` + +Start your LiteLLM Proxy server: + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +
-
+
= ({
]);
};
+ const updateChatImageUI = (imageUrl: string, model?: string) => {
+ setChatHistory((prev) => {
+ const last = prev[prev.length - 1];
+ // If the last message is from assistant and has content, add image to it
+ if (last && last.role === "assistant" && !last.isImage) {
+ const updated = {
+ ...last,
+ image: {
+ url: imageUrl,
+ detail: "auto"
+ },
+ model: last.model ?? model
+ };
+ return [...prev.slice(0, -1), updated];
+ } else {
+ // Otherwise create a new assistant message with just the image
+ return [
+ ...prev,
+ {
+ role: "assistant",
+ content: "",
+ model,
+ image: {
+ url: imageUrl,
+ detail: "auto"
+ }
+ }
+ ];
+ }
+ });
+ };
+
const handleKeyDown = (event: React.KeyboardEvent) => {
if (event.key === 'Enter' && !event.shiftKey) {
event.preventDefault(); // Prevent default to avoid newline
@@ -611,7 +643,8 @@ const ChatUI: React.FC = ({
traceId,
selectedVectorStores.length > 0 ? selectedVectorStores : undefined,
selectedGuardrails.length > 0 ? selectedGuardrails : undefined,
- selectedMCPTools // Pass the selected tool directly
+ selectedMCPTools, // Pass the selected tool directly
+ updateChatImageUI // Pass the image callback
);
} else if (endpointType === EndpointType.IMAGE) {
// For image generation
@@ -1057,6 +1090,18 @@ const ChatUI: React.FC = ({
>
{typeof message.content === "string" ? message.content : ""}
+
+ {/* Show generated image from chat completions */}
+ {message.image && (
+ = ({
setPage,
userRole,
defaultSelectedKey,
+ collapsed = false,
}) => {
- const [collapsed, setCollapsed] = useState(false);
- const toggleCollapse = () => {
- setCollapsed(!collapsed);
- };
// Note: If a menu item does not have a role, it is visible to all roles.
const menuItems: MenuItem[] = [
- { key: "1", page: "api-keys", label: "Virtual Keys", icon: = ({
collapsible
trigger={null}
style={{
- transition: 'all 0.2s',
+ transition: 'all 0.3s cubic-bezier(0.4, 0, 0.2, 1)', // Material Design easing
position: 'relative',
}}
>
- ,
- label: '',
- onClick: toggleCollapse,
- style: {
- cursor: 'pointer',
+
- {filteredMenuItems[0]?.label}
-
- ),
- onClick: !filteredMenuItems[0]?.children ? () => {
- const newSearchParams = new URLSearchParams(window.location.search);
- newSearchParams.set('page', filteredMenuItems[0]?.page || '');
- window.history.pushState(null, '', `?${newSearchParams.toString()}`);
- setPage(filteredMenuItems[0]?.page || '');
- } : undefined
},
- // Rest of the menu items (or all items when collapsed)
- ...(collapsed ? filteredMenuItems : filteredMenuItems.slice(1)).map(item => ({
+ }}
+ >
+
+ 
+
+ )}
)}
diff --git a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx
index e611294c8de..70f5e36f863 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx
+++ b/ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.tsx
@@ -17,7 +17,8 @@ export async function makeOpenAIChatCompletionRequest(
traceId?: string,
vector_store_ids?: string[],
guardrails?: string[],
- selectedMCPTool?: string
+ selectedMCPTool?: string,
+ onImageGenerated?: (imageUrl: string, model?: string) => void
) {
// base url should be the current base_url
const isLocal = process.env.NODE_ENV === "development";
@@ -103,6 +104,12 @@ export async function makeOpenAIChatCompletionRequest(
fullResponseContent += content;
}
+ // Process image generation if present
+ if (delta && delta.image && onImageGenerated) {
+ console.log("Image generated:", delta.image);
+ onImageGenerated(delta.image.url, chunk.model);
+ }
+
// Process reasoning content if present - using type assertion
if (delta && delta.reasoning_content) {
const reasoningContent = delta.reasoning_content;
diff --git a/ui/litellm-dashboard/src/components/chat_ui/types.ts b/ui/litellm-dashboard/src/components/chat_ui/types.ts
index e7b25f8a38a..e0eba9f2aad 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/types.ts
+++ b/ui/litellm-dashboard/src/components/chat_ui/types.ts
@@ -7,6 +7,10 @@ export interface Delta {
audio?: any;
refusal?: any;
provider_specific_fields?: any;
+ image?: {
+ url: string;
+ detail: string;
+ };
}
export interface CompletionTokensDetails {
@@ -67,6 +71,10 @@ export interface MessageType {
};
toolName?: string;
imagePreviewUrl?: string; // For storing image preview URL in chat history
+ image?: {
+ url: string;
+ detail: string;
+ };
}
export interface MultimodalContent {
diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx
index 4715b3b642e..43c7a210637 100644
--- a/ui/litellm-dashboard/src/components/leftnav.tsx
+++ b/ui/litellm-dashboard/src/components/leftnav.tsx
@@ -30,6 +30,7 @@ import {
} from '@ant-design/icons';
import { old_admin_roles, v2_admin_role_names, all_admin_roles, rolesAllowedToSeeUsage, rolesWithWriteAccess, internalUserRoles, isAdminRole } from '../utils/roles';
import UsageIndicator from './usage_indicator';
+import { ConfigProvider } from 'antd';
const { Sider } = Layout;
// Define the props type
@@ -38,6 +39,7 @@ interface SidebarProps {
setPage: (page: string) => void;
userRole: string;
defaultSelectedKey: string;
+ collapsed?: boolean;
}
// Create a more comprehensive menu item configuration
@@ -57,65 +59,73 @@ const Sidebar: React.FC
+ {
- e.stopPropagation();
- toggleCollapse();
- }}
- style={{
- cursor: 'pointer',
- padding: '8px 12px',
- borderRadius: '4px',
- display: 'flex',
- alignItems: 'center',
- justifyContent: 'center',
- fontSize: '12px',
- color: '#888',
- transition: 'all 0.2s',
- marginLeft: '8px',
- marginRight: '-8px',
- }}
- onMouseEnter={(e) => {
- e.currentTarget.style.background = 'rgba(0, 0, 0, 0.04)';
- e.currentTarget.style.color = '#666';
- }}
- onMouseLeave={(e) => {
- e.currentTarget.style.background = 'transparent';
- e.currentTarget.style.color = '#888';
- }}
- title="Collapse navigation sidebar"
- >
-
-