scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

[Feat] UI Allow testing /v1/messages on the Test Key Page (#11930)

Browse source 
* Add ANTHROPIC_MESSAGES to chat UI

* add anthropic-ai/sdk

* add anthropic messages to UI

* add makeAnthropicMessagesRequest

* fixes for endpoint selector
This commit is contained in:
Ishaan Jaff 2025-06-20 13:31:24 -07:00 • committed by GitHub
parent cfee865f16
commit 2a698062e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 169 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
ui/litellm-dashboard/package-lock.json generated
View file

@ -8,6 +8,7 @@
"name": "litellm-dashboard",
"version": "0.1.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.54.0",
"@headlessui/react": "^1.7.18",
"@headlessui/tailwindcss": "^0.2.0",
"@heroicons/react": "^1.0.6",
@ -138,6 +139,14 @@
"react": ">=16.9.0"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.54.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.54.0.tgz",
"integrity": "sha512-xyoCtHJnt/qg5GG6IgK+UJEndz8h8ljzt/caKXmq3LfBF81nC/BW6E4x2rOWCZcvsLyVW+e8U5mtIr6UCE/kJw==",
"bin": {
"anthropic-ai-sdk": "bin/cli"
}
},
"node_modules/@babel/runtime": {
"version": "7.27.0",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz",

1
ui/litellm-dashboard/package.json
View file

@ -9,6 +9,7 @@
"lint": "next lint"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.54.0",
"@headlessui/react": "^1.7.18",
"@headlessui/tailwindcss": "^0.2.0",
"@heroicons/react": "^1.0.6",

58
ui/litellm-dashboard/src/components/chat_ui.tsx
View file

@ -29,6 +29,7 @@ import { makeOpenAIChatCompletionRequest } from "./chat_ui/llm_calls/chat_comple
import { makeOpenAIImageGenerationRequest } from "./chat_ui/llm_calls/image_generation";
import { makeOpenAIImageEditsRequest } from "./chat_ui/llm_calls/image_edits";
import { makeOpenAIResponsesRequest } from "./chat_ui/llm_calls/responses_api";
import { makeAnthropicMessagesRequest } from "./chat_ui/llm_calls/anthropic_messages";
import { fetchAvailableModels, ModelGroup } from "./chat_ui/llm_calls/fetch_models";
import { litellmModeMapping, ModelMode, EndpointType, getEndpointType } from "./chat_ui/mode_endpoint_mapping";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
@ -123,12 +124,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
if (uniqueModels.length > 0) {
setModelInfo(uniqueModels);
setSelectedModel(uniqueModels[0].model_group);
// Auto-set endpoint based on the first model's mode
if (uniqueModels[0].mode) {
const initialEndpointType = determineEndpointType(uniqueModels[0].model_group, uniqueModels);
setEndpointType(initialEndpointType);
}
}
} catch (error) {
console.error("Error fetching model info:", error);
@ -407,6 +403,23 @@ const ChatUI: React.FC<ChatUIProps> = ({
selectedVectorStores.length > 0 ? selectedVectorStores : undefined,
selectedGuardrails.length > 0 ? selectedGuardrails : undefined
);
} else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) {
const apiChatHistory = [...chatHistory.filter(msg => !msg.isImage).map(({ role, content }) => ({ role, content })), newUserMessage];
await makeAnthropicMessagesRequest(
apiChatHistory,
(role, delta, model) => updateTextUI(role, delta, model),
selectedModel,
effectiveApiKey,
selectedTags,
signal,
updateReasoningContent,
updateTimingData,
updateUsageData,
traceId,
selectedVectorStores.length > 0 ? selectedVectorStores : undefined,
selectedGuardrails.length > 0 ? selectedGuardrails : undefined
);
}
}
} catch (error) {
@ -449,18 +462,10 @@ const ChatUI: React.FC<ChatUIProps> = ({
console.log(`selected ${value}`);
setSelectedModel(value);
// Use the utility function to determine the endpoint type
if (value !== 'custom') {
const newEndpointType = determineEndpointType(value, modelInfo);
setEndpointType(newEndpointType);
}
setShowCustomModelInput(value === 'custom');
};
const handleEndpointChange = (value: string) => {
setEndpointType(value);
};
const antIcon = <LoadingOutlined style={{ fontSize: 24 }} spin />;
@ -543,7 +548,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
</Text>
<EndpointSelector
endpointType={endpointType}
onEndpointChange={handleEndpointChange}
onEndpointChange={setEndpointType}
className="mb-4"
/>
</div>
@ -762,8 +767,10 @@ const ChatUI: React.FC<ChatUIProps> = ({
onChange={(e) => setInputMessage(e.target.value)}
onKeyDown={handleKeyDown}
placeholder={
endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES
? "Type your message... (Shift+Enter for new line)"
endpointType === EndpointType.CHAT ||
endpointType === EndpointType.RESPONSES ||
endpointType === EndpointType.ANTHROPIC_MESSAGES
? "Type your message... (Shift+Enter for new line)"
: endpointType === EndpointType.IMAGE_EDITS
? "Describe how you want to edit the image..."
: "Describe the image you want to generate..."
@ -785,10 +792,21 @@ const ChatUI: React.FC<ChatUIProps> = ({
<Button
onClick={handleSendMessage}
className="ml-2 text-white"
icon={endpointType === EndpointType.CHAT ? SendOutlined : RobotOutlined}
icon={
endpointType === EndpointType.CHAT ||
endpointType === EndpointType.RESPONSES ||
endpointType === EndpointType.ANTHROPIC_MESSAGES
? SendOutlined
: RobotOutlined
}
>
{endpointType === EndpointType.CHAT ? "Send" :
endpointType === EndpointType.IMAGE_EDITS ? "Edit" : "Generate"}
{endpointType === EndpointType.CHAT ||
endpointType === EndpointType.RESPONSES ||
endpointType === EndpointType.ANTHROPIC_MESSAGES
? "Send"
: endpointType === EndpointType.IMAGE_EDITS
? "Edit"
: "Generate"}
</Button>
)}
</div>

2
ui/litellm-dashboard/src/components/chat_ui/EndpointSelector.tsx
View file

@ -21,6 +21,7 @@ const EndpointSelector: React.FC<EndpointSelectorProps> = ({
const endpointOptions = [
{ value: EndpointType.CHAT, label: '/v1/chat/completions' },
{ value: EndpointType.RESPONSES, label: '/v1/responses' },
{ value: EndpointType.ANTHROPIC_MESSAGES, label: '/v1/messages' },
{ value: EndpointType.IMAGE, label: '/v1/images/generations' },
{ value: EndpointType.IMAGE_EDITS, label: '/v1/images/edits' },
];
@ -29,6 +30,7 @@ const EndpointSelector: React.FC<EndpointSelectorProps> = ({
<div className={className}>
<Text>Endpoint Type:</Text>
<Select
showSearch
value={endpointType}
style={{ width: "100%" }}
onChange={onEndpointChange}

116
ui/litellm-dashboard/src/components/chat_ui/llm_calls/anthropic_messages.tsx Normal file
View file

@ -0,0 +1,116 @@
import { message } from "antd";
import Anthropic from "@anthropic-ai/sdk";
import { MessageType } from "../types";
import { TokenUsage } from "../ResponseMetrics";
import { getProxyBaseUrl } from "@/components/networking";
export async function makeAnthropicMessagesRequest(
messages: MessageType[],
updateTextUI: (role: string, delta: string, model?: string) => void,
selectedModel: string,
accessToken: string | null,
tags: string[] = [],
signal?: AbortSignal,
onReasoningContent?: (content: string) => void,
onTimingData?: (timeToFirstToken: number) => void,
onUsageData?: (usage: TokenUsage) => void,
traceId?: string,
vector_store_ids?: string[],
guardrails?: string[],
) {
if (!accessToken) {
throw new Error("API key is required");
}
const isLocal = process.env.NODE_ENV === "development";
if (isLocal !== true) {
console.log = function () {};
}
const proxyBaseUrl = getProxyBaseUrl();
// Prepare headers with tags and trace ID
const headers: Record<string, string> = {};
if (tags && tags.length > 0) {
headers['x-litellm-tags'] = tags.join(',');
}
const client = new Anthropic({
apiKey: accessToken,
baseURL: proxyBaseUrl,
dangerouslyAllowBrowser: true,
defaultHeaders: headers,
});
try {
const startTime = Date.now();
let firstTokenReceived = false;
const requestBody: any = {
model: selectedModel,
messages: messages.map((m) => ({ role: m.role, content: m.content })),
stream: true,
max_tokens: 1024,
// @ts-ignore - litellm specific parameter
litellm_trace_id: traceId,
};
if (vector_store_ids) requestBody.vector_store_ids = vector_store_ids;
if (guardrails) requestBody.guardrails = guardrails;
// Use the streaming helper method for cleaner async iteration
// @ts-ignore - The SDK types might not include all litellm-specific parameters
const stream = client.messages.stream(requestBody, { signal });
for await (const messageStreamEvent of stream) {
console.log("Stream event:", messageStreamEvent);
// Process content block deltas
if (messageStreamEvent.type === 'content_block_delta') {
const delta = messageStreamEvent.delta;
// Measure time to first token
if (!firstTokenReceived) {
firstTokenReceived = true;
const timeToFirstToken = Date.now() - startTime;
console.log("First token received! Time:", timeToFirstToken, "ms");
if (onTimingData) {
onTimingData(timeToFirstToken);
}
}
// Handle different types of deltas
if (delta.type === 'text_delta') {
updateTextUI("assistant", delta.text, selectedModel);
}
// @ts-ignore - reasoning_content might not be in the official types yet
else if (delta.type === 'reasoning_delta' && onReasoningContent) {
// @ts-ignore
onReasoningContent(delta.text);
}
}
// Process usage data from message_delta events
if (messageStreamEvent.type === 'message_delta' && (messageStreamEvent as any).usage && onUsageData) {
const usage = (messageStreamEvent as any).usage;
console.log("Usage data found:", usage);
const usageData: TokenUsage = {
completionTokens: usage.output_tokens,
promptTokens: usage.input_tokens,
totalTokens: usage.input_tokens + usage.output_tokens,
};
onUsageData(usageData);
}
}
} catch (error) {
if (signal?.aborted) {
console.log("Anthropic messages request was cancelled");
} else {
message.error(
`Error occurred while generating model response. Please try again. Error: ${error}`,
20,
);
}
throw error;
}
}

3
ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx
View file

@ -6,6 +6,7 @@ export enum ModelMode {
CHAT = "chat",
RESPONSES = "responses",
IMAGE_EDITS = "image_edits",
ANTHROPIC_MESSAGES = "anthropic_messages",
// add additional modes as needed
}
@ -15,6 +16,7 @@ export enum ModelMode {
CHAT = "chat",
RESPONSES = "responses",
IMAGE_EDITS = "image_edits",
ANTHROPIC_MESSAGES = "anthropic_messages",
// add additional endpoint types if required
}
@ -24,6 +26,7 @@ export enum ModelMode {
[ModelMode.CHAT]: EndpointType.CHAT,
[ModelMode.RESPONSES]: EndpointType.RESPONSES,
[ModelMode.IMAGE_EDITS]: EndpointType.IMAGE_EDITS,
[ModelMode.ANTHROPIC_MESSAGES]: EndpointType.ANTHROPIC_MESSAGES,
};
export const getEndpointType = (mode: string): EndpointType => {