This commit is contained in:
Ishaan Jaffer 2026-03-03 10:36:56 -08:00
parent c6aca6f0c0
commit 952e504fa6
3 changed files with 342 additions and 24 deletions

View file

@ -9779,6 +9779,122 @@
}
]
},
"dashscope/qwen3-max-2026-01-23": {
"litellm_provider": "dashscope",
"max_input_tokens": 258048,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"tiered_pricing": [
{
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 6e-06,
"range": [
0,
32000.0
]
},
{
"input_cost_per_token": 2.4e-06,
"output_cost_per_token": 1.2e-05,
"range": [
32000.0,
128000.0
]
},
{
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"range": [
128000.0,
252000.0
]
}
]
},
"dashscope/qwen3-next-80b-a3b-instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true
},
"dashscope/qwen3-next-80b-a3b-thinking": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"dashscope/qwen3-vl-235b-a22b-instruct": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-235b-a22b-thinking": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-instruct": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.4e-07,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-thinking": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.87e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 260096,
@ -25806,6 +25922,30 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-sonnet-4.6": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"source": "https://openrouter.ai/anthropic/claude-sonnet-4.6",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
@ -26156,6 +26296,39 @@
"supports_web_search": true,
"tpm": 800000
},
"openrouter/google/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"source": "https://openrouter.ai/google/gemini-3.1-pro-preview",
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/gryphe/mythomax-l2-13b": {
"input_cost_per_token": 1.875e-06,
"litellm_provider": "openrouter",
@ -26533,6 +26706,29 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5.1-codex-max": {
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://openrouter.ai/openai/gpt-5.1-codex-max",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/openai/gpt-5.2": {
"input_cost_per_image": 0,
"cache_read_input_token_cost": 1.75e-07,
@ -26687,6 +26883,19 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"openrouter/qwen/qwen3-coder-plus": {
"input_cost_per_token": 1e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 997952,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 5e-06,
"source": "https://openrouter.ai/qwen/qwen3-coder-plus",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3-235b-a22b-2507": {
"input_cost_per_token": 7.1e-08,
"litellm_provider": "openrouter",
@ -26822,6 +27031,19 @@
"supports_vision": true,
"supports_prompt_caching": false
},
"openrouter/z-ai/glm-5": {
"input_cost_per_token": 8e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.56e-06,
"source": "https://openrouter.ai/z-ai/glm-5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/minimax/minimax-m2.1": {
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.2e-06,
@ -34315,6 +34537,36 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-5-code": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 5e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,

View file

@ -12,8 +12,11 @@ import {
} from "antd";
import {
ArrowRightOutlined,
CheckCircleOutlined,
DeleteOutlined,
PauseCircleOutlined,
PlusOutlined,
PlayCircleOutlined,
} from "@ant-design/icons";
import { fetchAvailableModels, ModelGroup } from "../playground/llm_calls/fetch_models";
@ -24,6 +27,7 @@ export interface RolloutConfig {
canaryPercent: number;
autoRollback: boolean;
errorThreshold: number;
active: boolean;
}
const ROLLOUT_STORAGE_KEY = "litellm_model_rollout_configs";
@ -43,6 +47,7 @@ export function findRolloutForModel(model: string): RolloutConfig | null {
const configs = getRolloutConfigs();
return configs.find(
(c) =>
c.active &&
c.liveModel &&
c.candidateModel &&
c.liveModel !== c.candidateModel &&
@ -97,6 +102,7 @@ const ModelRollout: React.FC<ModelRolloutProps> = ({ accessToken }) => {
canaryPercent: 10,
autoRollback: true,
errorThreshold: 5,
active: false,
},
]);
};
@ -111,18 +117,21 @@ const ModelRollout: React.FC<ModelRolloutProps> = ({ accessToken }) => {
const statusFor = (c: RolloutConfig) => {
if (!c.liveModel || !c.candidateModel) return { label: "Not Configured", color: "default" as const };
if (!c.active) return { label: "Draft", color: "default" as const };
if (c.canaryPercent === 100) return { label: "Fully Promoted", color: "green" as const };
if (c.canaryPercent === 0) return { label: "Not Started", color: "default" as const };
return { label: `${c.canaryPercent}% canary`, color: "blue" as const };
};
const canActivate = (c: RolloutConfig) => !!c.liveModel && !!c.candidateModel && c.liveModel !== c.candidateModel;
return (
<div className="w-full max-w-screen p-6 overflow-x-hidden box-border">
<div className="flex items-center justify-between mb-4">
<div>
<h1 className="text-xl font-semibold">Model Rollout</h1>
<span className="text-sm text-gray-500">
Manage A/B tests across your models &middot; {configs.length} active
Manage A/B tests across your models &middot; {configs.filter((c) => c.active).length} active
</span>
</div>
<Button type="primary" icon={<PlusOutlined />} onClick={addNew}>
@ -141,7 +150,7 @@ const ModelRollout: React.FC<ModelRolloutProps> = ({ accessToken }) => {
const status = statusFor(config);
const livePercent = 100 - config.canaryPercent;
return (
<div key={config.id} className="bg-white rounded-lg shadow">
<div key={config.id} className="bg-white rounded-lg shadow" style={config.active ? { borderLeft: "3px solid #52c41a" } : { borderLeft: "3px solid #d9d9d9" }}>
{/* Header */}
<div className="px-6 py-4 border-b flex items-center justify-between">
<div className="flex items-center gap-3">
@ -160,14 +169,35 @@ const ModelRollout: React.FC<ModelRolloutProps> = ({ accessToken }) => {
</Tag>
<Tag color={status.color}>{status.label}</Tag>
</div>
<Popconfirm
title="Delete this A/B test?"
onConfirm={() => remove(config.id)}
okText="Delete"
okButtonProps={{ danger: true }}
>
<Button type="text" danger icon={<DeleteOutlined />} size="small" />
</Popconfirm>
<div className="flex items-center gap-2">
{config.active ? (
<Button
size="small"
icon={<PauseCircleOutlined />}
onClick={() => update(config.id, { active: false })}
>
Deactivate
</Button>
) : (
<Button
size="small"
type="primary"
icon={<PlayCircleOutlined />}
disabled={!canActivate(config)}
onClick={() => update(config.id, { active: true })}
>
Activate
</Button>
)}
<Popconfirm
title="Delete this A/B test?"
onConfirm={() => remove(config.id)}
okText="Delete"
okButtonProps={{ danger: true }}
>
<Button type="text" danger icon={<DeleteOutlined />} size="small" />
</Popconfirm>
</div>
</div>
{/* Body */}

View file

@ -240,7 +240,13 @@ const ChatUI: React.FC<ChatUIProps> = ({
const [maxTokens, setMaxTokens] = useState<number>(2048);
const [useAdvancedParams, setUseAdvancedParams] = useState<boolean>(false);
const [mockTestFallbacks, setMockTestFallbacks] = useState<boolean>(false);
const [lastRolloutInfo, setLastRolloutInfo] = useState<{ requestedModel: string; actualModel: string } | null>(null);
const [lastRolloutInfo, setLastRolloutInfo] = useState<{
requestedModel: string;
actualModel: string;
liveModel: string;
candidateModel: string;
canaryPercent: number;
} | null>(null);
// Code Interpreter state (using custom hook)
const codeInterpreter = useCodeInterpreter();
@ -981,7 +987,13 @@ const ChatUI: React.FC<ChatUIProps> = ({
effectiveModel = rolloutConfig.liveModel;
}
rolloutApplied = effectiveModel !== selectedModel;
setLastRolloutInfo({ requestedModel: selectedModel!, actualModel: effectiveModel! });
setLastRolloutInfo({
requestedModel: selectedModel!,
actualModel: effectiveModel!,
liveModel: rolloutConfig.liveModel,
candidateModel: rolloutConfig.candidateModel,
canaryPercent: rolloutConfig.canaryPercent,
});
} else {
setLastRolloutInfo(null);
}
@ -1545,24 +1557,48 @@ const ChatUI: React.FC<ChatUIProps> = ({
<div
style={{
marginTop: 8,
padding: "8px 10px",
padding: "10px 12px",
borderRadius: 8,
background: lastRolloutInfo.requestedModel !== lastRolloutInfo.actualModel ? "#e6f7ff" : "#f6ffed",
border: lastRolloutInfo.requestedModel !== lastRolloutInfo.actualModel ? "1px solid #91d5ff" : "1px solid #b7eb8f",
background: "#f8f9fc",
border: "1px solid #e5e7eb",
fontSize: 12,
}}
>
<div style={{ fontWeight: 600, marginBottom: 2, color: "#333" }}>
<SwapOutlined style={{ marginRight: 4 }} /> Model Rollout Active
<div style={{ fontWeight: 600, marginBottom: 6, color: "#333", display: "flex", alignItems: "center", justifyContent: "space-between" }}>
<span><SwapOutlined style={{ marginRight: 4 }} /> A/B Test Active</span>
<span style={{
fontSize: 11,
fontWeight: 500,
padding: "1px 8px",
borderRadius: 10,
background: lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? "#f6ffed" : "#e6f7ff",
color: lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? "#52c41a" : "#1677ff",
border: lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? "1px solid #b7eb8f" : "1px solid #91d5ff",
}}>
Routed {lastRolloutInfo.actualModel}
</span>
</div>
{/* Split bar */}
<div style={{ display: "flex", borderRadius: 4, overflow: "hidden", height: 6, marginBottom: 6 }}>
<div style={{ width: `${100 - lastRolloutInfo.canaryPercent}%`, background: "#52c41a" }} />
<div style={{ width: `${lastRolloutInfo.canaryPercent}%`, background: "#1677ff" }} />
</div>
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}>
<span style={{ color: "#666" }}>Requested:</span>
<span style={{ fontWeight: 500 }}>{lastRolloutInfo.requestedModel}</span>
</div>
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}>
<span style={{ color: "#666" }}>Sent to:</span>
<span style={{ fontWeight: 600, color: lastRolloutInfo.requestedModel !== lastRolloutInfo.actualModel ? "#1677ff" : "#52c41a" }}>
{lastRolloutInfo.actualModel}
<span style={{
fontWeight: lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? 600 : 400,
color: lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? "#52c41a" : "#888",
}}>
{lastRolloutInfo.actualModel === lastRolloutInfo.liveModel ? "● " : ""}
{lastRolloutInfo.liveModel}
<span style={{ fontWeight: 500, color: "#52c41a", marginLeft: 4 }}>{100 - lastRolloutInfo.canaryPercent}%</span>
</span>
<span style={{
fontWeight: lastRolloutInfo.actualModel === lastRolloutInfo.candidateModel ? 600 : 400,
color: lastRolloutInfo.actualModel === lastRolloutInfo.candidateModel ? "#1677ff" : "#888",
}}>
{lastRolloutInfo.actualModel === lastRolloutInfo.candidateModel ? "● " : ""}
{lastRolloutInfo.candidateModel}
<span style={{ fontWeight: 500, color: "#1677ff", marginLeft: 4 }}>{lastRolloutInfo.canaryPercent}%</span>
</span>
</div>
</div>