From 180fbd5995948768c17f3659a46fec1e58cbda2c Mon Sep 17 00:00:00 2001 From: Hiroki Nakashima Date: Wed, 5 Feb 2025 07:46:13 +0900 Subject: [PATCH] feat: add LiteLLM API provider support (#1618) --- src/api/index.ts | 3 + src/api/providers/litellm.ts | 60 +++++++++++++++++++ src/core/webview/ClineProvider.ts | 18 ++++++ src/shared/api.ts | 16 +++++ .../src/components/settings/ApiOptions.tsx | 41 ++++++++++++- 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/api/providers/litellm.ts diff --git a/src/api/index.ts b/src/api/index.ts index 2ef82f8659..5ed9e6de8c 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -13,6 +13,7 @@ import { ApiStream } from "./transform/stream" import { DeepSeekHandler } from "./providers/deepseek" import { MistralHandler } from "./providers/mistral" import { VsCodeLmHandler } from "./providers/vscode-lm" +import { LiteLlmHandler } from "./providers/litellm" export interface ApiHandler { createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream @@ -50,6 +51,8 @@ export function buildApiHandler(configuration: ApiConfiguration): ApiHandler { return new MistralHandler(options) case "vscode-lm": return new VsCodeLmHandler(options) + case "litellm": + return new LiteLlmHandler(options) default: return new AnthropicHandler(options) } diff --git a/src/api/providers/litellm.ts b/src/api/providers/litellm.ts new file mode 100644 index 0000000000..80ad5e2c75 --- /dev/null +++ b/src/api/providers/litellm.ts @@ -0,0 +1,60 @@ +import { Anthropic } from "@anthropic-ai/sdk" +import OpenAI from "openai" +import { ApiHandlerOptions, liteLlmDefaultModelId, liteLlmModelInfoSaneDefaults } from "../../shared/api" +import { ApiHandler } from ".." +import { ApiStream } from "../transform/stream" +import { convertToOpenAiMessages } from "../transform/openai-format" + +export class LiteLlmHandler implements ApiHandler { + private options: ApiHandlerOptions + private client: OpenAI + + constructor(options: ApiHandlerOptions) { + this.options = options + this.client = new OpenAI({ + baseURL: this.options.liteLlmBaseUrl || "http://localhost:4000", + apiKey: "not-needed", + }) + } + + async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream { + const formattedMessages = convertToOpenAiMessages(messages) + const systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = { + role: "system", + content: systemPrompt, + } + + const stream = await this.client.chat.completions.create({ + model: this.options.liteLlmModelId || liteLlmDefaultModelId, + messages: [systemMessage, ...formattedMessages], + temperature: 0, + stream: true, + stream_options: { include_usage: true }, + }) + + for await (const chunk of stream) { + const delta = chunk.choices[0]?.delta + if (delta?.content) { + yield { + type: "text", + text: delta.content, + } + } + + if (chunk.usage) { + yield { + type: "usage", + inputTokens: chunk.usage.prompt_tokens || 0, + outputTokens: chunk.usage.completion_tokens || 0, + } + } + } + } + + getModel() { + return { + id: this.options.liteLlmModelId || liteLlmDefaultModelId, + info: liteLlmModelInfoSaneDefaults, + } + } +} diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 12a36daddf..a15b99ed05 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -76,6 +76,8 @@ type GlobalStateKey = | "previousModeApiProvider" | "previousModeModelId" | "previousModeModelInfo" + | "liteLlmBaseUrl" + | "liteLlmModelId" export const GlobalFileNames = { apiConversationHistory: "api_conversation_history.json", @@ -443,6 +445,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { openRouterModelId, openRouterModelInfo, vsCodeLmModelSelector, + liteLlmBaseUrl, + liteLlmModelId, } = message.apiConfiguration await this.updateGlobalState("apiProvider", apiProvider) await this.updateGlobalState("apiModelId", apiModelId) @@ -471,6 +475,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { await this.updateGlobalState("openRouterModelId", openRouterModelId) await this.updateGlobalState("openRouterModelInfo", openRouterModelInfo) await this.updateGlobalState("vsCodeLmModelSelector", vsCodeLmModelSelector) + await this.updateGlobalState("liteLlmBaseUrl", liteLlmBaseUrl) + await this.updateGlobalState("liteLlmModelId", liteLlmModelId) if (this.cline) { this.cline.api = buildApiHandler(message.apiConfiguration) } @@ -535,6 +541,9 @@ export class ClineProvider implements vscode.WebviewViewProvider { case "lmstudio": await this.updateGlobalState("previousModeModelId", apiConfiguration.lmStudioModelId) break + case "litellm": + await this.updateGlobalState("previousModeModelId", apiConfiguration.liteLlmModelId) + break } // Restore the model used in previous mode @@ -563,6 +572,9 @@ export class ClineProvider implements vscode.WebviewViewProvider { case "lmstudio": await this.updateGlobalState("lmStudioModelId", newModelId) break + case "litellm": + await this.updateGlobalState("liteLlmModelId", newModelId) + break } if (this.cline) { @@ -1364,6 +1376,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { browserSettings, chatSettings, vsCodeLmModelSelector, + liteLlmBaseUrl, + liteLlmModelId, userInfo, authToken, previousModeApiProvider, @@ -1403,6 +1417,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.getGlobalState("browserSettings") as Promise, this.getGlobalState("chatSettings") as Promise, this.getGlobalState("vsCodeLmModelSelector") as Promise, + this.getGlobalState("liteLlmBaseUrl") as Promise, + this.getGlobalState("liteLlmModelId") as Promise, this.getGlobalState("userInfo") as Promise, this.getSecret("authToken") as Promise, this.getGlobalState("previousModeApiProvider") as Promise, @@ -1453,6 +1469,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { openRouterModelId, openRouterModelInfo, vsCodeLmModelSelector, + liteLlmBaseUrl, + liteLlmModelId, }, lastShownAnnouncementId, customInstructions, diff --git a/src/shared/api.ts b/src/shared/api.ts index 34020dec28..5c4f1c9486 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -11,10 +11,13 @@ export type ApiProvider = | "deepseek" | "mistral" | "vscode-lm" + | "litellm" export interface ApiHandlerOptions { apiModelId?: string apiKey?: string // anthropic + liteLlmBaseUrl?: string + liteLlmModelId?: string anthropicBaseUrl?: string openRouterApiKey?: string openRouterModelId?: string @@ -419,3 +422,16 @@ export const mistralModels = { outputPrice: 0.9, }, } as const satisfies Record + +// LiteLLM +// https://docs.litellm.ai/docs/ +export type LiteLLMModelId = string +export const liteLlmDefaultModelId = "gpt-3.5-turbo" +export const liteLlmModelInfoSaneDefaults: ModelInfo = { + maxTokens: 4096, + contextWindow: 8192, + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, +} diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index ceb75a0ee4..84018b9efd 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -133,7 +133,7 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is VSCodeDropdown has an open bug where dynamically rendered options don't auto select the provided value prop. You can see this for yourself by comparing it with normal select/option elements, which work as expected. https://github.com/microsoft/vscode-webview-ui-toolkit/issues/433 - In our case, when the user switches between providers, we recalculate the selectedModelId depending on the provider, the default model for that provider, and a modelId that the user may have selected. Unfortunately, the VSCodeDropdown component wouldn't select this calculated value, and would default to the first "Select a model..." option instead, which makes it seem like the model was cleared out when it wasn't. + In our case, when the user switches between providers, we recalculate the selectedModelId depending on the provider, the default model for that provider, and a modelId that the user may have selected. Unfortunately, the VSCodeDropdown component wouldn't select this calculated value, and would default to the first "Select a model..." option instead, which makes it seem like the model was cleared out when it wasn't. As a workaround, we create separate instances of the dropdown for each provider, and then conditionally render the one that matches the current provider. */ @@ -187,6 +187,7 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is VS Code LM API LM Studio Ollama + LiteLLM @@ -739,6 +740,38 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is )} + {selectedProvider === "litellm" && ( +
+ + Base URL (optional) + + + Model ID + +

+ LiteLLM provides a unified interface to access various LLM providers' models. See their{" "} + + quickstart guide + {" "} + for more information. +

+
+ )} + {selectedProvider === "ollama" && (