diff --git a/src/api/providers/lmstudio.ts b/src/api/providers/lmstudio.ts index 5308ebb85c..9a3ab187bf 100644 --- a/src/api/providers/lmstudio.ts +++ b/src/api/providers/lmstudio.ts @@ -30,13 +30,24 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan ] try { - const stream = await this.client.chat.completions.create({ + // Create params object with optional draft model + const params: any = { model: this.getModel().id, messages: openAiMessages, temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE, stream: true, - }) - for await (const chunk of stream) { + } + + // Add draft model if speculative decoding is enabled and a draft model is specified + if (this.options.lmStudioSpeculativeDecodingEnabled && this.options.lmStudioDraftModelId) { + params.draft_model = this.options.lmStudioDraftModelId + } + + const results = await this.client.chat.completions.create(params) + + // Stream handling + // @ts-ignore + for await (const chunk of results) { const delta = chunk.choices[0]?.delta if (delta?.content) { yield { @@ -62,12 +73,20 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan async completePrompt(prompt: string): Promise { try { - const response = await this.client.chat.completions.create({ + // Create params object with optional draft model + const params: any = { model: this.getModel().id, messages: [{ role: "user", content: prompt }], temperature: this.options.modelTemperature ?? LMSTUDIO_DEFAULT_TEMPERATURE, stream: false, - }) + } + + // Add draft model if speculative decoding is enabled and a draft model is specified + if (this.options.lmStudioSpeculativeDecodingEnabled && this.options.lmStudioDraftModelId) { + params.draft_model = this.options.lmStudioDraftModelId + } + + const response = await this.client.chat.completions.create(params) return response.choices[0]?.message.content || "" } catch (error) { throw new Error( diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 0a2e385b6a..d0e68420b5 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1676,6 +1676,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { modelTemperature, modelMaxTokens, modelMaxThinkingTokens, + lmStudioDraftModelId, + lmStudioSpeculativeDecodingEnabled, } = apiConfiguration await Promise.all([ this.updateGlobalState("apiProvider", apiProvider), @@ -1725,6 +1727,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.updateGlobalState("modelTemperature", modelTemperature), this.updateGlobalState("modelMaxTokens", modelMaxTokens), this.updateGlobalState("anthropicThinking", modelMaxThinkingTokens), + this.updateGlobalState("lmStudioDraftModelId", lmStudioDraftModelId), + this.updateGlobalState("lmStudioSpeculativeDecodingEnabled", lmStudioSpeculativeDecodingEnabled), ]) if (this.cline) { this.cline.api = buildApiHandler(apiConfiguration) @@ -2221,6 +2225,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { modelMaxThinkingTokens, maxOpenTabsContext, browserToolEnabled, + lmStudioSpeculativeDecodingEnabled, + lmStudioDraftModelId, ] = await Promise.all([ this.getGlobalState("apiProvider") as Promise, this.getGlobalState("apiModelId") as Promise, @@ -2306,6 +2312,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.getGlobalState("anthropicThinking") as Promise, this.getGlobalState("maxOpenTabsContext") as Promise, this.getGlobalState("browserToolEnabled") as Promise, + this.getGlobalState("lmStudioSpeculativeDecodingEnabled") as Promise, + this.getGlobalState("lmStudioDraftModelId") as Promise, ]) let apiProvider: ApiProvider @@ -2371,6 +2379,8 @@ export class ClineProvider implements vscode.WebviewViewProvider { modelTemperature, modelMaxTokens, modelMaxThinkingTokens, + lmStudioSpeculativeDecodingEnabled, + lmStudioDraftModelId, }, lastShownAnnouncementId, customInstructions, diff --git a/src/shared/api.ts b/src/shared/api.ts index 462afc46cd..2ce7162640 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -49,6 +49,8 @@ export interface ApiHandlerOptions { ollamaBaseUrl?: string lmStudioModelId?: string lmStudioBaseUrl?: string + lmStudioDraftModelId?: string + lmStudioSpeculativeDecodingEnabled?: boolean geminiApiKey?: string openAiNativeApiKey?: string mistralApiKey?: string diff --git a/src/shared/globalState.ts b/src/shared/globalState.ts index 64a222be81..88f9824151 100644 --- a/src/shared/globalState.ts +++ b/src/shared/globalState.ts @@ -41,6 +41,8 @@ export type GlobalStateKey = | "ollamaBaseUrl" | "lmStudioModelId" | "lmStudioBaseUrl" + | "lmStudioDraftModelId" + | "lmStudioSpeculativeDecodingEnabled" | "anthropicBaseUrl" | "azureApiVersion" | "openAiStreamingEnabled" diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index e69be19ef2..b9050a999f 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -1107,6 +1107,79 @@ const ApiOptions = ({ ))} )} +
+ { + // Explicitly set the boolean value using direct method + setApiConfigurationField("lmStudioSpeculativeDecodingEnabled", checked) + }}> + Enable Speculative Decoding + +
+ {apiConfiguration?.lmStudioSpeculativeDecodingEnabled && ( + <> + + Draft Model ID + +
+ + + Draft model must be from the same model family for speculative decoding to work + correctly. + +
+ {lmStudioModels.length > 0 && ( + <> +
+ Select Draft Model +
+ + {lmStudioModels.map((model) => ( + + {model} + + ))} + + {lmStudioModels.length === 0 && ( +
+ + No draft models found. Please ensure LM Studio is running with Server Mode + enabled. +
+ )} + + )} + + )} +