diff --git a/vscode-extension/README.md b/vscode-extension/README.md index cea8af6b578..68fbae3d921 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -4,7 +4,7 @@ Chat in VS Code with every model your [LiteLLM AI Gateway](https://docs.litellm. ## What you get -The model list comes from the gateway's `GET /model_group/info` endpoint, scoped to the virtual key you configure, so the picker shows exactly the chat models that key can use. Each model carries its input and output price per 1M tokens in the picker and in the Language Models editor, and its context limits come from the gateway too, so VS Code sizes prompts correctly. A model whose gateway entry lists `supported_reasoning_efforts` gets a Reasoning Effort submenu in the picker's Configure Model menu, and the chosen effort is sent as `reasoning_effort` on every request to that model. Requests go to `POST /v1/chat/completions` on the gateway as streaming chat completions with tools and images passed through, so routing, fallbacks, guardrails, and spend tracking all apply as usual +The model list comes from the gateway's `GET /model_group/info` endpoint, scoped to the virtual key you configure, so the picker shows exactly the chat models that key can use. Each model carries its input and output price per 1M tokens in the picker and in the Language Models editor, and its context limits come from the gateway too, so VS Code sizes prompts correctly. A model whose gateway entry lists `supported_reasoning_efforts` gets a Reasoning Effort submenu in the picker's Configure Model menu, and the chosen effort is sent as `reasoning_effort` on every request to that model. Chat-mode models use streaming `POST /v1/chat/completions`; Responses-mode models use streaming `POST /v1/responses`. Both pass tools and images through, so routing, fallbacks, guardrails, and spend tracking apply as usual ## Setup @@ -17,7 +17,7 @@ Add the same provider again with another name to reach a second gateway or a sec ## Requirements -VS Code 1.115 or newer and a LiteLLM AI Gateway the key can reach. The key needs access to at least one model group whose mode is `chat` +VS Code 1.115 or newer and a LiteLLM AI Gateway the key can reach. The key needs access to at least one model group whose mode is `chat` or `responses` ## Development diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 81e9425ef4e..1964bd1cdc6 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "litellm-vscode", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "litellm-vscode", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "dependencies": { "openai": "^7.18.0" diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..b9a7fd0629b 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -2,7 +2,7 @@ "name": "litellm-vscode", "displayName": "LiteLLM", "description": "Chat with every model behind your LiteLLM AI Gateway in VS Code, with live pricing and reasoning effort controls in the model picker", - "version": "0.1.0", + "version": "0.2.0", "publisher": "litellm", "license": "MIT", "repository": { diff --git a/vscode-extension/src/gateway.ts b/vscode-extension/src/gateway.ts index 0ffe5e0f3f3..720f5437da8 100644 --- a/vscode-extension/src/gateway.ts +++ b/vscode-extension/src/gateway.ts @@ -1,7 +1,8 @@ -import OpenAI from "openai"; +import OpenAI, { APIError } from "openai"; import type { ChatCompletionChunk, ChatCompletionCreateParamsStreaming } from "openai/resources/chat/completions"; +import type { ResponseCreateParamsStreaming, ResponseStreamEvent } from "openai/resources/responses/responses"; import packageJson from "../package.json"; -import { parseModelGroups, type ConfigurationValues, type ModelGroupInfo } from "./models"; +import { MODE_CHAT, MODE_RESPONSES, parseModelGroups, type ConfigurationValues, type ModelGroupInfo } from "./models"; export interface GatewayConfig { readonly baseUrl: string; @@ -26,6 +27,11 @@ export interface GatewayClient { params: ChatCompletionCreateParamsStreaming, signal: AbortSignal, ): Promise>; + streamResponsesCompletion( + config: GatewayConfig, + params: ResponseCreateParamsStreaming, + signal: AbortSignal, + ): Promise>; } export const USER_AGENT = `litellm-vscode/${packageJson.version}`; @@ -109,6 +115,35 @@ export const createGatewayClient = (fetchImpl: typeof fetch = fetch): GatewayCli fetch: fetchImpl, maxRetries: 0, }); - return client.chat.completions.create(params, { signal }); + return retryGatewayTimeout( + () => client.chat.completions.create(params, { signal }) as Promise>, + signal, + ); + }, + streamResponsesCompletion(config, params, signal) { + const client = new OpenAI({ + apiKey: config.apiKey, + baseURL: openAiBaseUrl(config.baseUrl), + defaultHeaders: { "User-Agent": USER_AGENT }, + fetch: fetchImpl, + maxRetries: 0, + }); + return retryGatewayTimeout( + () => client.responses.create(params, { signal }) as Promise>, + signal, + ); }, }); + +const retryGatewayTimeout = async (request: () => Promise, signal: AbortSignal): Promise => { + try { + return await request(); + } catch (error) { + if (signal.aborted || !(error instanceof APIError) || error.status !== 504) { + throw error; + } + return request(); + } +}; + +export const isResponsesMode = (mode: string | undefined): boolean => mode === MODE_RESPONSES; diff --git a/vscode-extension/src/messages.ts b/vscode-extension/src/messages.ts index 855f85435ad..3b446df41ca 100644 --- a/vscode-extension/src/messages.ts +++ b/vscode-extension/src/messages.ts @@ -8,7 +8,13 @@ import type { ChatCompletionTool, ChatCompletionToolMessageParam, } from "openai/resources/chat/completions"; -import { estimateTokens } from "./models"; +import type { + FunctionTool, + ResponseCreateParamsStreaming, + ResponseInputContent, + ResponseInputItem, +} from "openai/resources/responses/responses"; +import { MODE_CHAT, MODE_RESPONSES, estimateTokens } from "./models"; export interface ChatRequestInput { readonly model: string; @@ -19,6 +25,15 @@ export interface ChatRequestInput { readonly modelOptions: { readonly [key: string]: unknown }; } +export interface ResponsesRequestInput { + readonly model: string; + readonly messages: readonly vscode.LanguageModelChatRequestMessage[]; + readonly tools: readonly vscode.LanguageModelChatTool[]; + readonly requireToolCall: boolean; + readonly reasoningEffort: string | undefined; + readonly modelOptions: { readonly [key: string]: unknown }; +} + interface TextPart { readonly value: string; } @@ -186,3 +201,92 @@ export const buildChatCompletionParams = (input: ChatRequestInput): ChatCompleti ? {} : { reasoning_effort: input.reasoningEffort as ChatCompletionCreateParamsStreaming["reasoning_effort"] }), }); + +const inputContent = (parts: readonly unknown[]): readonly ResponseInputContent[] => + parts.flatMap((part): readonly ResponseInputContent[] => { + if (isImagePart(part)) { + return [{ type: "input_image", image_url: dataUrl(part), detail: "auto" }]; + } + const text = textOf(part); + return text === "" ? [] : [{ type: "input_text", text }]; + }); + +const responsesUserItems = (parts: readonly unknown[]): readonly ResponseInputItem[] => { + const toolResults = parts.filter(isToolResultPart); + const toolResultImages = toolResults.flatMap((result) => result.content.filter(isImagePart)); + const remaining = parts.filter((part) => !isToolResultPart(part)); + const userContent = inputContent([...remaining, ...toolResultImages]); + const outputs: readonly ResponseInputItem[] = toolResults.map((result) => ({ + type: "function_call_output", + call_id: result.callId, + output: result.content.filter((item) => !isImagePart(item)).map(textOf).join(""), + })); + const userItem: readonly ResponseInputItem[] = + userContent.length === 0 ? [] : [{ type: "message", role: "user", content: [...userContent] }]; + return [...outputs, ...userItem]; +}; + +const responsesAssistantItems = (parts: readonly unknown[]): readonly ResponseInputItem[] => { + const text = parts.filter(isTextPart).map((part) => part.value).join(""); + const message: readonly ResponseInputItem[] = text === "" ? [] : [{ type: "message", role: "assistant", content: text }]; + const calls: readonly ResponseInputItem[] = parts.filter(isToolCallPart).map((part) => ({ + type: "function_call", + call_id: part.callId, + name: part.name, + arguments: JSON.stringify(part.input), + })); + return [...message, ...calls]; +}; + +const convertResponsesMessage = (message: vscode.LanguageModelChatRequestMessage): readonly ResponseInputItem[] => { + const role: number = message.role; + switch (role) { + case USER_ROLE: + return responsesUserItems(message.content); + case ASSISTANT_ROLE: + return responsesAssistantItems(message.content); + case SYSTEM_ROLE: + return [{ type: "message", role: "system", content: message.content.map(textOf).join("") }]; + default: + return []; + } +}; + +export const toResponsesInput = (messages: readonly vscode.LanguageModelChatRequestMessage[]): readonly ResponseInputItem[] => + messages.flatMap(convertResponsesMessage); + +const EMPTY_OBJECT_SCHEMA = { type: "object", properties: {} } as const; + +const toResponsesTool = (tool: vscode.LanguageModelChatTool): FunctionTool => ({ + type: "function", + name: tool.name, + description: tool.description, + parameters: (tool.inputSchema as Record | undefined) ?? EMPTY_OBJECT_SCHEMA, + strict: false, +}); + +const RESPONSES_NUMERIC_OPTIONS = ["temperature", "top_p"] as const; + +const responsesModelOptions = (modelOptions: { readonly [key: string]: unknown }): Record => + Object.fromEntries([ + ...RESPONSES_NUMERIC_OPTIONS.flatMap((key) => { + const value = modelOptions[key]; + return typeof value === "number" ? [[key, value] as const] : []; + }), + ...(typeof modelOptions.max_tokens === "number" ? [["max_output_tokens", modelOptions.max_tokens] as const] : []), + ]); + +export const buildResponsesCompletionParams = (input: ResponsesRequestInput): ResponseCreateParamsStreaming => ({ + model: input.model, + input: [...toResponsesInput(input.messages)], + stream: true, + ...responsesModelOptions(input.modelOptions), + ...(input.tools.length === 0 ? {} : { tools: input.tools.map(toResponsesTool) }), + ...(input.tools.length === 0 || !input.requireToolCall ? {} : { tool_choice: "required" }), + ...(input.reasoningEffort === undefined + ? {} + : { reasoning: { effort: input.reasoningEffort as NonNullable["effort"] } }), +}); + +export { MODE_CHAT, MODE_RESPONSES } from "./models"; +export { estimateTokens } from "./models"; diff --git a/vscode-extension/src/models.ts b/vscode-extension/src/models.ts index d76f6557b8b..09711f3b506 100644 --- a/vscode-extension/src/models.ts +++ b/vscode-extension/src/models.ts @@ -33,11 +33,16 @@ export interface ModelDescriptor { readonly version: string; readonly detail: string; readonly tooltip: string; + readonly inputCost: number | undefined; + readonly outputCost: number | undefined; + readonly pricing: string | undefined; + readonly priceCategory: "low" | "medium" | "high" | "very_high" | undefined; readonly maxInputTokens: number; readonly maxOutputTokens: number; readonly imageInput: boolean; readonly toolCalling: boolean; readonly configurationSchema: ConfigurationSchema | undefined; + readonly mode: string; } export type ModelGroupsParseResult = @@ -86,7 +91,7 @@ export const parseModelGroups = (body: unknown): ModelGroupsParseResult => { return { kind: "ok", groups }; }; -const isChatGroup = (group: ModelGroupInfo): boolean => group.mode === undefined || group.mode === "chat"; +const isChatGroup = (group: ModelGroupInfo): boolean => group.mode === undefined || group.mode === "chat" || group.mode === "responses"; export const formatUsdPerMillionTokens = (costPerToken: number): string => { const perMillion = costPerToken * 1_000_000; @@ -106,6 +111,38 @@ const pricingDetail = (group: ModelGroupInfo): string => { return `${input} in / ${output} out per 1M tokens`; }; +const priceCategoryFor = (inputCost: number, outputCost: number): "low" | "medium" | "high" | "very_high" => { + const blended = (3 * inputCost + outputCost) / 4; + if (blended < 1) { + return "low"; + } + if (blended < 8) { + return "medium"; + } + return blended < 40 ? "high" : "very_high"; +}; + +const pricingFor = ( + group: ModelGroupInfo, +): Pick => { + const inputCost = group.inputCostPerToken === undefined ? undefined : group.inputCostPerToken * 1_000_000; + const outputCost = group.outputCostPerToken === undefined ? undefined : group.outputCostPerToken * 1_000_000; + if (inputCost === undefined || outputCost === undefined) { + return { + inputCost, + outputCost, + pricing: undefined, + priceCategory: undefined, + }; + } + return { + inputCost, + outputCost, + pricing: `${formatUsdPerMillionTokens(group.inputCostPerToken ?? 0)} in / ${formatUsdPerMillionTokens(group.outputCostPerToken ?? 0)} out per 1M tokens`, + priceCategory: priceCategoryFor(inputCost, outputCost), + }; +}; + const capitalize = (value: string): string => value.charAt(0).toUpperCase() + value.slice(1); const effortSchema = (efforts: readonly string[]): ConfigurationSchema | undefined => { @@ -152,11 +189,13 @@ const describeGroup = (group: ModelGroupInfo): ModelDescriptor => ({ version: "1.0", detail: pricingDetail(group), tooltip: tooltipFor(group), + ...pricingFor(group), maxInputTokens: group.maxInputTokens ?? ASSUMED_MAX_INPUT_TOKENS, maxOutputTokens: group.maxOutputTokens ?? ASSUMED_MAX_OUTPUT_TOKENS, imageInput: group.supportsVision, toolCalling: group.supportsFunctionCalling, configurationSchema: effortSchema(group.supportedReasoningEfforts), + mode: group.mode ?? "chat", }); export const describeModels = (groups: readonly ModelGroupInfo[]): readonly ModelDescriptor[] => @@ -168,3 +207,6 @@ export const reasoningEffortFrom = (configuration: ConfigurationValues | undefin }; export const estimateTokens = (text: string): number => Math.ceil(text.length / 4); + +export const MODE_CHAT = "chat"; +export const MODE_RESPONSES = "responses"; diff --git a/vscode-extension/src/provider.ts b/vscode-extension/src/provider.ts index 3cafd449147..557534bda78 100644 --- a/vscode-extension/src/provider.ts +++ b/vscode-extension/src/provider.ts @@ -1,14 +1,18 @@ import * as vscode from "vscode"; -import { gatewayConfigFrom, summarizeErrorBody, type GatewayClient, type GatewayConfig, type GatewayConfigResult, type ModelGroupsResult } from "./gateway"; -import { buildChatCompletionParams, estimateMessageTokens } from "./messages"; -import { describeModels, estimateTokens, reasoningEffortFrom, type ModelDescriptor } from "./models"; +import { APIError } from "openai"; +import { isResponsesMode, gatewayConfigFrom, summarizeErrorBody, type GatewayClient, type GatewayConfig, type GatewayConfigResult, type ModelGroupsResult } from "./gateway"; +import { buildChatCompletionParams, buildResponsesCompletionParams, estimateMessageTokens } from "./messages"; +import { describeModels, estimateTokens, reasoningEffortFrom, MODE_CHAT, MODE_RESPONSES, type ModelDescriptor, type ModelGroupInfo } from "./models"; import { responseParts, type ResponsePart } from "./stream"; export interface LiteLLMModel extends vscode.LanguageModelChatInformation { readonly gateway: GatewayConfig; + readonly mode: string; } export const TRUNCATED_MESSAGE = "The model stopped at its output token limit before finishing the response"; +export const GATEWAY_TIMEOUT_MESSAGE = + "LiteLLM gateway timed out (HTTP 504). Check the gateway or upstream model and try again."; const RECONFIGURE_HINT = 'Fix it from the gear on its row in Manage Language Models: "Update API Key" for the key, "Open in Language Models (JSON)" for the URL'; @@ -31,18 +35,23 @@ const discoveryFailure = (result: Exclude, ba } }; -const toModel = (descriptor: ModelDescriptor, gateway: GatewayConfig): LiteLLMModel => ({ +const toModel = (descriptor: ModelDescriptor, gateway: GatewayConfig, mode: string): LiteLLMModel => ({ id: descriptor.id, name: descriptor.name, family: descriptor.family, version: descriptor.version, detail: descriptor.detail, tooltip: descriptor.tooltip, + ...(descriptor.inputCost === undefined ? {} : { inputCost: descriptor.inputCost }), + ...(descriptor.outputCost === undefined ? {} : { outputCost: descriptor.outputCost }), + ...(descriptor.pricing === undefined ? {} : { pricing: descriptor.pricing }), + ...(descriptor.priceCategory === undefined ? {} : { priceCategory: descriptor.priceCategory }), maxInputTokens: descriptor.maxInputTokens, maxOutputTokens: descriptor.maxOutputTokens, capabilities: { imageInput: descriptor.imageInput, toolCalling: descriptor.toolCalling }, ...(descriptor.configurationSchema === undefined ? {} : { configurationSchema: descriptor.configurationSchema }), gateway, + mode, }); const toVscodePart = (part: ResponsePart): vscode.LanguageModelResponsePart => { @@ -58,6 +67,9 @@ const toVscodePart = (part: ResponsePart): vscode.LanguageModelResponsePart => { } }; +const chatRequestError = (error: unknown): unknown => + error instanceof APIError && error.status === 504 ? new Error(GATEWAY_TIMEOUT_MESSAGE) : error; + const withAbortSignal = async (token: vscode.CancellationToken, run: (signal: AbortSignal) => Promise): Promise => { const controller = new AbortController(); const subscription = token.onCancellationRequested(() => controller.abort()); @@ -97,7 +109,7 @@ export class LiteLLMChatProvider implements vscode.LanguageModelChatProvider toModel(descriptor, configured.config)); + return describeModels(result.groups).map((descriptor) => toModel(descriptor, configured.config, descriptor.mode)); } async provideLanguageModelChatResponse( @@ -106,6 +118,24 @@ export class LiteLLMChatProvider implements vscode.LanguageModelChatProvider, token: vscode.CancellationToken, + ): Promise { + if (model.mode === MODE_RESPONSES) { + await this.streamResponses(model, messages, options, progress, token); + } else { + await this.streamChat(model, messages, options, progress, token); + } + } + + async provideTokenCount(_model: LiteLLMModel, text: string | vscode.LanguageModelChatRequestMessage): Promise { + return typeof text === "string" ? estimateTokens(text) : estimateMessageTokens(text); + } + + private async streamChat( + model: LiteLLMModel, + messages: readonly vscode.LanguageModelChatRequestMessage[], + options: vscode.ProvideLanguageModelChatResponseOptions, + progress: vscode.Progress, + token: vscode.CancellationToken, ): Promise { const params = buildChatCompletionParams({ model: model.id, @@ -126,11 +156,37 @@ export class LiteLLMChatProvider implements vscode.LanguageModelChatProvider { - return typeof text === "string" ? estimateTokens(text) : estimateMessageTokens(text); + private async streamResponses( + model: LiteLLMModel, + messages: readonly vscode.LanguageModelChatRequestMessage[], + options: vscode.ProvideLanguageModelChatResponseOptions, + progress: vscode.Progress, + token: vscode.CancellationToken, + ): Promise { + const params = buildResponsesCompletionParams({ + model: model.id, + messages, + tools: options.tools ?? [], + requireToolCall: options.toolMode === vscode.LanguageModelChatToolMode.Required, + reasoningEffort: reasoningEffortFrom(options.modelConfiguration), + modelOptions: options.modelOptions ?? {}, + }); + try { + await withAbortSignal(token, async (signal) => { + const chunks = await this.gateway.streamResponsesCompletion(model.gateway, params, signal); + for await (const part of responseParts(chunks)) { + progress.report(toVscodePart(part)); + } + }); + } catch (error) { + if (token.isCancellationRequested) { + return; + } + throw chatRequestError(error); + } } } diff --git a/vscode-extension/src/stream.ts b/vscode-extension/src/stream.ts index f1677262bbf..3f6c12ce3c2 100644 --- a/vscode-extension/src/stream.ts +++ b/vscode-extension/src/stream.ts @@ -1,4 +1,5 @@ import type { ChatCompletionChunk } from "openai/resources/chat/completions"; +import type { ResponseStreamEvent } from "openai/resources/responses/responses"; export type ResponsePart = | { readonly kind: "text"; readonly value: string } @@ -9,6 +10,7 @@ export type ResponsePart = interface PendingToolCall { readonly index: number; readonly callId: string; + readonly itemId: string; readonly name: string; readonly arguments: string; } @@ -22,11 +24,11 @@ export interface ChunkOutcome { export const NO_PENDING_TOOL_CALLS: PendingToolCalls = []; -type ToolCallDelta = NonNullable[number]; +type ChatToolCallDelta = NonNullable[number]; const nonEmpty = (value: string | undefined): string | undefined => (value === undefined || value === "" ? undefined : value); -const targetOf = (pending: PendingToolCalls, delta: ToolCallDelta): PendingToolCall | undefined => { +const targetOfChat = (pending: PendingToolCalls, delta: ChatToolCallDelta): PendingToolCall | undefined => { const id = nonEmpty(delta.id); if (id !== undefined) { return pending.find((call) => call.callId === id); @@ -35,9 +37,9 @@ const targetOf = (pending: PendingToolCalls, delta: ToolCallDelta): PendingToolC return sameIndex.at(-1) ?? (delta.index === undefined ? pending.at(-1) : undefined); }; -const mergeToolCallDelta = (pending: PendingToolCalls, delta: ToolCallDelta): PendingToolCalls => { - const target = targetOf(pending, delta); - const base: PendingToolCall = target ?? { index: delta.index ?? pending.length, callId: nonEmpty(delta.id) ?? "", name: "", arguments: "" }; +const mergeChatToolCallDelta = (pending: PendingToolCalls, delta: ChatToolCallDelta): PendingToolCalls => { + const target = targetOfChat(pending, delta); + const base: PendingToolCall = target ?? { index: delta.index ?? pending.length, callId: nonEmpty(delta.id) ?? "", itemId: "", name: "", arguments: "" }; const merged: PendingToolCall = { ...base, name: nonEmpty(delta.function?.name) ?? base.name, @@ -46,15 +48,21 @@ const mergeToolCallDelta = (pending: PendingToolCalls, delta: ToolCallDelta): Pe return target === undefined ? [...pending, merged] : pending.map((call) => (call === target ? merged : call)); }; -export const applyChunk = (pending: PendingToolCalls, chunk: ChatCompletionChunk): ChunkOutcome => { - const choice = chunk.choices[0]; - if (choice === undefined) { - return { pending, parts: [] }; - } - const text = typeof choice.delta.content === "string" && choice.delta.content !== "" ? [{ kind: "text", value: choice.delta.content } as const] : []; - const truncated = choice.finish_reason === "length" ? [{ kind: "truncated" } as const] : []; - const nextPending = (choice.delta.tool_calls ?? []).reduce(mergeToolCallDelta, pending); - return { pending: nextPending, parts: [...text, ...truncated] }; +const targetOfResponses = (pending: PendingToolCalls, itemId: string): PendingToolCall | undefined => + pending.find((call) => call.itemId === itemId); + +const startResponsesToolCall = (pending: PendingToolCalls, item: { id?: string; call_id: string; name: string }, index: number): PendingToolCalls => [ + ...pending.filter((call) => call.itemId !== (item.id ?? item.call_id)), + { index, callId: item.call_id, itemId: item.id ?? item.call_id, name: item.name, arguments: "" }, +]; + +const updateResponsesToolCallArguments = ( + pending: PendingToolCalls, + itemId: string, + update: (current: string) => string, +): PendingToolCalls => { + const target = targetOfResponses(pending, itemId); + return target === undefined ? pending : pending.map((call) => (call === target ? { ...call, arguments: update(call.arguments) } : call)); }; const parseArguments = (raw: string): object | undefined => { @@ -79,7 +87,7 @@ const finishToolCall = (call: PendingToolCall): ResponsePart => { export const flushToolCalls = (pending: PendingToolCalls): readonly ResponsePart[] => [...pending].sort((left, right) => left.index - right.index).map(finishToolCall); -export async function* responseParts(chunks: AsyncIterable): AsyncGenerator { +export async function* responseParts(chunks: AsyncIterable): AsyncGenerator { let pending: PendingToolCalls = NO_PENDING_TOOL_CALLS; for await (const chunk of chunks) { const outcome = applyChunk(pending, chunk); @@ -88,3 +96,59 @@ export async function* responseParts(chunks: AsyncIterable) } yield* flushToolCalls(pending); } + +const applyChatChunk = (pending: PendingToolCalls, chunk: ChatCompletionChunk): ChunkOutcome => { + const choice = chunk.choices?.[0]; + if (choice === undefined) { + return { pending, parts: [] as const }; + } + const text = typeof choice.delta.content === "string" && choice.delta.content !== "" ? [{ kind: "text" as const, value: choice.delta.content }] : []; + const truncated = choice.finish_reason === "length" ? [{ kind: "truncated" as const }] : []; + const nextPending = (choice.delta.tool_calls ?? []).reduce(mergeChatToolCallDelta, pending); + return { pending: nextPending, parts: [...text, ...truncated] as const }; +}; + +const applyResponsesEvent = (pending: PendingToolCalls, event: ResponseStreamEvent): ChunkOutcome => { + const eventType = event.type; + + if (eventType === "response.output_text.delta") { + const text = nonEmpty(event.delta); + if (text !== undefined) { + return { pending, parts: [{ kind: "text", value: text }] as const }; + } + return { pending, parts: [] as const }; + } + + if (eventType === "response.output_text.done") { + return { pending, parts: [] as const }; + } + + if (eventType === "response.function_call_arguments.delta") { + return { pending: updateResponsesToolCallArguments(pending, event.item_id, (current) => current + event.delta), parts: [] as const }; + } + + if (eventType === "response.function_call_arguments.done") { + return { pending: updateResponsesToolCallArguments(pending, event.item_id, () => event.arguments), parts: [] as const }; + } + + if (eventType === "response.output_item.added") { + if (event.item.type === "function_call") { + return { pending: startResponsesToolCall(pending, event.item, event.output_index), parts: [] as const }; + } + return { pending, parts: [] as const }; + } + + if (eventType === "response.incomplete") { + const truncated = event.response.incomplete_details?.reason === "max_output_tokens"; + return { pending, parts: truncated ? ([{ kind: "truncated" }] as const) : ([] as const) }; + } + + return { pending, parts: [] as const }; +}; + +export const applyChunk = (pending: PendingToolCalls, chunk: ChatCompletionChunk | ResponseStreamEvent): ChunkOutcome => { + if ("type" in chunk) { + return applyResponsesEvent(pending, chunk as ResponseStreamEvent); + } + return applyChatChunk(pending, chunk as ChatCompletionChunk); +}; diff --git a/vscode-extension/src/vscode-pricing.d.ts b/vscode-extension/src/vscode-pricing.d.ts new file mode 100644 index 00000000000..099c0ecded7 --- /dev/null +++ b/vscode-extension/src/vscode-pricing.d.ts @@ -0,0 +1,8 @@ +declare module "vscode" { + interface LanguageModelChatInformation { + readonly inputCost?: number; + readonly outputCost?: number; + readonly pricing?: string; + readonly priceCategory?: string; + } +} \ No newline at end of file diff --git a/vscode-extension/test/gateway.test.ts b/vscode-extension/test/gateway.test.ts index 3a70a68b377..a0b7479448b 100644 --- a/vscode-extension/test/gateway.test.ts +++ b/vscode-extension/test/gateway.test.ts @@ -202,4 +202,87 @@ describe("streamChatCompletion", () => { ).rejects.toThrow(/upstream unavailable/); expect(gateway.requests).toHaveLength(1); }); + + it("retries one initial 504 before returning the response stream", async () => { + const gateway = await startGateway((_request, response) => { + if (gateway.requests.length === 1) { + response.writeHead(504, { "content-type": "text/html" }); + response.end("Gateway Time-out"); + return; + } + sse(response, [ + { id: "c", object: "chat.completion.chunk", created: 0, model: "gpt-5.6", choices: [{ index: 0, delta: { content: "Hi" }, finish_reason: null }] }, + { id: "c", object: "chat.completion.chunk", created: 0, model: "gpt-5.6", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, + ]); + }); + const params = buildChatCompletionParams({ + model: "gpt-5.6", + messages: [{ role: 1, content: [{ value: "hello" }], name: undefined }], + tools: [], + requireToolCall: false, + reasoningEffort: undefined, + modelOptions: {}, + }); + + const chunks = await createGatewayClient().streamChatCompletion( + { baseUrl: gateway.url, apiKey: "sk-test" }, + params, + new AbortController().signal, + ); + const contents: string[] = []; + for await (const chunk of chunks) { + contents.push(chunk.choices[0]?.delta.content ?? ""); + } + + expect(contents.join("")).toBe("Hi"); + expect(gateway.requests).toHaveLength(2); + }); + + it("stops after one retry when the gateway keeps returning 504", async () => { + const gateway = await startGateway((_request, response) => { + response.writeHead(504, { "content-type": "text/html" }); + response.end("Gateway Time-out"); + }); + const params = buildChatCompletionParams({ + model: "gpt-5.6", + messages: [{ role: 1, content: [{ value: "hello" }], name: undefined }], + tools: [], + requireToolCall: false, + reasoningEffort: undefined, + modelOptions: {}, + }); + + await expect( + createGatewayClient().streamChatCompletion( + { baseUrl: gateway.url, apiKey: "sk-test" }, + params, + new AbortController().signal, + ), + ).rejects.toMatchObject({ status: 504 }); + expect(gateway.requests).toHaveLength(2); + }); + + it("does not retry a 504 after cancellation", async () => { + const controller = new AbortController(); + const fetchImpl: typeof fetch = async () => { + controller.abort(); + return new Response("Gateway Time-out", { status: 504 }); + }; + const params = buildChatCompletionParams({ + model: "gpt-5.6", + messages: [{ role: 1, content: [{ value: "hello" }], name: undefined }], + tools: [], + requireToolCall: false, + reasoningEffort: undefined, + modelOptions: {}, + }); + + await expect( + createGatewayClient(fetchImpl).streamChatCompletion( + { baseUrl: "http://127.0.0.1:4000", apiKey: "sk-test" }, + params, + controller.signal, + ), + ).rejects.toMatchObject({ status: 504 }); + }); }); diff --git a/vscode-extension/test/messages.test.ts b/vscode-extension/test/messages.test.ts index ba11e479ad2..567f8154359 100644 --- a/vscode-extension/test/messages.test.ts +++ b/vscode-extension/test/messages.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type * as vscode from "vscode"; -import { ESTIMATED_TOKENS_PER_IMAGE, buildChatCompletionParams, estimateMessageTokens, toChatCompletionMessages, type ChatRequestInput } from "../src/messages"; +import { ESTIMATED_TOKENS_PER_IMAGE, buildChatCompletionParams, buildResponsesCompletionParams, estimateMessageTokens, toChatCompletionMessages, type ChatRequestInput } from "../src/messages"; const USER = 1 as vscode.LanguageModelChatMessageRole; const ASSISTANT = 2 as vscode.LanguageModelChatMessageRole; @@ -166,3 +166,68 @@ describe("buildChatCompletionParams", () => { expect(params).not.toHaveProperty("foo"); }); }); + +describe("buildResponsesCompletionParams", () => { + it("sends the conversation as input items, never as chat messages", () => { + const params = buildResponsesCompletionParams( + request({ + messages: [ + message(SYSTEM, [text("be terse")]), + message(USER, [text("read a"), image([1, 2, 3])]), + message(ASSISTANT, [text("checking"), toolCall("call_1", "read_file", { path: "a.ts" })]), + message(USER, [toolResult("call_1", [text("export const a = 1;")]), text("thanks")]), + ], + }), + ); + expect(params).not.toHaveProperty("messages"); + expect(params).toMatchObject({ model: "gpt-5.6", stream: true }); + expect(params.input).toEqual([ + { type: "message", role: "system", content: "be terse" }, + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "read a" }, + { type: "input_image", image_url: "data:image/png;base64,AQID", detail: "auto" }, + ], + }, + { type: "message", role: "assistant", content: "checking" }, + { type: "function_call", call_id: "call_1", name: "read_file", arguments: '{"path":"a.ts"}' }, + { type: "function_call_output", call_id: "call_1", output: "export const a = 1;" }, + { type: "message", role: "user", content: [{ type: "input_text", text: "thanks" }] }, + ]); + }); + + it("declares flat function tools and requires a call only when VS Code does", () => { + const tools: readonly vscode.LanguageModelChatTool[] = [ + { name: "read_file", description: "Read a file", inputSchema: { type: "object", properties: { path: { type: "string" } } } }, + { name: "noop", description: "No input" }, + ]; + const auto = buildResponsesCompletionParams(request({ tools })); + expect(auto.tools).toEqual([ + { + type: "function", + name: "read_file", + description: "Read a file", + parameters: { type: "object", properties: { path: { type: "string" } } }, + strict: false, + }, + { type: "function", name: "noop", description: "No input", parameters: { type: "object", properties: {} }, strict: false }, + ]); + expect(auto.tool_choice).toBeUndefined(); + expect(buildResponsesCompletionParams(request({ tools, requireToolCall: true })).tool_choice).toBe("required"); + expect(buildResponsesCompletionParams(request({ requireToolCall: true })).tool_choice).toBeUndefined(); + }); + + it("nests the picked reasoning effort and maps max_tokens to max_output_tokens", () => { + expect(buildResponsesCompletionParams(request({ reasoningEffort: "high" })).reasoning).toEqual({ effort: "high" }); + expect(buildResponsesCompletionParams(request()).reasoning).toBeUndefined(); + const params = buildResponsesCompletionParams( + request({ modelOptions: { temperature: 0.2, max_tokens: 500, seed: 7, top_p: 0.9, presence_penalty: 1 } }), + ); + expect(params).toMatchObject({ temperature: 0.2, top_p: 0.9, max_output_tokens: 500 }); + expect(params).not.toHaveProperty("max_tokens"); + expect(params).not.toHaveProperty("seed"); + expect(params).not.toHaveProperty("presence_penalty"); + }); +}); diff --git a/vscode-extension/test/models.test.ts b/vscode-extension/test/models.test.ts index 584b71ef8d4..2634bffeef7 100644 --- a/vscode-extension/test/models.test.ts +++ b/vscode-extension/test/models.test.ts @@ -85,6 +85,10 @@ describe("describeModels", () => { name: "gpt-5.6", family: "gpt-5.6", detail: "$4.00 in / $20.00 out per 1M tokens", + inputCost: 4, + outputCost: 20, + pricing: "$4.00 in / $20.00 out per 1M tokens", + priceCategory: "high", maxInputTokens: 922000, maxOutputTokens: 128000, imageInput: true, @@ -136,6 +140,12 @@ describe("describeModels", () => { expect(models.map((model) => model.id)).toEqual(["unlabeled", "gpt-5.6"]); }); + it("includes responses mode groups", () => { + const models = describeModels(parsed(gatewayGroup({ model_group: "responses-model", mode: "responses" }))); + expect(models.map((model) => model.id)).toEqual(["responses-model"]); + expect(models[0]?.maxInputTokens).toBeGreaterThanOrEqual(100000); + }); + it("falls back to assumed context limits and says so", () => { const [model] = describeModels(parsed(gatewayGroup({ max_input_tokens: null, max_output_tokens: null }))); expect(model).toMatchObject({ maxInputTokens: ASSUMED_MAX_INPUT_TOKENS, maxOutputTokens: ASSUMED_MAX_OUTPUT_TOKENS }); @@ -151,9 +161,12 @@ describe("describeModels", () => { ), ); expect(both?.detail).toBe("No pricing configured"); + expect(both?.inputCost).toBeUndefined(); + expect(both?.pricing).toBeUndefined(); expect(both?.tooltip).toContain("Input: no price configured"); expect(inputOnly?.detail).toBe("$4.00 in / n/a out per 1M tokens"); expect(free?.detail).toBe("$0.00 in / $0.00 out per 1M tokens"); + expect(free).toMatchObject({ pricing: "$0.00 in / $0.00 out per 1M tokens", priceCategory: "low" }); }); }); diff --git a/vscode-extension/test/provider.test.ts b/vscode-extension/test/provider.test.ts index 07fc4ea5b59..a9baa833d3e 100644 --- a/vscode-extension/test/provider.test.ts +++ b/vscode-extension/test/provider.test.ts @@ -1,9 +1,10 @@ import type { ChatCompletionChunk, ChatCompletionCreateParamsStreaming } from "openai/resources/chat/completions"; +import { APIError } from "openai"; import { describe, expect, it } from "vitest"; import type * as vscode from "vscode"; import type { GatewayClient, GatewayConfig, ModelGroupsResult } from "../src/gateway"; -import { ESTIMATED_TOKENS_PER_IMAGE } from "../src/messages"; -import { LiteLLMChatProvider, TRUNCATED_MESSAGE, type LiteLLMModel } from "../src/provider"; +import { ESTIMATED_TOKENS_PER_IMAGE, MODE_RESPONSES } from "../src/messages"; +import { GATEWAY_TIMEOUT_MESSAGE, LiteLLMChatProvider, TRUNCATED_MESSAGE, type LiteLLMModel } from "../src/provider"; import { CancellationTokenSource, LanguageModelTextPart, LanguageModelToolCallPart } from "./vscode-mock"; interface StreamRequest { @@ -14,6 +15,7 @@ interface StreamRequest { interface FakeGateway extends GatewayClient { readonly listCalls: readonly GatewayConfig[]; readonly streamRequests: readonly StreamRequest[]; + readonly responsesRequests: readonly any[]; } const chunk = (delta: ChatCompletionChunk.Choice.Delta, finishReason: ChatCompletionChunk.Choice["finish_reason"] = null): ChatCompletionChunk => ({ @@ -24,6 +26,14 @@ const chunk = (delta: ChatCompletionChunk.Choice.Delta, finishReason: ChatComple choices: [{ index: 0, delta, finish_reason: finishReason }], }); +const responsesEvent = (type: string, overrides?: Record): any => ({ + type, + object: "response", + id: "resp_test", + created_at: 1234567890, + ...overrides, +}); + const modelGroups: ModelGroupsResult = { kind: "ok", groups: [ @@ -45,12 +55,15 @@ const modelGroups: ModelGroupsResult = { const fakeGateway = ( listResult: ModelGroupsResult = modelGroups, stream: (signal: AbortSignal) => AsyncIterable = () => (async function* () {})(), + responsesStream?: (signal: AbortSignal) => AsyncIterable, ): FakeGateway => { const listCalls: GatewayConfig[] = []; const streamRequests: StreamRequest[] = []; + const responsesRequests: any[] = []; return { listCalls, streamRequests, + responsesRequests, async listModelGroups(config) { listCalls.push(config); return listResult; @@ -59,6 +72,10 @@ const fakeGateway = ( streamRequests.push({ config, params }); return stream(signal); }, + async streamResponsesCompletion(config, params, signal) { + responsesRequests.push({ config, params }); + return responsesStream ? responsesStream(signal) : (async function* () {})(); + }, }; }; @@ -78,6 +95,7 @@ const model = (overrides: Partial = {}): LiteLLMModel => ({ maxOutputTokens: 128000, capabilities: { imageInput: true, toolCalling: true }, gateway, + mode: "chat", ...overrides, }); @@ -134,10 +152,15 @@ describe("provideLanguageModelChatInformation", () => { expect.objectContaining({ id: "gpt-5.6", detail: "$4.00 in / $20.00 out per 1M tokens", + inputCost: 4, + outputCost: 20, + pricing: "$4.00 in / $20.00 out per 1M tokens", + priceCategory: "high", maxInputTokens: 922000, capabilities: { imageInput: true, toolCalling: true }, configurationSchema: expect.objectContaining({ properties: expect.objectContaining({ reasoningEffort: expect.anything() }) }), gateway, + mode: "chat", }), ]); }); @@ -153,6 +176,33 @@ describe("provideLanguageModelChatInformation", () => { ), ).rejects.toThrow("LiteLLM gateway at http://127.0.0.1:4000 answered 401 for /model_group/info: Authentication Error, Invalid proxy server token passed"); }); + + it("lists responses mode models preserved from gateway data", async () => { + const responsesGroups: ModelGroupsResult = { + kind: "ok", + groups: [ + { + modelGroup: "responses-model", + providers: ["openai"], + mode: MODE_RESPONSES, + maxInputTokens: 128000, + maxOutputTokens: 16000, + inputCostPerToken: 5e-6, + outputCostPerToken: 2.5e-5, + supportsVision: false, + supportsFunctionCalling: false, + supportedReasoningEfforts: ["default", "low", "high"], + }, + ], + }; + const client = fakeGateway(responsesGroups); + const models = await new LiteLLMChatProvider(client).provideLanguageModelChatInformation( + prepare({ baseUrl: "http://127.0.0.1:4000/v1/", apiKey: "sk-test" }), + token(), + ); + expect(models.length).toBe(1); + expect(models[0]).toMatchObject({ id: "responses-model", mode: MODE_RESPONSES }); + }); }); describe("provideLanguageModelChatResponse", () => { @@ -210,6 +260,18 @@ describe("provideLanguageModelChatResponse", () => { expect(cancellation.disposedListeners).toBe(1); }); + it("explains a gateway 504 and suggests checking upstream health", async () => { + const timeout = new APIError(504, undefined, "Gateway Time-out", new Headers()); + const client = fakeGateway(modelGroups, () => { + throw timeout; + }); + const { run } = collect(new LiteLLMChatProvider(client)); + + await expect(run).rejects.toThrow( + GATEWAY_TIMEOUT_MESSAGE, + ); + }); + it("reports the text it got and then fails when the model hits its output limit", async () => { const client = fakeGateway(modelGroups, () => (async function* () { @@ -231,6 +293,35 @@ describe("provideLanguageModelChatResponse", () => { const { run } = collect(new LiteLLMChatProvider(client)); await expect(run).rejects.toThrow("invalid JSON arguments for tool grep"); }); + + it("calls responses API for mode='responses' models and preserves mode", async () => { + const responsesEvents = (signal: AbortSignal): AsyncIterable => ({ + [Symbol.asyncIterator]: async function* () { + yield responsesEvent("response.created", { output: [] }); + yield responsesEvent("response.output_text.delta", { content_index: 0, delta: "Hello" }); + yield responsesEvent("response.output_text.delta", { content_index: 0, delta: " world" }); + yield responsesEvent("response.output_text.done", { content_index: 0, text: "Hello world" }); + yield responsesEvent("response.completed", { status: "completed" }); + }, + }); + const client = fakeGateway(modelGroups, () => (async function* () {})(), responsesEvents); + const parts: vscode.LanguageModelResponsePart[] = []; + await new LiteLLMChatProvider(client).provideLanguageModelChatResponse( + model({ mode: MODE_RESPONSES }), + [userMessage([{ value: "hi" }])], + responseOptions(), + { report: (part) => parts.push(part) }, + token(), + ); + expect(client.responsesRequests).toHaveLength(1); + expect(client.responsesRequests[0].params).toMatchObject({ + model: "gpt-5.6", + stream: true, + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }], + }); + expect(client.responsesRequests[0].params).not.toHaveProperty("messages"); + expect(parts).toEqual([new LanguageModelTextPart("Hello"), new LanguageModelTextPart(" world")]); + }); }); describe("provideTokenCount", () => { diff --git a/vscode-extension/test/stream.test.ts b/vscode-extension/test/stream.test.ts index eb37013fb10..40dcc227034 100644 --- a/vscode-extension/test/stream.test.ts +++ b/vscode-extension/test/stream.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import type { ChatCompletionChunk } from "openai/resources/chat/completions"; +import type { ResponseStreamEvent } from "openai/resources/responses/responses"; import { responseParts, type ResponsePart } from "../src/stream"; type ToolCallDelta = NonNullable[number]; @@ -91,3 +92,55 @@ describe("responseParts", () => { ]); }); }); + +describe("responseParts with Responses API events", () => { + const collectEvents = async (events: readonly object[]): Promise => { + async function* source(): AsyncGenerator { + yield* events as readonly ResponseStreamEvent[]; + } + const parts: ResponsePart[] = []; + for await (const part of responseParts(source())) { + parts.push(part); + } + return parts; + }; + + it("reports a function call under its call_id, not the output item id", async () => { + expect( + await collectEvents([ + { type: "response.output_item.added", output_index: 0, item: { type: "function_call", id: "fc_1", call_id: "call_1", name: "read_file", arguments: "" } }, + { type: "response.function_call_arguments.delta", item_id: "fc_1", output_index: 0, delta: '{"pa' }, + { type: "response.function_call_arguments.delta", item_id: "fc_1", output_index: 0, delta: 'th":"a"}' }, + { type: "response.function_call_arguments.done", item_id: "fc_1", output_index: 0, arguments: '{"path":"a"}' }, + { type: "response.completed", response: {} }, + ]), + ).toEqual([{ kind: "tool_call", callId: "call_1", name: "read_file", input: { path: "a" } }]); + }); + + it("keeps parallel calls apart and orders them by output index", async () => { + expect( + await collectEvents([ + { type: "response.output_item.added", output_index: 1, item: { type: "function_call", id: "fc_b", call_id: "call_b", name: "grep", arguments: "" } }, + { type: "response.output_item.added", output_index: 0, item: { type: "function_call", id: "fc_a", call_id: "call_a", name: "read_file", arguments: "" } }, + { type: "response.function_call_arguments.delta", item_id: "fc_a", output_index: 0, delta: '{"p":1}' }, + { type: "response.function_call_arguments.delta", item_id: "fc_b", output_index: 1, delta: '{"q":2}' }, + ]), + ).toEqual([ + { kind: "tool_call", callId: "call_a", name: "read_file", input: { p: 1 } }, + { kind: "tool_call", callId: "call_b", name: "grep", input: { q: 2 } }, + ]); + }); + + it("streams output text and flags a response cut off at max_output_tokens", async () => { + expect( + await collectEvents([ + { type: "response.output_text.delta", delta: "half" }, + { type: "response.incomplete", response: { incomplete_details: { reason: "max_output_tokens" } } }, + ]), + ).toEqual([{ kind: "text", value: "half" }, { kind: "truncated" }]); + }); + + it("does not flag an incomplete response that stopped for another reason", async () => { + expect(await collectEvents([{ type: "response.incomplete", response: { incomplete_details: { reason: "content_filter" } } }])).toEqual([]); + }); +});