diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index afb349e5e0..cdfdf23719 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -300,4 +300,152 @@ describe("VsCodeLmHandler", () => { await expect(promise).rejects.toThrow("VSCode LM completion error: Completion failed") }) }) + + describe("GitHub Copilot gpt-4.1 bug handling", () => { + it("should skip chunks that contain only the model ID", async () => { + const mockModel = { + ...mockLanguageModelChat, + id: "copilot/gpt-4.1", + vendor: "copilot", + family: "gpt-4.1", + } + ;(vscode.lm.selectChatModels as Mock).mockResolvedValueOnce([mockModel]) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user" as const, + content: "Hello", + }, + ] + + // Mock a response that includes the model ID as a chunk followed by actual content + mockModel.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + yield new vscode.LanguageModelTextPart("copilot/gpt-4.1") // Bug: model ID as response + yield new vscode.LanguageModelTextPart("Hello! How can I help you?") // Actual content + return + })(), + text: (async function* () { + yield "copilot/gpt-4.1" + yield "Hello! How can I help you?" + return + })(), + }) + + // Override the default client with our test client + handler["client"] = mockModel + + const stream = handler.createMessage(systemPrompt, messages) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should only have the actual content and usage, not the model ID + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ + type: "text", + text: "Hello! How can I help you?", + }) + expect(chunks[1]).toMatchObject({ + type: "usage", + inputTokens: expect.any(Number), + outputTokens: expect.any(Number), + }) + }) + + it("should throw error when only model ID is returned", async () => { + const mockModel = { + ...mockLanguageModelChat, + id: "copilot/gpt-4.1", + vendor: "copilot", + family: "gpt-4.1", + } + ;(vscode.lm.selectChatModels as Mock).mockResolvedValueOnce([mockModel]) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user" as const, + content: "Hello", + }, + ] + + // Mock a response that only contains the model ID + mockModel.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + yield new vscode.LanguageModelTextPart("copilot/gpt-4.1") + return + })(), + text: (async function* () { + yield "copilot/gpt-4.1" + return + })(), + }) + + // Override the default client with our test client + handler["client"] = mockModel + + const stream = handler.createMessage(systemPrompt, messages) + + // Collect all chunks and expect an error + await expect(async () => { + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + }).rejects.toThrow( + 'The VS Code Language Model API returned only the model ID "copilot/gpt-4.1" instead of generating a response', + ) + }) + + it("should handle variations of model ID patterns", async () => { + const mockModel = { + ...mockLanguageModelChat, + id: "github/gpt-4.1", + vendor: "github", + family: "gpt-4.1", + } + ;(vscode.lm.selectChatModels as Mock).mockResolvedValueOnce([mockModel]) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user" as const, + content: "Hello", + }, + ] + + // Mock a response with different model ID variations + mockModel.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + yield new vscode.LanguageModelTextPart(" GitHub/gpt-4.1 ") // With spaces and different case + yield new vscode.LanguageModelTextPart("Actual response content") + return + })(), + text: (async function* () { + yield " GitHub/gpt-4.1 " + yield "Actual response content" + return + })(), + }) + + // Override the default client with our test client + handler["client"] = mockModel + + const stream = handler.createMessage(systemPrompt, messages) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should skip the model ID and only return actual content + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ + type: "text", + text: "Actual response content", + }) + }) + }) }) diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index 6474371bee..6f864cfb65 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -363,6 +363,8 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan // Accumulate the text and count at the end of the stream to reduce token counting overhead. let accumulatedText: string = "" + let hasValidContent = false + let detectedModelIdAsResponse = false try { // Create the response stream with minimal required options @@ -382,12 +384,39 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan // Consume the stream and handle both text and tool call chunks for await (const chunk of response.stream) { if (chunk instanceof vscode.LanguageModelTextPart) { + // Debug logging for GitHub Copilot gpt-4.1 issue + if (client.id && (client.id.includes("gpt-4.1") || client.id.includes("copilot"))) { + console.debug("Roo Code : Processing chunk for model:", client.id, { + chunkType: typeof chunk, + chunkValue: chunk.value, + chunkValueType: typeof chunk.value, + chunkValueLength: chunk.value?.length, + isModelId: chunk.value === client.id || chunk.value === `${client.vendor}/${client.family}`, + }) + } + // Validate text part value if (typeof chunk.value !== "string") { console.warn("Roo Code : Invalid text part value received:", chunk.value) continue } + // Check if the chunk value is just the model ID (GitHub Copilot gpt-4.1 bug) + const modelIdPattern = /^(copilot|github)\/gpt-4\.1$/i + const trimmedValue = chunk.value.trim() + + if (modelIdPattern.test(trimmedValue) || trimmedValue === client.id) { + console.warn( + "Roo Code : Detected model ID as response content, this appears to be a VS Code LM API bug:", + chunk.value, + ) + detectedModelIdAsResponse = true + // Skip this chunk as it's not actual content + continue + } + + // If we get here, we have valid content + hasValidContent = true accumulatedText += chunk.value yield { type: "text", @@ -444,6 +473,13 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan } } + // If we only received the model ID and no valid content, throw an error + if (detectedModelIdAsResponse && !hasValidContent) { + throw new Error( + `Roo Code : The VS Code Language Model API returned only the model ID "${client.id}" instead of generating a response. This is a known issue with GitHub Copilot's gpt-4.1 model. Please try using a different model or wait for a fix from the VS Code/GitHub Copilot team.`, + ) + } + // Count tokens in the accumulated text after stream completion const totalOutputTokens: number = await this.internalCountTokens(accumulatedText)