From f4873804cafa173178b5555d012276fc9627a38b Mon Sep 17 00:00:00 2001 From: Roo Code Date: Mon, 22 Sep 2025 20:59:04 +0000 Subject: [PATCH] fix: handle empty responses from Docker Model Runner and similar APIs - Add fallback to yield empty text chunk when no content is received - Prevents "The language model did not provide any assistant messages" error - Fixes issue with Docker Model Runner integration - Add tests for empty response handling Fixes #8226 --- .../__tests__/base-openai-compatible.spec.ts | 246 ++++++++++++++++++ .../base-openai-compatible-provider.ts | 12 + src/api/providers/lm-studio.ts | 15 ++ 3 files changed, 273 insertions(+) create mode 100644 src/api/providers/__tests__/base-openai-compatible.spec.ts diff --git a/src/api/providers/__tests__/base-openai-compatible.spec.ts b/src/api/providers/__tests__/base-openai-compatible.spec.ts new file mode 100644 index 0000000000..ecf09cf817 --- /dev/null +++ b/src/api/providers/__tests__/base-openai-compatible.spec.ts @@ -0,0 +1,246 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { Anthropic } from "@anthropic-ai/sdk" +import OpenAI from "openai" + +import { BaseOpenAiCompatibleProvider } from "../base-openai-compatible-provider" +import { type ModelInfo } from "@roo-code/types" + +// Create a concrete implementation for testing +class TestOpenAiCompatibleProvider extends BaseOpenAiCompatibleProvider<"test-model"> { + constructor(options: any) { + super({ + providerName: "TestProvider", + baseURL: "https://test.api.com/v1", + defaultProviderModelId: "test-model", + providerModels: { + "test-model": { + maxTokens: 4096, + contextWindow: 8192, + supportsPromptCache: false, + supportsComputerUse: false, + supportsStreaming: true, + inputPrice: 0, + outputPrice: 0, + } as ModelInfo, + }, + ...options, + }) + } +} + +describe("BaseOpenAiCompatibleProvider", () => { + let provider: TestOpenAiCompatibleProvider + let mockCreate: ReturnType + + beforeEach(() => { + mockCreate = vi.fn() + vi.spyOn(OpenAI.Chat.Completions.prototype, "create").mockImplementation(mockCreate) + }) + + describe("createMessage", () => { + it("should handle empty response from API gracefully", async () => { + provider = new TestOpenAiCompatibleProvider({ + apiKey: "test-key", + }) + + // Mock an empty stream response (no content in chunks) + const mockStream = { + async *[Symbol.asyncIterator]() { + // Yield chunks with no content + yield { + choices: [ + { + delta: { + // No content field + }, + }, + ], + } + // Yield usage information + yield { + choices: [ + { + delta: {}, + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 0, + }, + } + }, + } + + mockCreate.mockResolvedValue(mockStream) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello", + }, + ] + + const stream = provider.createMessage(systemPrompt, messages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have at least one text chunk (even if empty) and one usage chunk + expect(chunks).toHaveLength(2) + + // Should have a usage chunk + const usageChunk = chunks.find((c) => c.type === "usage") + expect(usageChunk).toEqual({ + type: "usage", + inputTokens: 100, + outputTokens: 0, + }) + + // Should have an empty text chunk (added by our fix) + const textChunk = chunks.find((c) => c.type === "text") + expect(textChunk).toEqual({ + type: "text", + text: "", + }) + }) + + it("should handle normal response with content", async () => { + provider = new TestOpenAiCompatibleProvider({ + apiKey: "test-key", + }) + + // Mock a normal stream response with content + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [ + { + delta: { + content: "Hello, ", + }, + }, + ], + } + yield { + choices: [ + { + delta: { + content: "world!", + }, + }, + ], + } + yield { + choices: [ + { + delta: {}, + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 10, + }, + } + }, + } + + mockCreate.mockResolvedValue(mockStream) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello", + }, + ] + + const stream = provider.createMessage(systemPrompt, messages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have text chunks and usage chunk + expect(chunks).toHaveLength(3) + + // First two chunks should be text + expect(chunks[0]).toEqual({ + type: "text", + text: "Hello, ", + }) + expect(chunks[1]).toEqual({ + type: "text", + text: "world!", + }) + + // Last chunk should be usage information + expect(chunks[2]).toEqual({ + type: "usage", + inputTokens: 100, + outputTokens: 10, + }) + }) + + it("should handle response with only usage and no content", async () => { + provider = new TestOpenAiCompatibleProvider({ + apiKey: "test-key", + }) + + // Mock a stream response with only usage, no content + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [ + { + delta: {}, + }, + ], + usage: { + prompt_tokens: 50, + completion_tokens: 0, + }, + } + }, + } + + mockCreate.mockResolvedValue(mockStream) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Test", + }, + ] + + const stream = provider.createMessage(systemPrompt, messages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have empty text chunk and usage chunk + expect(chunks).toHaveLength(2) + + // Should have a usage chunk + const usageChunk = chunks.find((c) => c.type === "usage") + expect(usageChunk).toEqual({ + type: "usage", + inputTokens: 50, + outputTokens: 0, + }) + + // Should have an empty text chunk (added by our fix) + const textChunk = chunks.find((c) => c.type === "text") + expect(textChunk).toEqual({ + type: "text", + text: "", + }) + }) + }) +}) diff --git a/src/api/providers/base-openai-compatible-provider.ts b/src/api/providers/base-openai-compatible-provider.ts index fb6c5d0377..b5b4414f1f 100644 --- a/src/api/providers/base-openai-compatible-provider.ts +++ b/src/api/providers/base-openai-compatible-provider.ts @@ -98,11 +98,13 @@ export abstract class BaseOpenAiCompatibleProvider metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { const stream = await this.createStream(systemPrompt, messages, metadata) + let hasContent = false for await (const chunk of stream) { const delta = chunk.choices[0]?.delta if (delta?.content) { + hasContent = true yield { type: "text", text: delta.content, @@ -117,6 +119,16 @@ export abstract class BaseOpenAiCompatibleProvider } } } + + // If no content was received, yield an empty text chunk to prevent + // "The language model did not provide any assistant messages" error + // This can happen with some Docker Model Runner configurations + if (!hasContent) { + yield { + type: "text", + text: "", + } + } } async completePrompt(prompt: string): Promise { diff --git a/src/api/providers/lm-studio.ts b/src/api/providers/lm-studio.ts index 6c58a96ae1..2256c02b09 100644 --- a/src/api/providers/lm-studio.ts +++ b/src/api/providers/lm-studio.ts @@ -80,6 +80,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan } let assistantText = "" + let hasContent = false try { const params: OpenAI.Chat.ChatCompletionCreateParamsStreaming & { draft_model?: string } = { @@ -113,6 +114,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan const delta = chunk.choices[0]?.delta if (delta?.content) { + hasContent = true assistantText += delta.content for (const processedChunk of matcher.update(delta.content)) { yield processedChunk @@ -121,9 +123,22 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan } for (const processedChunk of matcher.final()) { + if (processedChunk.text) { + hasContent = true + } yield processedChunk } + // If no content was received, yield an empty text chunk to prevent + // "The language model did not provide any assistant messages" error + // This can happen with some model configurations + if (!hasContent) { + yield { + type: "text", + text: "", + } + } + let outputTokens = 0 try { outputTokens = await this.countTokens([{ type: "text", text: assistantText }])