diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index ae36fc1399..2525d8c1ab 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -45,6 +45,51 @@ vitest.mock("../fetchers/modelCache", () => ({ }), })) +// Mock XmlMatcher +vitest.mock("../../../utils/xml-matcher", () => ({ + XmlMatcher: vitest.fn().mockImplementation((tagName, transform) => { + return { + update: vitest.fn((chunk) => { + const results = [] + // Simple mock implementation for testing + const toolCallRegex = new RegExp(`<${tagName}>(.+?)`, "g") + let lastIndex = 0 + let match + + while ((match = toolCallRegex.exec(chunk)) !== null) { + // Add text before the match + if (match.index > lastIndex) { + results.push({ + type: tagName, + data: chunk.substring(lastIndex, match.index), + matched: false, + }) + } + // Add the matched content + results.push({ + type: tagName, + data: match[1], + matched: true, + }) + lastIndex = toolCallRegex.lastIndex + } + + // Add remaining text + if (lastIndex < chunk.length) { + results.push({ + type: tagName, + data: chunk.substring(lastIndex), + matched: false, + }) + } + + return transform ? results.map(transform) : results + }), + final: vitest.fn(() => []), + } + }), +})) + describe("OpenRouterHandler", () => { const mockOptions: ApiHandlerOptions = { openRouterApiKey: "test-key", @@ -320,4 +365,352 @@ describe("OpenRouterHandler", () => { await expect(handler.completePrompt("test prompt")).rejects.toThrow("Unexpected error") }) }) + + describe("GPT-OSS tool calling support", () => { + it("handles standard OpenAI-style tool calls in stream", async () => { + const handler = new OpenRouterHandler({ + ...mockOptions, + openRouterModelId: "openai/gpt-oss-120b", + }) + + const mockStream = { + async *[Symbol.asyncIterator]() { + // Simulate tool call chunks + yield { + id: "test-id", + choices: [ + { + delta: { + tool_calls: [ + { + id: "tool_1", + function: { name: "get_weather" }, + }, + ], + }, + }, + ], + } + yield { + id: "test-id", + choices: [ + { + delta: { + tool_calls: [ + { + function: { arguments: '{"location": ' }, + }, + ], + }, + }, + ], + } + yield { + id: "test-id", + choices: [ + { + delta: { + tool_calls: [ + { + function: { arguments: '"San Francisco"}' }, + }, + ], + }, + }, + ], + } + yield { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 10, completion_tokens: 20 }, + } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + const chunks = [] + + for await (const chunk of generator) { + chunks.push(chunk) + } + + // Should have tool call chunk and usage chunk + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ + type: "tool_call", + id: "tool_1", + name: "get_weather", + arguments: '{"location": "San Francisco"}', + }) + expect(chunks[1]).toEqual({ + type: "usage", + inputTokens: 10, + outputTokens: 20, + cacheReadTokens: undefined, + reasoningTokens: undefined, + totalCost: 0, + }) + }) + + it("handles tool calls within reasoning blocks for GPT-OSS models", async () => { + const handler = new OpenRouterHandler({ + ...mockOptions, + openRouterModelId: "openai/gpt-oss-20b", + }) + + const mockStream = { + async *[Symbol.asyncIterator]() { + // Simulate reasoning with embedded tool call + yield { + id: "test-id", + choices: [ + { + delta: { + reasoning: + 'Let me check the weather. get_weather{"location": "New York"}', + }, + }, + ], + } + yield { + id: "test-id", + choices: [ + { + delta: { + content: "The weather in New York is sunny.", + }, + }, + ], + } + yield { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 15, completion_tokens: 25 }, + } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + const chunks = [] + + for await (const chunk of generator) { + chunks.push(chunk) + } + + // Should have reasoning, tool call, text, and usage chunks + expect(chunks).toHaveLength(4) + expect(chunks[0]).toEqual({ + type: "reasoning", + text: "Let me check the weather. ", + }) + expect(chunks[1]).toEqual({ + type: "tool_call", + id: "tool_1", + name: "get_weather", + arguments: '{"location": "New York"}', + }) + expect(chunks[2]).toEqual({ + type: "text", + text: "The weather in New York is sunny.", + }) + expect(chunks[3]).toEqual({ + type: "usage", + inputTokens: 15, + outputTokens: 25, + cacheReadTokens: undefined, + reasoningTokens: undefined, + totalCost: 0, + }) + }) + + it("handles multiple tool calls in reasoning for GPT-OSS", async () => { + const handler = new OpenRouterHandler({ + ...mockOptions, + openRouterModelId: "openai/gpt-oss-120b", + }) + + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + id: "test-id", + choices: [ + { + delta: { + reasoning: + 'I\'ll check multiple things. get_weather{"location": "LA"} and then get_time{"timezone": "PST"}', + }, + }, + ], + } + yield { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 20, completion_tokens: 30 }, + } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + const chunks = [] + + for await (const chunk of generator) { + chunks.push(chunk) + } + + // Should have reasoning text, two tool calls, more reasoning text, and usage + expect(chunks).toHaveLength(5) + expect(chunks[0]).toEqual({ + type: "reasoning", + text: "I'll check multiple things. ", + }) + expect(chunks[1]).toEqual({ + type: "tool_call", + id: "tool_1", + name: "get_weather", + arguments: '{"location": "LA"}', + }) + expect(chunks[2]).toEqual({ + type: "reasoning", + text: " and then ", + }) + expect(chunks[3]).toEqual({ + type: "tool_call", + id: "tool_2", + name: "get_time", + arguments: '{"timezone": "PST"}', + }) + expect(chunks[4]).toEqual({ + type: "usage", + inputTokens: 20, + outputTokens: 30, + cacheReadTokens: undefined, + reasoningTokens: undefined, + totalCost: 0, + }) + }) + + it("handles non-GPT-OSS models without tool call parsing in reasoning", async () => { + const handler = new OpenRouterHandler({ + ...mockOptions, + openRouterModelId: "anthropic/claude-sonnet-4", + }) + + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + id: "test-id", + choices: [ + { + delta: { + reasoning: "This contains but should not be parsed as tool call", + }, + }, + ], + } + yield { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 5, completion_tokens: 10 }, + } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + const chunks = [] + + for await (const chunk of generator) { + chunks.push(chunk) + } + + // Should only have reasoning and usage chunks, no tool call parsing + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ + type: "reasoning", + text: "This contains but should not be parsed as tool call", + }) + expect(chunks[1]).toEqual({ + type: "usage", + inputTokens: 5, + outputTokens: 10, + cacheReadTokens: undefined, + reasoningTokens: undefined, + totalCost: 0, + }) + }) + + it("handles malformed tool calls gracefully", async () => { + const handler = new OpenRouterHandler({ + ...mockOptions, + openRouterModelId: "openai/gpt-oss-20b", + }) + + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + id: "test-id", + choices: [ + { + delta: { + reasoning: "Invalid tool call: missing closing tag", + }, + }, + ], + } + yield { + id: "test-id", + choices: [ + { + delta: { + reasoning: " and another no_args", + }, + }, + ], + } + yield { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 5, completion_tokens: 10 }, + } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + const chunks = [] + + for await (const chunk of generator) { + chunks.push(chunk) + } + + // Should handle malformed tool calls gracefully + // The first malformed one should be treated as reasoning text + // The second one without arguments should be ignored + expect(chunks.some((chunk) => chunk.type === "reasoning")).toBe(true) + expect(chunks[chunks.length - 1].type).toBe("usage") + }) + }) }) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 6565daa238..9ba4318c16 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -18,6 +18,7 @@ import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transfor import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" import type { OpenRouterReasoningParams } from "../transform/reasoning" import { getModelParams } from "../transform/model-params" +import { XmlMatcher } from "../../utils/xml-matcher" import { getModels } from "./fetchers/modelCache" import { getModelEndpoints } from "./fetchers/modelEndpointCache" @@ -138,6 +139,20 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH let lastUsage: CompletionUsage | undefined = undefined + // For GPT-OSS models, we need to handle tool calls that may appear within reasoning blocks + const isGptOss = modelId.includes("gpt-oss") + const toolCallMatcher = isGptOss + ? new XmlMatcher("tool_call", (chunk) => ({ + type: "tool_call" as const, + data: chunk.data, + matched: chunk.matched, + })) + : null + + // Track accumulated tool call data for streaming + let currentToolCall: { id?: string; name?: string; arguments?: string } | null = null + let toolCallIdCounter = 0 + for await (const chunk of stream) { // OpenRouter returns an error object instead of the OpenAI SDK throwing an error. if ("error" in chunk) { @@ -148,19 +163,121 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH const delta = chunk.choices[0]?.delta + // Handle reasoning content (which may contain tool calls for GPT-OSS) if ("reasoning" in delta && delta.reasoning && typeof delta.reasoning === "string") { - yield { type: "reasoning", text: delta.reasoning } + if (isGptOss && toolCallMatcher) { + // Process reasoning content through the tool call matcher + for (const matchChunk of toolCallMatcher.update(delta.reasoning)) { + if (matchChunk.type === "tool_call" && matchChunk.matched) { + // Parse the tool call from the matched content + try { + const toolCallContent = matchChunk.data + // Extract tool name and arguments from the XML-like format + const nameMatch = toolCallContent.match(/([^<]+)<\/name>/) + const argsMatch = toolCallContent.match(/([\s\S]*?)<\/arguments>/) + + if (nameMatch && argsMatch) { + // Emit a tool call chunk + yield { + type: "tool_call" as const, + id: `tool_${++toolCallIdCounter}`, + name: nameMatch[1], + arguments: argsMatch[1], + } + } + } catch (e) { + console.warn("Failed to parse tool call from reasoning:", e) + // If parsing fails, treat it as regular reasoning text + yield { type: "reasoning", text: delta.reasoning } + } + } else { + // Regular reasoning text + yield { type: "reasoning", text: matchChunk.data } + } + } + } else { + // Non-GPT-OSS models or no tool call matching + yield { type: "reasoning", text: delta.reasoning } + } } + // Handle regular content if (delta?.content) { yield { type: "text", text: delta.content } } + // Handle standard OpenAI-style tool calls (if they appear in the stream) + if (delta?.tool_calls) { + for (const toolCall of delta.tool_calls) { + if (toolCall.id) { + // Start of a new tool call + if (currentToolCall) { + // Emit the previous tool call if it exists + if (currentToolCall.id && currentToolCall.name && currentToolCall.arguments) { + yield { + type: "tool_call" as const, + id: currentToolCall.id, + name: currentToolCall.name, + arguments: currentToolCall.arguments, + } + } + } + currentToolCall = { + id: toolCall.id, + name: toolCall.function?.name || "", + arguments: toolCall.function?.arguments || "", + } + } else if (currentToolCall && toolCall.function) { + // Continue accumulating the current tool call + if (toolCall.function.name) { + currentToolCall.name = (currentToolCall.name || "") + toolCall.function.name + } + if (toolCall.function.arguments) { + currentToolCall.arguments = (currentToolCall.arguments || "") + toolCall.function.arguments + } + } + } + } + if (chunk.usage) { lastUsage = chunk.usage } } + // Finalize any remaining tool call matcher content + if (isGptOss && toolCallMatcher) { + for (const matchChunk of toolCallMatcher.final()) { + if (matchChunk.type === "tool_call" && matchChunk.matched) { + try { + const toolCallContent = matchChunk.data + const nameMatch = toolCallContent.match(/([^<]+)<\/name>/) + const argsMatch = toolCallContent.match(/([\s\S]*?)<\/arguments>/) + + if (nameMatch && argsMatch) { + yield { + type: "tool_call" as const, + id: `tool_${++toolCallIdCounter}`, + name: nameMatch[1], + arguments: argsMatch[1], + } + } + } catch (e) { + console.warn("Failed to parse tool call from reasoning:", e) + } + } + } + } + + // Emit any remaining accumulated tool call + if (currentToolCall && currentToolCall.id && currentToolCall.name && currentToolCall.arguments) { + yield { + type: "tool_call" as const, + id: currentToolCall.id, + name: currentToolCall.name, + arguments: currentToolCall.arguments, + } + } + if (lastUsage) { yield { type: "usage", diff --git a/src/api/transform/stream.ts b/src/api/transform/stream.ts index 89655a3f56..09f05b0925 100644 --- a/src/api/transform/stream.ts +++ b/src/api/transform/stream.ts @@ -1,6 +1,11 @@ export type ApiStream = AsyncGenerator -export type ApiStreamChunk = ApiStreamTextChunk | ApiStreamUsageChunk | ApiStreamReasoningChunk | ApiStreamError +export type ApiStreamChunk = + | ApiStreamTextChunk + | ApiStreamUsageChunk + | ApiStreamReasoningChunk + | ApiStreamToolCallChunk + | ApiStreamError export interface ApiStreamError { type: "error" @@ -18,6 +23,13 @@ export interface ApiStreamReasoningChunk { text: string } +export interface ApiStreamToolCallChunk { + type: "tool_call" + id: string + name: string + arguments: string +} + export interface ApiStreamUsageChunk { type: "usage" inputTokens: number