diff --git a/src/api/transform/__tests__/openai-format.spec.ts b/src/api/transform/__tests__/openai-format.spec.ts index d5d4404837..111f79101f 100644 --- a/src/api/transform/__tests__/openai-format.spec.ts +++ b/src/api/transform/__tests__/openai-format.spec.ts @@ -319,7 +319,55 @@ describe("convertToOpenAiMessages", () => { ) }) - it("should NOT merge text when images are present (fall back to user message)", () => { + it("should merge text and send images separately when mergeToolResultText is true", () => { + const anthropicMessages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-123", + content: "Tool result content", + }, + { + type: "text", + text: "Context info", + }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "base64data", + }, + }, + ], + }, + ] + + const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true }) + + // Should produce a tool message with merged text AND a user message for the image + expect(openAiMessages).toHaveLength(2) + + // First message: tool message with merged text + const toolMessage = openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam + expect(toolMessage.role).toBe("tool") + expect(toolMessage.tool_call_id).toBe("tool-123") + expect(toolMessage.content).toBe( + "Tool result content\n\nContext info", + ) + + // Second message: user message with only the image + expect(openAiMessages[1].role).toBe("user") + const userContent = openAiMessages[1].content as Array<{ type: string; image_url?: { url: string } }> + expect(Array.isArray(userContent)).toBe(true) + expect(userContent).toHaveLength(1) + expect(userContent[0].type).toBe("image_url") + expect(userContent[0].image_url?.url).toBe("data:image/png;base64,base64data") + }) + + it("should send only images as user message when no text content exists with mergeToolResultText", () => { const anthropicMessages: Anthropic.Messages.MessageParam[] = [ { role: "user", @@ -343,9 +391,13 @@ describe("convertToOpenAiMessages", () => { const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true }) - // Should produce a tool message AND a user message (because image is present) + // Should produce a tool message AND a user message (only image, no text to merge) expect(openAiMessages).toHaveLength(2) expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).role).toBe("tool") + // Tool message content should NOT be modified since there's no text to merge + expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).content).toBe( + "Tool result content", + ) expect(openAiMessages[1].role).toBe("user") }) diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts index ad02be5541..f182a2aac1 100644 --- a/src/api/transform/openai-format.ts +++ b/src/api/transform/openai-format.ts @@ -138,24 +138,40 @@ export function convertToOpenAiMessages( // Process non-tool messages if (nonToolMessages.length > 0) { - // Check if we should merge text into the last tool message - // This is critical for reasoning/thinking models where a user message - // after tool results causes the model to drop all previous reasoning_content - const hasOnlyTextContent = nonToolMessages.every((part) => part.type === "text") const hasToolMessages = toolMessages.length > 0 - const shouldMergeIntoToolMessage = - options?.mergeToolResultText && hasToolMessages && hasOnlyTextContent - if (shouldMergeIntoToolMessage) { + if (options?.mergeToolResultText && hasToolMessages) { + // When mergeToolResultText is enabled, separate text and images + // Merge text into the last tool message, and send images separately + // This is critical for providers like NVIDIA NIM that don't allow user messages after tool messages + const textMessages = nonToolMessages.filter( + (part) => part.type === "text", + ) as Anthropic.TextBlockParam[] + const imageMessages = nonToolMessages.filter( + (part) => part.type === "image", + ) as Anthropic.ImageBlockParam[] + // Merge text content into the last tool message - const lastToolMessage = openAiMessages[ - openAiMessages.length - 1 - ] as OpenAI.Chat.ChatCompletionToolMessageParam - if (lastToolMessage?.role === "tool") { - const additionalText = nonToolMessages - .map((part) => (part as Anthropic.TextBlockParam).text) - .join("\n") - lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}` + if (textMessages.length > 0) { + const lastToolMessage = openAiMessages[ + openAiMessages.length - 1 + ] as OpenAI.Chat.ChatCompletionToolMessageParam + if (lastToolMessage?.role === "tool") { + const additionalText = textMessages.map((part) => part.text).join("\n") + lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}` + } + } + + // Send images as a separate user message if any + // Note: Images must still be sent as user messages since tool messages don't support images + if (imageMessages.length > 0) { + openAiMessages.push({ + role: "user", + content: imageMessages.map((part) => ({ + type: "image_url", + image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` }, + })), + }) } } else { // Standard behavior: add user message with text/image content