diff --git a/src/api/transform/__tests__/openai-format.spec.ts b/src/api/transform/__tests__/openai-format.spec.ts
index d5d4404837..111f79101f 100644
--- a/src/api/transform/__tests__/openai-format.spec.ts
+++ b/src/api/transform/__tests__/openai-format.spec.ts
@@ -319,7 +319,55 @@ describe("convertToOpenAiMessages", () => {
)
})
- it("should NOT merge text when images are present (fall back to user message)", () => {
+ it("should merge text and send images separately when mergeToolResultText is true", () => {
+ const anthropicMessages: Anthropic.Messages.MessageParam[] = [
+ {
+ role: "user",
+ content: [
+ {
+ type: "tool_result",
+ tool_use_id: "tool-123",
+ content: "Tool result content",
+ },
+ {
+ type: "text",
+ text: "Context info",
+ },
+ {
+ type: "image",
+ source: {
+ type: "base64",
+ media_type: "image/png",
+ data: "base64data",
+ },
+ },
+ ],
+ },
+ ]
+
+ const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true })
+
+ // Should produce a tool message with merged text AND a user message for the image
+ expect(openAiMessages).toHaveLength(2)
+
+ // First message: tool message with merged text
+ const toolMessage = openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam
+ expect(toolMessage.role).toBe("tool")
+ expect(toolMessage.tool_call_id).toBe("tool-123")
+ expect(toolMessage.content).toBe(
+ "Tool result content\n\nContext info",
+ )
+
+ // Second message: user message with only the image
+ expect(openAiMessages[1].role).toBe("user")
+ const userContent = openAiMessages[1].content as Array<{ type: string; image_url?: { url: string } }>
+ expect(Array.isArray(userContent)).toBe(true)
+ expect(userContent).toHaveLength(1)
+ expect(userContent[0].type).toBe("image_url")
+ expect(userContent[0].image_url?.url).toBe("data:image/png;base64,base64data")
+ })
+
+ it("should send only images as user message when no text content exists with mergeToolResultText", () => {
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
{
role: "user",
@@ -343,9 +391,13 @@ describe("convertToOpenAiMessages", () => {
const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true })
- // Should produce a tool message AND a user message (because image is present)
+ // Should produce a tool message AND a user message (only image, no text to merge)
expect(openAiMessages).toHaveLength(2)
expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).role).toBe("tool")
+ // Tool message content should NOT be modified since there's no text to merge
+ expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).content).toBe(
+ "Tool result content",
+ )
expect(openAiMessages[1].role).toBe("user")
})
diff --git a/src/api/transform/openai-format.ts b/src/api/transform/openai-format.ts
index ad02be5541..f182a2aac1 100644
--- a/src/api/transform/openai-format.ts
+++ b/src/api/transform/openai-format.ts
@@ -138,24 +138,40 @@ export function convertToOpenAiMessages(
// Process non-tool messages
if (nonToolMessages.length > 0) {
- // Check if we should merge text into the last tool message
- // This is critical for reasoning/thinking models where a user message
- // after tool results causes the model to drop all previous reasoning_content
- const hasOnlyTextContent = nonToolMessages.every((part) => part.type === "text")
const hasToolMessages = toolMessages.length > 0
- const shouldMergeIntoToolMessage =
- options?.mergeToolResultText && hasToolMessages && hasOnlyTextContent
- if (shouldMergeIntoToolMessage) {
+ if (options?.mergeToolResultText && hasToolMessages) {
+ // When mergeToolResultText is enabled, separate text and images
+ // Merge text into the last tool message, and send images separately
+ // This is critical for providers like NVIDIA NIM that don't allow user messages after tool messages
+ const textMessages = nonToolMessages.filter(
+ (part) => part.type === "text",
+ ) as Anthropic.TextBlockParam[]
+ const imageMessages = nonToolMessages.filter(
+ (part) => part.type === "image",
+ ) as Anthropic.ImageBlockParam[]
+
// Merge text content into the last tool message
- const lastToolMessage = openAiMessages[
- openAiMessages.length - 1
- ] as OpenAI.Chat.ChatCompletionToolMessageParam
- if (lastToolMessage?.role === "tool") {
- const additionalText = nonToolMessages
- .map((part) => (part as Anthropic.TextBlockParam).text)
- .join("\n")
- lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}`
+ if (textMessages.length > 0) {
+ const lastToolMessage = openAiMessages[
+ openAiMessages.length - 1
+ ] as OpenAI.Chat.ChatCompletionToolMessageParam
+ if (lastToolMessage?.role === "tool") {
+ const additionalText = textMessages.map((part) => part.text).join("\n")
+ lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}`
+ }
+ }
+
+ // Send images as a separate user message if any
+ // Note: Images must still be sent as user messages since tool messages don't support images
+ if (imageMessages.length > 0) {
+ openAiMessages.push({
+ role: "user",
+ content: imageMessages.map((part) => ({
+ type: "image_url",
+ image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` },
+ })),
+ })
}
} else {
// Standard behavior: add user message with text/image content