fix: merge text content into tool message even when images are present

When mergeToolResultText is enabled and both text and images are present
after tool results, the text content is now merged into the last tool
message while images are sent as a separate user message.

This fixes the issue where strict tool message ordering would not work
when images were added to the chat, causing "Unexpected role user after
role tool" errors on providers like NVIDIA NIM.

src/api/transform/__tests__/openai-format.spec.ts

@@ -319,7 +319,55 @@ describe("convertToOpenAiMessages", () => {
 			)
 		})
 
-		it("should NOT merge text when images are present (fall back to user message)", () => {
+		it("should merge text and send images separately when mergeToolResultText is true", () => {
+			const anthropicMessages: Anthropic.Messages.MessageParam[] = [
+				{
+					role: "user",
+					content: [
+						{
+							type: "tool_result",
+							tool_use_id: "tool-123",
+							content: "Tool result content",
+						},
+						{
+							type: "text",
+							text: "<environment_details>Context info</environment_details>",
+						},
+						{
+							type: "image",
+							source: {
+								type: "base64",
+								media_type: "image/png",
+								data: "base64data",
+							},
+						},
+					],
+				},
+			]
+
+			const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true })
+
+			// Should produce a tool message with merged text AND a user message for the image
+			expect(openAiMessages).toHaveLength(2)
+
+			// First message: tool message with merged text
+			const toolMessage = openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam
+			expect(toolMessage.role).toBe("tool")
+			expect(toolMessage.tool_call_id).toBe("tool-123")
+			expect(toolMessage.content).toBe(
+				"Tool result content\n\n<environment_details>Context info</environment_details>",
+			)
+
+			// Second message: user message with only the image
+			expect(openAiMessages[1].role).toBe("user")
+			const userContent = openAiMessages[1].content as Array<{ type: string; image_url?: { url: string } }>
+			expect(Array.isArray(userContent)).toBe(true)
+			expect(userContent).toHaveLength(1)
+			expect(userContent[0].type).toBe("image_url")
+			expect(userContent[0].image_url?.url).toBe("data:image/png;base64,base64data")
+		})
+
+		it("should send only images as user message when no text content exists with mergeToolResultText", () => {
 			const anthropicMessages: Anthropic.Messages.MessageParam[] = [
 				{
 					role: "user",
@@ -343,9 +391,13 @@ describe("convertToOpenAiMessages", () => {
 
 			const openAiMessages = convertToOpenAiMessages(anthropicMessages, { mergeToolResultText: true })
 
-			// Should produce a tool message AND a user message (because image is present)
+			// Should produce a tool message AND a user message (only image, no text to merge)
 			expect(openAiMessages).toHaveLength(2)
 			expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).role).toBe("tool")
+			// Tool message content should NOT be modified since there's no text to merge
+			expect((openAiMessages[0] as OpenAI.Chat.ChatCompletionToolMessageParam).content).toBe(
+				"Tool result content",
+			)
 			expect(openAiMessages[1].role).toBe("user")
 		})
 

src/api/transform/openai-format.ts

@@ -138,24 +138,40 @@ export function convertToOpenAiMessages(
 
 				// Process non-tool messages
 				if (nonToolMessages.length > 0) {
-					// Check if we should merge text into the last tool message
-					// This is critical for reasoning/thinking models where a user message
-					// after tool results causes the model to drop all previous reasoning_content
-					const hasOnlyTextContent = nonToolMessages.every((part) => part.type === "text")
 					const hasToolMessages = toolMessages.length > 0
-					const shouldMergeIntoToolMessage =
-						options?.mergeToolResultText && hasToolMessages && hasOnlyTextContent
 
-					if (shouldMergeIntoToolMessage) {
+					if (options?.mergeToolResultText && hasToolMessages) {
+						// When mergeToolResultText is enabled, separate text and images
+						// Merge text into the last tool message, and send images separately
+						// This is critical for providers like NVIDIA NIM that don't allow user messages after tool messages
+						const textMessages = nonToolMessages.filter(
+							(part) => part.type === "text",
+						) as Anthropic.TextBlockParam[]
+						const imageMessages = nonToolMessages.filter(
+							(part) => part.type === "image",
+						) as Anthropic.ImageBlockParam[]
+
 						// Merge text content into the last tool message
-						const lastToolMessage = openAiMessages[
-							openAiMessages.length - 1
-						] as OpenAI.Chat.ChatCompletionToolMessageParam
-						if (lastToolMessage?.role === "tool") {
-							const additionalText = nonToolMessages
-								.map((part) => (part as Anthropic.TextBlockParam).text)
-								.join("\n")
-							lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}`
+						if (textMessages.length > 0) {
+							const lastToolMessage = openAiMessages[
+								openAiMessages.length - 1
+							] as OpenAI.Chat.ChatCompletionToolMessageParam
+							if (lastToolMessage?.role === "tool") {
+								const additionalText = textMessages.map((part) => part.text).join("\n")
+								lastToolMessage.content = `${lastToolMessage.content}\n\n${additionalText}`
+							}
+						}
+
+						// Send images as a separate user message if any
+						// Note: Images must still be sent as user messages since tool messages don't support images
+						if (imageMessages.length > 0) {
+							openAiMessages.push({
+								role: "user",
+								content: imageMessages.map((part) => ({
+									type: "image_url",
+									image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` },
+								})),
+							})
 						}
 					} else {
 						// Standard behavior: add user message with text/image content