feat: add support for image file @mentions (#10189)

Co-authored-by: Roo Code <roomote@roocode.com>
This commit is contained in:
Hannes Rudolph 2026-01-07 09:44:04 -07:00 committed by GitHub
parent e287a82144
commit 43f7ce025f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 561 additions and 10 deletions

View file

@ -0,0 +1,193 @@
import * as path from "path"
import { resolveImageMentions } from "../resolveImageMentions"
vi.mock("../../tools/helpers/imageHelpers", () => ({
isSupportedImageFormat: vi.fn((ext: string) =>
[".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".ico", ".tiff", ".tif", ".avif"].includes(
ext.toLowerCase(),
),
),
readImageAsDataUrlWithBuffer: vi.fn(),
validateImageForProcessing: vi.fn(),
ImageMemoryTracker: vi.fn().mockImplementation(() => ({
getTotalMemoryUsed: vi.fn().mockReturnValue(0),
addMemoryUsage: vi.fn(),
})),
DEFAULT_MAX_IMAGE_FILE_SIZE_MB: 5,
DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB: 20,
}))
import { validateImageForProcessing, readImageAsDataUrlWithBuffer } from "../../tools/helpers/imageHelpers"
const mockReadImageAsDataUrl = vi.mocked(readImageAsDataUrlWithBuffer)
const mockValidateImage = vi.mocked(validateImageForProcessing)
describe("resolveImageMentions", () => {
beforeEach(() => {
vi.clearAllMocks()
// Default: validation passes
mockValidateImage.mockResolvedValue({ isValid: true, sizeInMB: 0.1 })
})
it("should append a data URL when a local png mention is present", async () => {
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
const result = await resolveImageMentions({
text: "Please look at @/assets/cat.png",
images: [],
cwd: "/workspace",
})
expect(mockValidateImage).toHaveBeenCalled()
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"))
expect(result.text).toBe("Please look at @/assets/cat.png")
expect(result.images).toEqual([dataUrl])
})
it("should support gif images (matching read_file)", async () => {
const dataUrl = `data:image/gif;base64,${Buffer.from("gif-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("gif-bytes") })
const result = await resolveImageMentions({
text: "See @/animation.gif",
images: [],
cwd: "/workspace",
})
expect(result.images).toEqual([dataUrl])
})
it("should support svg images (matching read_file)", async () => {
const dataUrl = `data:image/svg+xml;base64,${Buffer.from("svg-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("svg-bytes") })
const result = await resolveImageMentions({
text: "See @/icon.svg",
images: [],
cwd: "/workspace",
})
expect(result.images).toEqual([dataUrl])
})
it("should ignore non-image mentions", async () => {
const result = await resolveImageMentions({
text: "See @/src/index.ts",
images: [],
cwd: "/workspace",
})
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
expect(result.images).toEqual([])
})
it("should skip unreadable files (fail-soft)", async () => {
mockReadImageAsDataUrl.mockRejectedValue(new Error("ENOENT"))
const result = await resolveImageMentions({
text: "See @/missing.webp",
images: [],
cwd: "/workspace",
})
expect(result.images).toEqual([])
})
it("should respect rooIgnoreController", async () => {
const dataUrl = `data:image/jpeg;base64,${Buffer.from("jpg-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("jpg-bytes") })
const rooIgnoreController = {
validateAccess: vi.fn().mockReturnValue(false),
}
const result = await resolveImageMentions({
text: "See @/secret.jpg",
images: [],
cwd: "/workspace",
rooIgnoreController,
})
expect(rooIgnoreController.validateAccess).toHaveBeenCalledWith("secret.jpg")
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
expect(result.images).toEqual([])
})
it("should dedupe when mention repeats", async () => {
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
const result = await resolveImageMentions({
text: "@/a.png and again @/a.png",
images: [],
cwd: "/workspace",
})
expect(result.images).toHaveLength(1)
})
it("should skip images when supportsImages is false", async () => {
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
const result = await resolveImageMentions({
text: "See @/cat.png",
images: [],
cwd: "/workspace",
supportsImages: false,
})
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
expect(result.images).toEqual([])
})
it("should skip images that exceed size limits", async () => {
mockValidateImage.mockResolvedValue({
isValid: false,
reason: "size_limit",
notice: "Image too large",
})
const result = await resolveImageMentions({
text: "See @/huge.png",
images: [],
cwd: "/workspace",
})
expect(mockValidateImage).toHaveBeenCalled()
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
expect(result.images).toEqual([])
})
it("should skip images that would exceed memory limit", async () => {
mockValidateImage.mockResolvedValue({
isValid: false,
reason: "memory_limit",
notice: "Would exceed memory limit",
})
const result = await resolveImageMentions({
text: "See @/large.png",
images: [],
cwd: "/workspace",
})
expect(result.images).toEqual([])
})
it("should pass custom size limits to validation", async () => {
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
await resolveImageMentions({
text: "See @/cat.png",
images: [],
cwd: "/workspace",
maxImageFileSize: 10,
maxTotalImageSize: 50,
})
expect(mockValidateImage).toHaveBeenCalledWith(expect.any(String), true, 10, 50, 0)
})
})

View file

@ -284,7 +284,13 @@ async function getFileOrFolderContent(
const stats = await fs.stat(absPath)
if (stats.isFile()) {
if (rooIgnoreController && !rooIgnoreController.validateAccess(absPath)) {
// Avoid trying to include image binary content as text context.
// Image mentions are handled separately via image attachment flow.
const isBinary = await isBinaryFile(absPath).catch(() => false)
if (isBinary) {
return `(Binary file ${mentionPath} omitted)`
}
if (rooIgnoreController && !rooIgnoreController.validateAccess(unescapedPath)) {
return `(File ${mentionPath} is ignored by .rooignore)`
}
try {

View file

@ -0,0 +1,145 @@
import * as path from "path"
import { mentionRegexGlobal, unescapeSpaces } from "../../shared/context-mentions"
import {
isSupportedImageFormat,
readImageAsDataUrlWithBuffer,
validateImageForProcessing,
ImageMemoryTracker,
DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
} from "../tools/helpers/imageHelpers"
const MAX_IMAGES_PER_MESSAGE = 20
export interface ResolveImageMentionsOptions {
text: string
images?: string[]
cwd: string
rooIgnoreController?: { validateAccess: (filePath: string) => boolean }
/** Whether the current model supports images. Defaults to true. */
supportsImages?: boolean
/** Maximum size per image file in MB. Defaults to 5MB. */
maxImageFileSize?: number
/** Maximum total size of all images in MB. Defaults to 20MB. */
maxTotalImageSize?: number
}
export interface ResolveImageMentionsResult {
text: string
images: string[]
}
function isPathWithinCwd(absPath: string, cwd: string): boolean {
const rel = path.relative(cwd, absPath)
return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)
}
function dedupePreserveOrder(values: string[]): string[] {
const seen = new Set<string>()
const result: string[] = []
for (const v of values) {
if (seen.has(v)) continue
seen.add(v)
result.push(v)
}
return result
}
/**
* Resolves local image file mentions like `@/path/to/image.png` found in `text` into `data:image/...;base64,...`
* and appends them to the outgoing `images` array.
*
* Behavior matches the read_file tool:
* - Supports the same image formats: png, jpg, jpeg, gif, webp, svg, bmp, ico, tiff, avif
* - Respects per-file size limits (default 5MB)
* - Respects total memory limits (default 20MB)
* - Skips images if model doesn't support them
* - Respects `.rooignore` via `rooIgnoreController.validateAccess` when provided
*/
export async function resolveImageMentions({
text,
images,
cwd,
rooIgnoreController,
supportsImages = true,
maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
}: ResolveImageMentionsOptions): Promise<ResolveImageMentionsResult> {
const existingImages = Array.isArray(images) ? images : []
if (existingImages.length >= MAX_IMAGES_PER_MESSAGE) {
return { text, images: existingImages.slice(0, MAX_IMAGES_PER_MESSAGE) }
}
// If model doesn't support images, skip image processing entirely
if (!supportsImages) {
return { text, images: existingImages }
}
const mentions = Array.from(text.matchAll(mentionRegexGlobal))
.map((m) => m[1])
.filter(Boolean)
if (mentions.length === 0) {
return { text, images: existingImages }
}
const imageMentions = mentions.filter((mention) => {
if (!mention.startsWith("/")) return false
const relPath = unescapeSpaces(mention.slice(1))
const ext = path.extname(relPath).toLowerCase()
return isSupportedImageFormat(ext)
})
if (imageMentions.length === 0) {
return { text, images: existingImages }
}
const imageMemoryTracker = new ImageMemoryTracker()
const newImages: string[] = []
for (const mention of imageMentions) {
if (existingImages.length + newImages.length >= MAX_IMAGES_PER_MESSAGE) {
break
}
const relPath = unescapeSpaces(mention.slice(1))
const absPath = path.resolve(cwd, relPath)
if (!isPathWithinCwd(absPath, cwd)) {
continue
}
if (rooIgnoreController && !rooIgnoreController.validateAccess(relPath)) {
continue
}
// Validate image size limits (matches read_file behavior)
try {
const validationResult = await validateImageForProcessing(
absPath,
supportsImages,
maxImageFileSize,
maxTotalImageSize,
imageMemoryTracker.getTotalMemoryUsed(),
)
if (!validationResult.isValid) {
// Skip this image due to size/memory limits, but continue processing others
continue
}
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath)
newImages.push(dataUrl)
// Track memory usage
if (validationResult.sizeInMB) {
imageMemoryTracker.addMemoryUsage(validationResult.sizeInMB)
}
} catch {
// Fail-soft: skip unreadable/missing files.
continue
}
}
const merged = dedupePreserveOrder([...existingImages, ...newImages]).slice(0, MAX_IMAGES_PER_MESSAGE)
return { text, images: merged }
}

View file

@ -3051,7 +3051,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
expect(mockCline.overwriteClineMessages).toHaveBeenCalledWith([mockMessages[0]])
expect(mockCline.overwriteApiConversationHistory).toHaveBeenCalledWith([{ ts: 1000 }])
// Verify submitUserMessage was called with the edited content
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with preserved images", undefined)
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with preserved images", [])
})
test("handles editing messages with file attachments", async () => {
@ -3104,7 +3104,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
})
expect(mockCline.overwriteClineMessages).toHaveBeenCalled()
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with file attachment", undefined)
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with file attachment", [])
})
})
@ -3635,7 +3635,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
await messageHandler({ type: "editMessageConfirm", messageTs: 2000, text: largeEditedContent })
expect(mockCline.overwriteClineMessages).toHaveBeenCalled()
expect(mockCline.submitUserMessage).toHaveBeenCalledWith(largeEditedContent, undefined)
expect(mockCline.submitUserMessage).toHaveBeenCalledWith(largeEditedContent, [])
})
test("handles deleting messages with large payloads", async () => {

View file

@ -55,6 +55,10 @@ describe("webviewMessageHandler - checkpoint operations", () => {
contextProxy: {
globalStorageUri: { fsPath: "/test/storage" },
},
getState: vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
}),
}
})
@ -124,7 +128,7 @@ describe("webviewMessageHandler - checkpoint operations", () => {
operation: "edit",
editData: {
editedContent: "Edited checkpoint message",
images: undefined,
images: [],
apiConversationHistoryIndex: 0,
},
})

View file

@ -71,6 +71,10 @@ describe("webviewMessageHandler - Edit Message with Timestamp Fallback", () => {
globalStorageUri: { fsPath: "/mock/storage" },
},
log: vi.fn(),
getState: vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
}),
} as unknown as ClineProvider
})

View file

@ -0,0 +1,130 @@
import * as fs from "fs/promises"
import * as path from "path"
import * as os from "os"
// Must mock dependencies before importing the handler module.
vi.mock("../../../api/providers/fetchers/modelCache")
import { webviewMessageHandler } from "../webviewMessageHandler"
import type { ClineProvider } from "../ClineProvider"
vi.mock("vscode", () => ({
window: {
showInformationMessage: vi.fn(),
showErrorMessage: vi.fn(),
},
workspace: {
workspaceFolders: [{ uri: { fsPath: "/mock/workspace" } }],
},
}))
// Mock imageHelpers - use actual implementations for functions that need real file access
vi.mock("../../tools/helpers/imageHelpers", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../tools/helpers/imageHelpers")>()
return {
...actual,
validateImageForProcessing: vi.fn().mockResolvedValue({ isValid: true, sizeInMB: 0.001 }),
ImageMemoryTracker: vi.fn().mockImplementation(() => ({
getTotalMemoryUsed: vi.fn().mockReturnValue(0),
addMemoryUsage: vi.fn(),
})),
}
})
describe("webviewMessageHandler - image mentions (integration)", () => {
it("resolves image mentions for newTask and passes images to createTask", async () => {
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "roo-image-mentions-"))
try {
const imgBytes = Buffer.from("png-bytes")
await fs.writeFile(path.join(tmpRoot, "cat.png"), imgBytes)
const mockProvider = {
cwd: tmpRoot,
getCurrentTask: vi.fn().mockReturnValue(undefined),
createTask: vi.fn().mockResolvedValue(undefined),
postMessageToWebview: vi.fn().mockResolvedValue(undefined),
getState: vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
}),
} as unknown as ClineProvider
await webviewMessageHandler(mockProvider, {
type: "newTask",
text: "Please look at @/cat.png",
images: [],
} as any)
expect(mockProvider.createTask).toHaveBeenCalledWith("Please look at @/cat.png", [
`data:image/png;base64,${imgBytes.toString("base64")}`,
])
} finally {
await fs.rm(tmpRoot, { recursive: true, force: true })
}
})
it("resolves image mentions for askResponse and passes images to handleWebviewAskResponse", async () => {
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "roo-image-mentions-"))
try {
const imgBytes = Buffer.from("jpg-bytes")
await fs.writeFile(path.join(tmpRoot, "cat.jpg"), imgBytes)
const handleWebviewAskResponse = vi.fn()
const mockProvider = {
cwd: tmpRoot,
getCurrentTask: vi.fn().mockReturnValue({
cwd: tmpRoot,
handleWebviewAskResponse,
}),
getState: vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
}),
} as unknown as ClineProvider
await webviewMessageHandler(mockProvider, {
type: "askResponse",
askResponse: "messageResponse",
text: "Please look at @/cat.jpg",
images: [],
} as any)
expect(handleWebviewAskResponse).toHaveBeenCalledWith("messageResponse", "Please look at @/cat.jpg", [
`data:image/jpeg;base64,${imgBytes.toString("base64")}`,
])
} finally {
await fs.rm(tmpRoot, { recursive: true, force: true })
}
})
it("resolves gif image mentions (matching read_file behavior)", async () => {
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "roo-image-mentions-"))
try {
const imgBytes = Buffer.from("gif-bytes")
await fs.writeFile(path.join(tmpRoot, "animation.gif"), imgBytes)
const mockProvider = {
cwd: tmpRoot,
getCurrentTask: vi.fn().mockReturnValue(undefined),
createTask: vi.fn().mockResolvedValue(undefined),
postMessageToWebview: vi.fn().mockResolvedValue(undefined),
getState: vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
}),
} as unknown as ClineProvider
await webviewMessageHandler(mockProvider, {
type: "newTask",
text: "See @/animation.gif",
images: [],
} as any)
expect(mockProvider.createTask).toHaveBeenCalledWith("See @/animation.gif", [
`data:image/gif;base64,${imgBytes.toString("base64")}`,
])
} finally {
await fs.rm(tmpRoot, { recursive: true, force: true })
}
})
})

View file

@ -117,6 +117,15 @@ vi.mock("../../../utils/fs")
vi.mock("../../../utils/path")
vi.mock("../../../utils/globalContext")
vi.mock("../../mentions/resolveImageMentions", () => ({
resolveImageMentions: vi.fn(async ({ text, images }: { text: string; images?: string[] }) => ({
text,
images: [...(images ?? []), "data:image/png;base64,from-mention"],
})),
}))
import { resolveImageMentions } from "../../mentions/resolveImageMentions"
describe("webviewMessageHandler - requestLmStudioModels", () => {
beforeEach(() => {
vi.clearAllMocks()
@ -159,6 +168,37 @@ describe("webviewMessageHandler - requestLmStudioModels", () => {
})
})
describe("webviewMessageHandler - image mentions", () => {
beforeEach(() => {
vi.clearAllMocks()
mockClineProvider.getState = vi.fn().mockResolvedValue({
maxImageFileSize: 5,
maxTotalImageSize: 20,
})
})
it("should resolve image mentions for askResponse payloads", async () => {
const mockHandleWebviewAskResponse = vi.fn()
vi.mocked(mockClineProvider.getCurrentTask).mockReturnValue({
cwd: "/mock/workspace",
rooIgnoreController: undefined,
handleWebviewAskResponse: mockHandleWebviewAskResponse,
} as any)
await webviewMessageHandler(mockClineProvider, {
type: "askResponse",
askResponse: "messageResponse",
text: "See @/img.png",
images: [],
})
expect(vi.mocked(resolveImageMentions)).toHaveBeenCalled()
expect(mockHandleWebviewAskResponse).toHaveBeenCalledWith("messageResponse", "See @/img.png", [
"data:image/png;base64,from-mention",
])
})
})
describe("webviewMessageHandler - requestOllamaModels", () => {
beforeEach(() => {
vi.clearAllMocks()

View file

@ -54,6 +54,7 @@ import { exportSettings, importSettingsWithFeedback } from "../config/importExpo
import { getOpenAiModels } from "../../api/providers/openai"
import { getVsCodeLmModels } from "../../api/providers/vscode-lm"
import { openMention } from "../mentions"
import { resolveImageMentions } from "../mentions/resolveImageMentions"
import { getWorkspacePath } from "../../utils/path"
import { Mode, defaultModeSlug } from "../../shared/modes"
import { getModels, flushModels } from "../../api/providers/fetchers/modelCache"
@ -79,6 +80,26 @@ export const webviewMessageHandler = async (
const getCurrentCwd = () => {
return provider.getCurrentTask()?.cwd || provider.cwd
}
/**
* Resolves image file mentions in incoming messages.
* Matches read_file behavior: respects size limits and model capabilities.
*/
const resolveIncomingImages = async (payload: { text?: string; images?: string[] }) => {
const text = payload.text ?? ""
const images = payload.images
const currentTask = provider.getCurrentTask()
const state = await provider.getState()
const resolved = await resolveImageMentions({
text,
images,
cwd: getCurrentCwd(),
rooIgnoreController: currentTask?.rooIgnoreController,
maxImageFileSize: state.maxImageFileSize,
maxTotalImageSize: state.maxTotalImageSize,
})
return resolved
}
/**
* Shared utility to find message indices based on timestamp.
* When multiple messages share the same timestamp (e.g., after condense),
@ -505,7 +526,8 @@ export const webviewMessageHandler = async (
// agentically running promises in old instance don't affect our new
// task. This essentially creates a fresh slate for the new task.
try {
await provider.createTask(message.text, message.images)
const resolved = await resolveIncomingImages({ text: message.text, images: message.images })
await provider.createTask(resolved.text, resolved.images)
// Task created successfully - notify the UI to reset
await provider.postMessageToWebview({ type: "invoke", invoke: "newChat" })
} catch (error) {
@ -522,7 +544,12 @@ export const webviewMessageHandler = async (
break
case "askResponse":
provider.getCurrentTask()?.handleWebviewAskResponse(message.askResponse!, message.text, message.images)
{
const resolved = await resolveIncomingImages({ text: message.text, images: message.images })
provider
.getCurrentTask()
?.handleWebviewAskResponse(message.askResponse!, resolved.text, resolved.images)
}
break
case "updateSettings":
@ -1877,11 +1904,12 @@ export const webviewMessageHandler = async (
break
case "editMessageConfirm":
if (message.messageTs && message.text) {
const resolved = await resolveIncomingImages({ text: message.text, images: message.images })
await handleEditMessageConfirm(
message.messageTs,
message.text,
resolved.text,
message.restoreCheckpoint,
message.images,
resolved.images,
)
}
break
@ -3052,7 +3080,8 @@ export const webviewMessageHandler = async (
*/
case "queueMessage": {
provider.getCurrentTask()?.messageQueueService.addMessage(message.text ?? "", message.images)
const resolved = await resolveIncomingImages({ text: message.text, images: message.images })
provider.getCurrentTask()?.messageQueueService.addMessage(resolved.text, resolved.images)
break
}
case "removeQueuedMessage": {