feat(transform, chat, gemini, media): Gemini enable video processing

This commit introduces support for video content by updating the Gemini transformer and generalizing media handling in the UI.

Key changes include:
- Extending gemini-format.ts to process video content blocks.
- Adding tests to gemini-format.spec.ts to validate video and mixed media handling.
- Refactoring the chat UI to use a generic MediaThumbnails component.
- Introducing a getMimeType utility for identifying media types from data URIs.

Closes: #6144

src/api/transform/__tests__/gemini-format.spec.ts

@@ -71,13 +71,77 @@ describe("convertAnthropicMessageToGemini", () => {
 		expect(result).toEqual({
 			role: "user",
 			parts: [
-				{ text: "Check out this image:" },
 				{
 					inlineData: {
 						data: "base64encodeddata",
 						mimeType: "image/jpeg",
 					},
 				},
+				{ text: "Check out this image:" },
+			],
+		})
+	})
+
+	it("should convert a message with a video", () => {
+		const anthropicMessage: Anthropic.Messages.MessageParam = {
+			role: "user",
+			content: [
+				{ type: "text", text: "Check out this video:" },
+				{
+					type: "video",
+					source: {
+						type: "base64",
+						media_type: "video/mp4",
+						data: "base64encodedvideodata",
+					},
+				} as any,
+			],
+		}
+
+		const result = convertAnthropicMessageToGemini(anthropicMessage)
+
+		expect(result).toEqual({
+			role: "user",
+			parts: [
+				{
+					inlineData: {
+						data: "base64encodedvideodata",
+						mimeType: "video/mp4",
+					},
+				},
+				{ text: "Check out this video:" },
+			],
+		})
+	})
+
+	it("should handle text after inlineData", () => {
+		const anthropicMessage: Anthropic.Messages.MessageParam = {
+			role: "user",
+			content: [
+				{
+					type: "image",
+					source: {
+						type: "base64",
+						media_type: "image/jpeg",
+						data: "base64encodeddata",
+					},
+				},
+				{ type: "text", text: "Check out this image:" },
+			],
+		}
+
+		const result = convertAnthropicMessageToGemini(anthropicMessage)
+
+		expect(result).toEqual({
+			role: "user",
+			parts: [
+				{
+					inlineData: {
+						data: "base64encodeddata",
+						mimeType: "image/jpeg",
+					},
+				},
+				{ text: "Check out this image:" },
 			],
 		})
 	})

src/api/transform/gemini-format.ts

@@ -1,12 +1,24 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import { Content, Part } from "@google/genai"
 
-export function convertAnthropicContentToGemini(content: string | Anthropic.ContentBlockParam[]): Part[] {
+// Extended type to support video content blocks that aren't in the standard Anthropic SDK
+interface VideoContentBlock {
+	type: "video"
+	source: {
+		type: "base64"
+		data: string
+		media_type: string
+	}
+}
+
+type ExtendedContentBlockParam = Anthropic.ContentBlockParam | VideoContentBlock
+
+export function convertAnthropicContentToGemini(content: string | ExtendedContentBlockParam[]): Part[] {
 	if (typeof content === "string") {
 		return [{ text: content }]
 	}
 
-	return content.flatMap((block): Part | Part[] => {
+	const parts = content.flatMap((block): Part | Part[] => {
 		switch (block.type) {
 			case "text":
 				return { text: block.text }
@@ -15,6 +27,11 @@ export function convertAnthropicContentToGemini(content: string | Anthropic.Cont
 					throw new Error("Unsupported image source type")
 				}
 
+				return { inlineData: { data: block.source.data, mimeType: block.source.media_type } }
+			case "video":
+				if (block.source.type !== "base64") {
+					throw new Error("Unsupported video source type")
+				}
 				return { inlineData: { data: block.source.data, mimeType: block.source.media_type } }
 			case "tool_use":
 				return {
@@ -68,6 +85,17 @@ export function convertAnthropicContentToGemini(content: string | Anthropic.Cont
 				throw new Error(`Unsupported content block type: ${block.type}`)
 		}
 	})
+
+	// Sort parts to ensure inlineData comes before text
+	return parts.sort((a, b) => {
+		if ("inlineData" in a && "text" in b) {
+			return -1
+		}
+		if ("text" in a && "inlineData" in b) {
+			return 1
+		}
+		return 0
+	})
 }
 
 export function convertAnthropicMessageToGemini(message: Anthropic.Messages.MessageParam): Content {

webview-ui/src/components/chat/ChatRow.tsx

@@ -1069,8 +1069,8 @@ export const ChatRowContent = ({
 										sendingDisabled={false}
 										selectApiConfigDisabled={true}
 										placeholderText={t("chat:editMessage.placeholder")}
-										selectedImages={editImages}
-										setSelectedImages={setEditImages}
+										selectedMedia={editImages}
+										setSelectedMedia={setEditImages}
 										onSend={handleSaveEdit}
 										onSelectImages={handleSelectImages}
 										shouldDisableImages={false}
@@ -1079,6 +1079,7 @@ export const ChatRowContent = ({
 										modeShortcutText=""
 										isEditMode={true}
 										onCancel={handleCancelEdit}
+										acceptedFileTypes={[]}
 									/>
 								</div>
 							) : (

webview-ui/src/components/chat/ChatTextArea.tsx

@@ -21,9 +21,9 @@ import {
 import { convertToMentionPath } from "@/utils/path-mentions"
 import { SelectDropdown, DropdownOptionType, Button, StandardTooltip } from "@/components/ui"
 
-import Thumbnails from "../common/Thumbnails"
 import ModeSelector from "./ModeSelector"
 import { MAX_IMAGES_PER_MESSAGE } from "./ChatView"
+import MediaThumbnails from "../common/MediaThumbnails"
 import ContextMenu from "./ContextMenu"
 import { VolumeX, Pin, Check, Image, WandSparkles, SendHorizontal } from "lucide-react"
 import { IndexingStatusBadge } from "./IndexingStatusBadge"
@@ -37,8 +37,8 @@ interface ChatTextAreaProps {
 	sendingDisabled: boolean
 	selectApiConfigDisabled: boolean
 	placeholderText: string
-	selectedImages: string[]
-	setSelectedImages: React.Dispatch<React.SetStateAction<string[]>>
+	selectedMedia: string[]
+	setSelectedMedia: React.Dispatch<React.SetStateAction<string[]>>
 	onSend: () => void
 	onSelectImages: () => void
 	shouldDisableImages: boolean
@@ -46,6 +46,7 @@ interface ChatTextAreaProps {
 	mode: Mode
 	setMode: (value: Mode) => void
 	modeShortcutText: string
+	acceptedFileTypes: string[]
 	// Edit mode props
 	isEditMode?: boolean
 	onCancel?: () => void
@@ -59,8 +60,8 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 			sendingDisabled,
 			selectApiConfigDisabled,
 			placeholderText,
-			selectedImages,
-			setSelectedImages,
+			selectedMedia,
+			setSelectedMedia,
 			onSend,
 			onSelectImages,
 			shouldDisableImages,
@@ -70,6 +71,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 			modeShortcutText,
 			isEditMode = false,
 			onCancel,
+			acceptedFileTypes,
 		},
 		ref,
 	) => {
@@ -598,17 +600,15 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 					return
 				}
 
-				const acceptedTypes = ["png", "jpeg", "webp"]
-
-				const imageItems = Array.from(items).filter((item) => {
+				const mediaItems = Array.from(items).filter((item) => {
 					const [type, subtype] = item.type.split("/")
-					return type === "image" && acceptedTypes.includes(subtype)
+					return (type === "image" || type === "video") && acceptedFileTypes.includes(subtype)
 				})
 
-				if (!shouldDisableImages && imageItems.length > 0) {
+				if (!shouldDisableImages && mediaItems.length > 0) {
 					e.preventDefault()
 
-					const imagePromises = imageItems.map((item) => {
+					const mediaPromises = mediaItems.map((item) => {
 						return new Promise<string | null>((resolve) => {
 							const blob = item.getAsFile()
 
@@ -633,17 +633,17 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 						})
 					})
 
-					const imageDataArray = await Promise.all(imagePromises)
-					const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
+					const mediaDataArray = await Promise.all(mediaPromises)
+					const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
 
 					if (dataUrls.length > 0) {
-						setSelectedImages((prevImages) => [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE))
+						setSelectedMedia((prevItems) => [...prevItems, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE))
 					} else {
 						console.warn(t("chat:noValidImages"))
 					}
 				}
 			},
-			[shouldDisableImages, setSelectedImages, cursorPosition, setInputValue, inputValue, t],
+			[shouldDisableImages, setSelectedMedia, cursorPosition, setInputValue, inputValue, t, acceptedFileTypes],
 		)
 
 		const handleMenuMouseDown = useCallback(() => {
@@ -732,15 +732,13 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 				const files = Array.from(e.dataTransfer.files)
 
 				if (files.length > 0) {
-					const acceptedTypes = ["png", "jpeg", "webp"]
-
-					const imageFiles = files.filter((file) => {
+					const mediaFiles = files.filter((file) => {
 						const [type, subtype] = file.type.split("/")
-						return type === "image" && acceptedTypes.includes(subtype)
+						return (type === "image" || type === "video") && acceptedFileTypes.includes(subtype)
 					})
 
-					if (!shouldDisableImages && imageFiles.length > 0) {
-						const imagePromises = imageFiles.map((file) => {
+					if (!shouldDisableImages && mediaFiles.length > 0) {
+						const mediaPromises = mediaFiles.map((file) => {
 							return new Promise<string | null>((resolve) => {
 								const reader = new FileReader()
 
@@ -758,12 +756,12 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 							})
 						})
 
-						const imageDataArray = await Promise.all(imagePromises)
-						const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
+						const mediaDataArray = await Promise.all(mediaPromises)
+						const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
 
 						if (dataUrls.length > 0) {
-							setSelectedImages((prevImages) =>
-								[...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE),
+							setSelectedMedia((prevItems) =>
+								[...prevItems, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE),
 							)
 
 							if (typeof vscode !== "undefined") {
@@ -783,8 +781,9 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 				setCursorPosition,
 				setIntendedCursorPosition,
 				shouldDisableImages,
-				setSelectedImages,
+				setSelectedMedia,
 				t,
+				acceptedFileTypes,
 			],
 		)
 
@@ -1268,10 +1267,10 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 					)}
 				</div>
 
-				{selectedImages.length > 0 && (
-					<Thumbnails
-						images={selectedImages}
-						setImages={setSelectedImages}
+				{selectedMedia.length > 0 && (
+					<MediaThumbnails
+						mediaItems={selectedMedia}
+						setMediaItems={setSelectedMedia}
 						style={{
 							left: "16px",
 							zIndex: 2,

webview-ui/src/components/chat/ChatView.tsx

@@ -9,7 +9,6 @@ import useSound from "use-sound"
 import { LRUCache } from "lru-cache"
 
 import { useDebounceEffect } from "@src/utils/useDebounceEffect"
-import { appendImages } from "@src/utils/imageUtils"
 
 import type { ClineAsk, ClineMessage } from "@roo-code/types"
 
@@ -153,7 +152,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 	const [inputValue, setInputValue] = useState("")
 	const textAreaRef = useRef<HTMLTextAreaElement>(null)
 	const [sendingDisabled, setSendingDisabled] = useState(false)
-	const [selectedImages, setSelectedImages] = useState<string[]>([])
+	const [selectedMedia, setSelectedMedia] = useState<string[]>([])
 
 	// we need to hold on to the ask because useEffect > lastMessage will always let us know when an ask comes in and handle it, but by the time handleMessage is called, the last message might not be the ask anymore (it could be a say that followed)
 	const [clineAsk, setClineAsk] = useState<ClineAsk | undefined>(undefined)
@@ -396,7 +395,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 						case "api_req_started":
 							if (secondLastMessage?.ask === "command_output") {
 								setSendingDisabled(true)
-								setSelectedImages([])
+								setSelectedMedia([])
 								setClineAsk(undefined)
 								setEnableButtons(false)
 							}
@@ -529,7 +528,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 		// Only reset message-specific state, preserving mode.
 		setInputValue("")
 		setSendingDisabled(true)
-		setSelectedImages([])
+		setSelectedMedia([])
 		setClineAsk(undefined)
 		setEnableButtons(false)
 		// Do not reset mode here as it should persist.
@@ -589,9 +588,9 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 			}
 
 			setInputValue(newValue)
-			setSelectedImages([...selectedImages, ...images])
+			setSelectedMedia([...selectedMedia, ...images])
 		},
-		[inputValue, selectedImages],
+		[inputValue, selectedMedia],
 	)
 
 	const startNewTask = useCallback(() => vscode.postMessage({ type: "clearTask" }), [])
@@ -627,7 +626,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 					}
 					// Clear input state after sending
 					setInputValue("")
-					setSelectedImages([])
+					setSelectedMedia([])
 					break
 				case "completion_result":
 				case "resume_completed_task":
@@ -683,7 +682,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 					}
 					// Clear input state after sending
 					setInputValue("")
-					setSelectedImages([])
+					setSelectedMedia([])
 					break
 				case "command_output":
 					vscode.postMessage({ type: "terminalOperation", terminalOperation: "abort" })
@@ -702,8 +701,31 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 
 	const selectImages = useCallback(() => vscode.postMessage({ type: "selectImages" }), [])
 
+	const acceptedFileTypes = useMemo(() => {
+		const modelId = apiConfiguration?.apiModelId
+		const isGeminiPro = modelId?.includes("gemini-2.5-pro")
+		const isGeminiFlash =
+			modelId?.includes("gemini-1.5-flash") ||
+			modelId?.includes("gemini-2.0-flash-001") ||
+			modelId?.includes("gemini-2.5-flash-preview-05-20") ||
+			modelId?.includes("gemini-2.5-flash") ||
+			modelId?.includes("gemini-2.0-flash-lite-preview-02-05") ||
+			modelId?.includes("gemini-2.0-flash-thinking-exp-01-21") ||
+			modelId?.includes("gemini-2.0-flash-thinking-exp-1219") ||
+			modelId?.includes("gemini-2.0-flash-exp") ||
+			modelId?.includes("gemini-2.5-flash-lite-preview-06-17")
+
+		if ((isGeminiPro || isGeminiFlash) && model?.supportsImages) {
+			return ["png", "jpeg", "webp", "heic", "heif", "mp4", "mov", "avi", "wmv", "flv", "webm"]
+		}
+		if (model?.supportsImages) {
+			return ["png", "jpeg", "webp", "heic", "heif"]
+		}
+		return []
+	}, [apiConfiguration, model])
+
 	const shouldDisableImages =
-		!model?.supportsImages || sendingDisabled || selectedImages.length >= MAX_IMAGES_PER_MESSAGE
+		!model?.supportsImages || sendingDisabled || selectedMedia.length >= MAX_IMAGES_PER_MESSAGE
 
 	const handleMessage = useCallback(
 		(e: MessageEvent) => {
@@ -726,8 +748,8 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 					// Only handle selectedImages if it's not for editing context
 					// When context is "edit", ChatRow will handle the images
 					if (message.context !== "edit") {
-						setSelectedImages((prevImages) =>
-							appendImages(prevImages, message.images, MAX_IMAGES_PER_MESSAGE),
+						setSelectedMedia((prevItems) =>
+							[...prevItems, ...(message.images ?? [])].slice(0, MAX_IMAGES_PER_MESSAGE),
 						)
 					}
 					break
@@ -1611,9 +1633,9 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 	useImperativeHandle(ref, () => ({
 		acceptInput: () => {
 			if (enableButtons && primaryButtonText) {
-				handlePrimaryButtonClick(inputValue, selectedImages)
-			} else if (!sendingDisabled && !isProfileDisabled && (inputValue.trim() || selectedImages.length > 0)) {
-				handleSendMessage(inputValue, selectedImages)
+				handlePrimaryButtonClick(inputValue, selectedMedia)
+			} else if (!sendingDisabled && !isProfileDisabled && (inputValue.trim() || selectedMedia.length > 0)) {
+				handleSendMessage(inputValue, selectedMedia)
 			}
 		},
 	}))
@@ -1813,7 +1835,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 												appearance="primary"
 												disabled={!enableButtons}
 												className={secondaryButtonText ? "flex-1 mr-[6px]" : "flex-[2] mr-0"}
-												onClick={() => handlePrimaryButtonClick(inputValue, selectedImages)}>
+												onClick={() => handlePrimaryButtonClick(inputValue, selectedMedia)}>
 												{primaryButtonText}
 											</VSCodeButton>
 										</StandardTooltip>
@@ -1835,7 +1857,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 												appearance="secondary"
 												disabled={!enableButtons && !(isStreaming && !didClickCancel)}
 												className={isStreaming ? "flex-[2] ml-0" : "flex-1 ml-[6px]"}
-												onClick={() => handleSecondaryButtonClick(inputValue, selectedImages)}>
+												onClick={() => handleSecondaryButtonClick(inputValue, selectedMedia)}>
 												{isStreaming ? t("chat:cancel.title") : secondaryButtonText}
 											</VSCodeButton>
 										</StandardTooltip>
@@ -1854,11 +1876,12 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 				sendingDisabled={sendingDisabled || isProfileDisabled}
 				selectApiConfigDisabled={sendingDisabled && clineAsk !== "api_req_failed"}
 				placeholderText={placeholderText}
-				selectedImages={selectedImages}
-				setSelectedImages={setSelectedImages}
-				onSend={() => handleSendMessage(inputValue, selectedImages)}
+				selectedMedia={selectedMedia}
+				setSelectedMedia={setSelectedMedia}
+				onSend={() => handleSendMessage(inputValue, selectedMedia)}
 				onSelectImages={selectImages}
 				shouldDisableImages={shouldDisableImages}
+				acceptedFileTypes={acceptedFileTypes}
 				onHeightChange={() => {
 					if (isAtBottom) {
 						scrollToBottomAuto()

webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx

@@ -54,12 +54,13 @@ describe("ChatTextArea", () => {
 		onSelectImages: vi.fn(),
 		shouldDisableImages: false,
 		placeholderText: "Type a message...",
-		selectedImages: [],
-		setSelectedImages: vi.fn(),
+		selectedMedia: [],
+		setSelectedMedia: vi.fn(),
 		onHeightChange: vi.fn(),
 		mode: defaultModeSlug,
 		setMode: vi.fn(),
 		modeShortcutText: "(⌘. for next mode)",
+		acceptedFileTypes: ["png", "jpeg", "gif", "mp4"],
 	}
 
 	beforeEach(() => {

webview-ui/src/components/common/MediaThumbnails.tsx

@@ -0,0 +1,43 @@
+import React from "react"
+import { FileVideo, X } from "lucide-react"
+import { getMimeType } from "../../utils/getMimeType"
+
+interface MediaThumbnailsProps {
+	mediaItems: string[]
+	setMediaItems: React.Dispatch<React.SetStateAction<string[]>>
+	style?: React.CSSProperties
+}
+
+const MediaThumbnails: React.FC<MediaThumbnailsProps> = ({ mediaItems, setMediaItems, style }) => {
+	const handleRemoveImage = (index: number) => {
+		setMediaItems((prevImages) => prevImages.filter((_, i) => i !== index))
+	}
+
+	return (
+		<div className="flex flex-wrap gap-2 p-2 bg-vscode-input-background" style={style}>
+			{mediaItems.map((item, index) => {
+				const mimeType = getMimeType(item)
+				const isVideo = mimeType?.startsWith("video/")
+
+				return (
+					<div key={index} className="relative w-16 h-16">
+						{isVideo ? (
+							<div className="w-full h-full flex items-center justify-center bg-vscode-input-background rounded">
+								<FileVideo className="w-8 h-8 text-vscode-descriptionForeground" />
+							</div>
+						) : (
+							<img src={item} alt={`thumbnail ${index}`} className="w-full h-full object-cover rounded" />
+						)}
+						<button
+							onClick={() => handleRemoveImage(index)}
+							className="absolute top-0 right-0 bg-red-500 text-white rounded-full p-1">
+							<X size={12} />
+						</button>
+					</div>
+				)
+			})}
+		</div>
+	)
+}
+
+export default MediaThumbnails

webview-ui/src/utils/getMimeType.ts

@@ -0,0 +1,4 @@
+export function getMimeType(dataUri: string): string | null {
+	const match = dataUri.match(/^data:(.*?);/)
+	return match ? match[1] : null
+}