diff --git a/src/api/transform/__tests__/gemini-format.spec.ts b/src/api/transform/__tests__/gemini-format.spec.ts index a9f0c15e9f..118cb9f392 100644 --- a/src/api/transform/__tests__/gemini-format.spec.ts +++ b/src/api/transform/__tests__/gemini-format.spec.ts @@ -71,13 +71,77 @@ describe("convertAnthropicMessageToGemini", () => { expect(result).toEqual({ role: "user", parts: [ - { text: "Check out this image:" }, { inlineData: { data: "base64encodeddata", mimeType: "image/jpeg", }, }, + { text: "Check out this image:" }, + ], + }) + }) + + it("should convert a message with a video", () => { + const anthropicMessage: Anthropic.Messages.MessageParam = { + role: "user", + content: [ + { type: "text", text: "Check out this video:" }, + { + type: "video", + source: { + type: "base64", + media_type: "video/mp4", + data: "base64encodedvideodata", + }, + } as any, + ], + } + + const result = convertAnthropicMessageToGemini(anthropicMessage) + + expect(result).toEqual({ + role: "user", + parts: [ + { + inlineData: { + data: "base64encodedvideodata", + mimeType: "video/mp4", + }, + }, + { text: "Check out this video:" }, + ], + }) + }) + + it("should handle text after inlineData", () => { + const anthropicMessage: Anthropic.Messages.MessageParam = { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/jpeg", + data: "base64encodeddata", + }, + }, + { type: "text", text: "Check out this image:" }, + ], + } + + const result = convertAnthropicMessageToGemini(anthropicMessage) + + expect(result).toEqual({ + role: "user", + parts: [ + { + inlineData: { + data: "base64encodeddata", + mimeType: "image/jpeg", + }, + }, + { text: "Check out this image:" }, ], }) }) diff --git a/src/api/transform/gemini-format.ts b/src/api/transform/gemini-format.ts index ee22cff32a..9b715f8f3d 100644 --- a/src/api/transform/gemini-format.ts +++ b/src/api/transform/gemini-format.ts @@ -1,12 +1,24 @@ import { Anthropic } from "@anthropic-ai/sdk" import { Content, Part } from "@google/genai" -export function convertAnthropicContentToGemini(content: string | Anthropic.ContentBlockParam[]): Part[] { +// Extended type to support video content blocks that aren't in the standard Anthropic SDK +interface VideoContentBlock { + type: "video" + source: { + type: "base64" + data: string + media_type: string + } +} + +type ExtendedContentBlockParam = Anthropic.ContentBlockParam | VideoContentBlock + +export function convertAnthropicContentToGemini(content: string | ExtendedContentBlockParam[]): Part[] { if (typeof content === "string") { return [{ text: content }] } - return content.flatMap((block): Part | Part[] => { + const parts = content.flatMap((block): Part | Part[] => { switch (block.type) { case "text": return { text: block.text } @@ -15,6 +27,11 @@ export function convertAnthropicContentToGemini(content: string | Anthropic.Cont throw new Error("Unsupported image source type") } + return { inlineData: { data: block.source.data, mimeType: block.source.media_type } } + case "video": + if (block.source.type !== "base64") { + throw new Error("Unsupported video source type") + } return { inlineData: { data: block.source.data, mimeType: block.source.media_type } } case "tool_use": return { @@ -68,6 +85,17 @@ export function convertAnthropicContentToGemini(content: string | Anthropic.Cont throw new Error(`Unsupported content block type: ${block.type}`) } }) + + // Sort parts to ensure inlineData comes before text + return parts.sort((a, b) => { + if ("inlineData" in a && "text" in b) { + return -1 + } + if ("text" in a && "inlineData" in b) { + return 1 + } + return 0 + }) } export function convertAnthropicMessageToGemini(message: Anthropic.Messages.MessageParam): Content { diff --git a/webview-ui/src/components/chat/ChatRow.tsx b/webview-ui/src/components/chat/ChatRow.tsx index 4fa921f443..ddaf01bb97 100644 --- a/webview-ui/src/components/chat/ChatRow.tsx +++ b/webview-ui/src/components/chat/ChatRow.tsx @@ -1069,8 +1069,8 @@ export const ChatRowContent = ({ sendingDisabled={false} selectApiConfigDisabled={true} placeholderText={t("chat:editMessage.placeholder")} - selectedImages={editImages} - setSelectedImages={setEditImages} + selectedMedia={editImages} + setSelectedMedia={setEditImages} onSend={handleSaveEdit} onSelectImages={handleSelectImages} shouldDisableImages={false} @@ -1079,6 +1079,7 @@ export const ChatRowContent = ({ modeShortcutText="" isEditMode={true} onCancel={handleCancelEdit} + acceptedFileTypes={[]} /> ) : ( diff --git a/webview-ui/src/components/chat/ChatTextArea.tsx b/webview-ui/src/components/chat/ChatTextArea.tsx index 6c541353eb..8a2ab96ee5 100644 --- a/webview-ui/src/components/chat/ChatTextArea.tsx +++ b/webview-ui/src/components/chat/ChatTextArea.tsx @@ -21,9 +21,9 @@ import { import { convertToMentionPath } from "@/utils/path-mentions" import { SelectDropdown, DropdownOptionType, Button, StandardTooltip } from "@/components/ui" -import Thumbnails from "../common/Thumbnails" import ModeSelector from "./ModeSelector" import { MAX_IMAGES_PER_MESSAGE } from "./ChatView" +import MediaThumbnails from "../common/MediaThumbnails" import ContextMenu from "./ContextMenu" import { VolumeX, Pin, Check, Image, WandSparkles, SendHorizontal } from "lucide-react" import { IndexingStatusBadge } from "./IndexingStatusBadge" @@ -37,8 +37,8 @@ interface ChatTextAreaProps { sendingDisabled: boolean selectApiConfigDisabled: boolean placeholderText: string - selectedImages: string[] - setSelectedImages: React.Dispatch> + selectedMedia: string[] + setSelectedMedia: React.Dispatch> onSend: () => void onSelectImages: () => void shouldDisableImages: boolean @@ -46,6 +46,7 @@ interface ChatTextAreaProps { mode: Mode setMode: (value: Mode) => void modeShortcutText: string + acceptedFileTypes: string[] // Edit mode props isEditMode?: boolean onCancel?: () => void @@ -59,8 +60,8 @@ const ChatTextArea = forwardRef( sendingDisabled, selectApiConfigDisabled, placeholderText, - selectedImages, - setSelectedImages, + selectedMedia, + setSelectedMedia, onSend, onSelectImages, shouldDisableImages, @@ -70,6 +71,7 @@ const ChatTextArea = forwardRef( modeShortcutText, isEditMode = false, onCancel, + acceptedFileTypes, }, ref, ) => { @@ -598,17 +600,15 @@ const ChatTextArea = forwardRef( return } - const acceptedTypes = ["png", "jpeg", "webp"] - - const imageItems = Array.from(items).filter((item) => { + const mediaItems = Array.from(items).filter((item) => { const [type, subtype] = item.type.split("/") - return type === "image" && acceptedTypes.includes(subtype) + return (type === "image" || type === "video") && acceptedFileTypes.includes(subtype) }) - if (!shouldDisableImages && imageItems.length > 0) { + if (!shouldDisableImages && mediaItems.length > 0) { e.preventDefault() - const imagePromises = imageItems.map((item) => { + const mediaPromises = mediaItems.map((item) => { return new Promise((resolve) => { const blob = item.getAsFile() @@ -633,17 +633,17 @@ const ChatTextArea = forwardRef( }) }) - const imageDataArray = await Promise.all(imagePromises) - const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null) + const mediaDataArray = await Promise.all(mediaPromises) + const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null) if (dataUrls.length > 0) { - setSelectedImages((prevImages) => [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE)) + setSelectedMedia((prevItems) => [...prevItems, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE)) } else { console.warn(t("chat:noValidImages")) } } }, - [shouldDisableImages, setSelectedImages, cursorPosition, setInputValue, inputValue, t], + [shouldDisableImages, setSelectedMedia, cursorPosition, setInputValue, inputValue, t, acceptedFileTypes], ) const handleMenuMouseDown = useCallback(() => { @@ -732,15 +732,13 @@ const ChatTextArea = forwardRef( const files = Array.from(e.dataTransfer.files) if (files.length > 0) { - const acceptedTypes = ["png", "jpeg", "webp"] - - const imageFiles = files.filter((file) => { + const mediaFiles = files.filter((file) => { const [type, subtype] = file.type.split("/") - return type === "image" && acceptedTypes.includes(subtype) + return (type === "image" || type === "video") && acceptedFileTypes.includes(subtype) }) - if (!shouldDisableImages && imageFiles.length > 0) { - const imagePromises = imageFiles.map((file) => { + if (!shouldDisableImages && mediaFiles.length > 0) { + const mediaPromises = mediaFiles.map((file) => { return new Promise((resolve) => { const reader = new FileReader() @@ -758,12 +756,12 @@ const ChatTextArea = forwardRef( }) }) - const imageDataArray = await Promise.all(imagePromises) - const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null) + const mediaDataArray = await Promise.all(mediaPromises) + const dataUrls = mediaDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null) if (dataUrls.length > 0) { - setSelectedImages((prevImages) => - [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE), + setSelectedMedia((prevItems) => + [...prevItems, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE), ) if (typeof vscode !== "undefined") { @@ -783,8 +781,9 @@ const ChatTextArea = forwardRef( setCursorPosition, setIntendedCursorPosition, shouldDisableImages, - setSelectedImages, + setSelectedMedia, t, + acceptedFileTypes, ], ) @@ -1268,10 +1267,10 @@ const ChatTextArea = forwardRef( )} - {selectedImages.length > 0 && ( - 0 && ( + (null) const [sendingDisabled, setSendingDisabled] = useState(false) - const [selectedImages, setSelectedImages] = useState([]) + const [selectedMedia, setSelectedMedia] = useState([]) // we need to hold on to the ask because useEffect > lastMessage will always let us know when an ask comes in and handle it, but by the time handleMessage is called, the last message might not be the ask anymore (it could be a say that followed) const [clineAsk, setClineAsk] = useState(undefined) @@ -396,7 +395,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction vscode.postMessage({ type: "clearTask" }), []) @@ -627,7 +626,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction vscode.postMessage({ type: "selectImages" }), []) + const acceptedFileTypes = useMemo(() => { + const modelId = apiConfiguration?.apiModelId + const isGeminiPro = modelId?.includes("gemini-2.5-pro") + const isGeminiFlash = + modelId?.includes("gemini-1.5-flash") || + modelId?.includes("gemini-2.0-flash-001") || + modelId?.includes("gemini-2.5-flash-preview-05-20") || + modelId?.includes("gemini-2.5-flash") || + modelId?.includes("gemini-2.0-flash-lite-preview-02-05") || + modelId?.includes("gemini-2.0-flash-thinking-exp-01-21") || + modelId?.includes("gemini-2.0-flash-thinking-exp-1219") || + modelId?.includes("gemini-2.0-flash-exp") || + modelId?.includes("gemini-2.5-flash-lite-preview-06-17") + + if ((isGeminiPro || isGeminiFlash) && model?.supportsImages) { + return ["png", "jpeg", "webp", "heic", "heif", "mp4", "mov", "avi", "wmv", "flv", "webm"] + } + if (model?.supportsImages) { + return ["png", "jpeg", "webp", "heic", "heif"] + } + return [] + }, [apiConfiguration, model]) + const shouldDisableImages = - !model?.supportsImages || sendingDisabled || selectedImages.length >= MAX_IMAGES_PER_MESSAGE + !model?.supportsImages || sendingDisabled || selectedMedia.length >= MAX_IMAGES_PER_MESSAGE const handleMessage = useCallback( (e: MessageEvent) => { @@ -726,8 +748,8 @@ const ChatViewComponent: React.ForwardRefRenderFunction - appendImages(prevImages, message.images, MAX_IMAGES_PER_MESSAGE), + setSelectedMedia((prevItems) => + [...prevItems, ...(message.images ?? [])].slice(0, MAX_IMAGES_PER_MESSAGE), ) } break @@ -1611,9 +1633,9 @@ const ChatViewComponent: React.ForwardRefRenderFunction ({ acceptInput: () => { if (enableButtons && primaryButtonText) { - handlePrimaryButtonClick(inputValue, selectedImages) - } else if (!sendingDisabled && !isProfileDisabled && (inputValue.trim() || selectedImages.length > 0)) { - handleSendMessage(inputValue, selectedImages) + handlePrimaryButtonClick(inputValue, selectedMedia) + } else if (!sendingDisabled && !isProfileDisabled && (inputValue.trim() || selectedMedia.length > 0)) { + handleSendMessage(inputValue, selectedMedia) } }, })) @@ -1813,7 +1835,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction handlePrimaryButtonClick(inputValue, selectedImages)}> + onClick={() => handlePrimaryButtonClick(inputValue, selectedMedia)}> {primaryButtonText} @@ -1835,7 +1857,7 @@ const ChatViewComponent: React.ForwardRefRenderFunction handleSecondaryButtonClick(inputValue, selectedImages)}> + onClick={() => handleSecondaryButtonClick(inputValue, selectedMedia)}> {isStreaming ? t("chat:cancel.title") : secondaryButtonText} @@ -1854,11 +1876,12 @@ const ChatViewComponent: React.ForwardRefRenderFunction handleSendMessage(inputValue, selectedImages)} + selectedMedia={selectedMedia} + setSelectedMedia={setSelectedMedia} + onSend={() => handleSendMessage(inputValue, selectedMedia)} onSelectImages={selectImages} shouldDisableImages={shouldDisableImages} + acceptedFileTypes={acceptedFileTypes} onHeightChange={() => { if (isAtBottom) { scrollToBottomAuto() diff --git a/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx b/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx index f53bab76a4..3d449f9eaa 100644 --- a/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx +++ b/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx @@ -54,12 +54,13 @@ describe("ChatTextArea", () => { onSelectImages: vi.fn(), shouldDisableImages: false, placeholderText: "Type a message...", - selectedImages: [], - setSelectedImages: vi.fn(), + selectedMedia: [], + setSelectedMedia: vi.fn(), onHeightChange: vi.fn(), mode: defaultModeSlug, setMode: vi.fn(), modeShortcutText: "(⌘. for next mode)", + acceptedFileTypes: ["png", "jpeg", "gif", "mp4"], } beforeEach(() => { diff --git a/webview-ui/src/components/common/MediaThumbnails.tsx b/webview-ui/src/components/common/MediaThumbnails.tsx new file mode 100644 index 0000000000..8a3bd4e76f --- /dev/null +++ b/webview-ui/src/components/common/MediaThumbnails.tsx @@ -0,0 +1,43 @@ +import React from "react" +import { FileVideo, X } from "lucide-react" +import { getMimeType } from "../../utils/getMimeType" + +interface MediaThumbnailsProps { + mediaItems: string[] + setMediaItems: React.Dispatch> + style?: React.CSSProperties +} + +const MediaThumbnails: React.FC = ({ mediaItems, setMediaItems, style }) => { + const handleRemoveImage = (index: number) => { + setMediaItems((prevImages) => prevImages.filter((_, i) => i !== index)) + } + + return ( +
+ {mediaItems.map((item, index) => { + const mimeType = getMimeType(item) + const isVideo = mimeType?.startsWith("video/") + + return ( +
+ {isVideo ? ( +
+ +
+ ) : ( + {`thumbnail + )} + +
+ ) + })} +
+ ) +} + +export default MediaThumbnails diff --git a/webview-ui/src/utils/getMimeType.ts b/webview-ui/src/utils/getMimeType.ts new file mode 100644 index 0000000000..ff31e5656f --- /dev/null +++ b/webview-ui/src/utils/getMimeType.ts @@ -0,0 +1,4 @@ +export function getMimeType(dataUri: string): string | null { + const match = dataUri.match(/^data:(.*?);/) + return match ? match[1] : null +}