From 2ac9984e284654ccacd3ba9c219c926d13af3918 Mon Sep 17 00:00:00 2001 From: hannesrudolph Date: Thu, 24 Jul 2025 00:06:41 -0600 Subject: [PATCH] fix: address PR #6150 review feedback - Consolidate duplicate getMimeType functions into shared utilities - Remove duplicate MediaThumbnails component, enhance Thumbnails to support video - Add JSDoc comments to VideoContentBlock interface - Convert inline styles to Tailwind classes in ChatRow - Add robust error handling for video processing - Create centralized media configuration for accepted file types - Ensure consistent test naming conventions - Fix ESLint warnings --- src/api/transform/gemini-format.ts | 48 +++- src/integrations/misc/process-images.ts | 20 +- src/shared/utils/media.ts | 54 +++++ webview-ui/src/components/chat/ChatRow.tsx | 215 ++++++------------ .../src/components/chat/ChatTextArea.tsx | 8 +- webview-ui/src/components/chat/ChatView.tsx | 24 +- .../chat/__tests__/ChatTextArea.spec.tsx | 3 +- .../src/components/common/MediaThumbnails.tsx | 43 ---- .../src/components/common/Thumbnails.tsx | 124 ++++++---- webview-ui/src/utils/getMimeType.ts | 4 - webview-ui/src/utils/media-config.ts | 84 +++++++ webview-ui/src/utils/media.ts | 27 +++ 12 files changed, 378 insertions(+), 276 deletions(-) create mode 100644 src/shared/utils/media.ts delete mode 100644 webview-ui/src/components/common/MediaThumbnails.tsx delete mode 100644 webview-ui/src/utils/getMimeType.ts create mode 100644 webview-ui/src/utils/media-config.ts create mode 100644 webview-ui/src/utils/media.ts diff --git a/src/api/transform/gemini-format.ts b/src/api/transform/gemini-format.ts index 9b715f8f3d..ec356eb45d 100644 --- a/src/api/transform/gemini-format.ts +++ b/src/api/transform/gemini-format.ts @@ -1,7 +1,17 @@ import { Anthropic } from "@anthropic-ai/sdk" import { Content, Part } from "@google/genai" -// Extended type to support video content blocks that aren't in the standard Anthropic SDK +/** + * Extended content block type to support video content that isn't in the standard Anthropic SDK. + * This interface extends the standard Anthropic content blocks to include video support for Gemini models. + * + * @interface VideoContentBlock + * @property {string} type - Must be "video" to identify this as a video content block + * @property {Object} source - The video source information + * @property {string} source.type - Must be "base64" for base64-encoded video data + * @property {string} source.data - The base64-encoded video data + * @property {string} source.media_type - The MIME type of the video (e.g., "video/mp4", "video/webm") + */ interface VideoContentBlock { type: "video" source: { @@ -11,6 +21,10 @@ interface VideoContentBlock { } } +/** + * Extended content block parameter type that includes both standard Anthropic content blocks + * and our custom video content block for Gemini model support. + */ type ExtendedContentBlockParam = Anthropic.ContentBlockParam | VideoContentBlock export function convertAnthropicContentToGemini(content: string | ExtendedContentBlockParam[]): Part[] { @@ -28,11 +42,39 @@ export function convertAnthropicContentToGemini(content: string | ExtendedConten } return { inlineData: { data: block.source.data, mimeType: block.source.media_type } } - case "video": + case "video": { if (block.source.type !== "base64") { - throw new Error("Unsupported video source type") + throw new Error("Unsupported video source type. Only base64 encoded videos are supported.") } + + // Validate video MIME type + const supportedVideoTypes = ["video/mp4", "video/webm", "video/ogg", "video/quicktime"] + if (!supportedVideoTypes.includes(block.source.media_type)) { + throw new Error( + `Unsupported video format: ${block.source.media_type}. Supported formats: ${supportedVideoTypes.join(", ")}`, + ) + } + + // Check if video data exists + if (!block.source.data || block.source.data.trim() === "") { + throw new Error("Video data is empty or missing") + } + + // Validate base64 format + try { + // Basic validation - check if it's valid base64 + const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/ + if (!base64Regex.test(block.source.data.replace(/\s/g, ""))) { + throw new Error("Invalid base64 format for video data") + } + } catch (e) { + throw new Error( + `Failed to validate video data: ${e instanceof Error ? e.message : "Unknown error"}`, + ) + } + return { inlineData: { data: block.source.data, mimeType: block.source.media_type } } + } case "tool_use": return { functionCall: { diff --git a/src/integrations/misc/process-images.ts b/src/integrations/misc/process-images.ts index cf3e201538..7ca7b455f1 100644 --- a/src/integrations/misc/process-images.ts +++ b/src/integrations/misc/process-images.ts @@ -1,6 +1,6 @@ import * as vscode from "vscode" import fs from "fs/promises" -import * as path from "path" +import { getMimeType } from "../../shared/utils/media" export async function selectImages(): Promise { const options: vscode.OpenDialogOptions = { @@ -23,23 +23,11 @@ export async function selectImages(): Promise { const buffer = await fs.readFile(imagePath) const base64 = buffer.toString("base64") const mimeType = getMimeType(imagePath) + if (!mimeType) { + throw new Error(`Unsupported file type: ${imagePath}`) + } const dataUrl = `data:${mimeType};base64,${base64}` return dataUrl }), ) } - -function getMimeType(filePath: string): string { - const ext = path.extname(filePath).toLowerCase() - switch (ext) { - case ".png": - return "image/png" - case ".jpeg": - case ".jpg": - return "image/jpeg" - case ".webp": - return "image/webp" - default: - throw new Error(`Unsupported file type: ${ext}`) - } -} diff --git a/src/shared/utils/media.ts b/src/shared/utils/media.ts new file mode 100644 index 0000000000..60307d9a1b --- /dev/null +++ b/src/shared/utils/media.ts @@ -0,0 +1,54 @@ +import * as path from "path" + +/** + * Get MIME type from either a file path or a data URI + * @param input - Either a file path or a data URI + * @returns The MIME type or null if not found + */ +export function getMimeType(input: string): string | null { + // Check if it's a data URI + if (input.startsWith("data:")) { + const match = input.match(/^data:(.*?);/) + return match ? match[1] : null + } + + // Otherwise, treat it as a file path + const ext = path.extname(input).toLowerCase() + switch (ext) { + case ".png": + return "image/png" + case ".jpeg": + case ".jpg": + return "image/jpeg" + case ".webp": + return "image/webp" + case ".mp4": + return "video/mp4" + case ".webm": + return "video/webm" + case ".ogg": + return "video/ogg" + case ".mov": + return "video/quicktime" + default: + return null + } +} + +/** + * Check if a MIME type represents a video + * @param mimeType - The MIME type to check + * @returns True if it's a video MIME type + */ +export function isVideoMimeType(mimeType: string | null): boolean { + return mimeType?.startsWith("video/") ?? false +} + +/** + * Check if a MIME type represents an image + * @param mimeType - The MIME type to check + * @returns True if it's an image MIME type + */ +export function isImageMimeType(mimeType: string | null): boolean { + return mimeType?.startsWith("image/") ?? false +} diff --git a/webview-ui/src/components/chat/ChatRow.tsx b/webview-ui/src/components/chat/ChatRow.tsx index ddaf01bb97..346d50584f 100644 --- a/webview-ui/src/components/chat/ChatRow.tsx +++ b/webview-ui/src/components/chat/ChatRow.tsx @@ -201,37 +201,26 @@ export const ChatRowContent = ({ const type = message.type === "ask" ? message.ask : message.say - const normalColor = "var(--vscode-foreground)" - const errorColor = "var(--vscode-errorForeground)" - const successColor = "var(--vscode-charts-green)" - const cancelledColor = "var(--vscode-descriptionForeground)" - const [icon, title] = useMemo(() => { switch (type) { case "error": return [ - , - {t("chat:error")}, + , + {t("chat:error")}, ] case "mistake_limit_reached": return [ - , - {t("chat:troubleMessage")}, + , + {t("chat:troubleMessage")}, ] case "command": return [ isCommandExecuting ? ( ) : ( - + ), - {t("chat:runCommand.title")}:, + {t("chat:runCommand.title")}:, ] case "use_mcp_server": const mcpServerUse = safeJsonParse(message.text) @@ -242,11 +231,9 @@ export const ChatRowContent = ({ isMcpServerResponding ? ( ) : ( - + ), - + {mcpServerUse.type === "use_mcp_tool" ? t("chat:mcp.wantsToUseTool", { serverName: mcpServerUse.serverName }) : t("chat:mcp.wantsToAccessResource", { serverName: mcpServerUse.serverName })} @@ -254,88 +241,60 @@ export const ChatRowContent = ({ ] case "completion_result": return [ - , - {t("chat:taskCompleted")}, + , + {t("chat:taskCompleted")}, ] case "api_req_retry_delayed": return [] case "api_req_started": - const getIconSpan = (iconName: string, color: string) => ( -
- + const getIconSpan = (iconName: string, colorClass: string) => ( +
+
) return [ apiReqCancelReason !== null && apiReqCancelReason !== undefined ? ( apiReqCancelReason === "user_cancelled" ? ( - getIconSpan("error", cancelledColor) + getIconSpan("error", "text-vscode-descriptionForeground") ) : ( - getIconSpan("error", errorColor) + getIconSpan("error", "text-vscode-errorForeground") ) ) : cost !== null && cost !== undefined ? ( - getIconSpan("check", successColor) + getIconSpan("check", "text-vscode-charts-green") ) : apiRequestFailedMessage ? ( - getIconSpan("error", errorColor) + getIconSpan("error", "text-vscode-errorForeground") ) : ( ), apiReqCancelReason !== null && apiReqCancelReason !== undefined ? ( apiReqCancelReason === "user_cancelled" ? ( - - {t("chat:apiRequest.cancelled")} - + {t("chat:apiRequest.cancelled")} ) : ( - + {t("chat:apiRequest.streamingFailed")} ) ) : cost !== null && cost !== undefined ? ( - {t("chat:apiRequest.title")} + {t("chat:apiRequest.title")} ) : apiRequestFailedMessage ? ( - {t("chat:apiRequest.failed")} + {t("chat:apiRequest.failed")} ) : ( - {t("chat:apiRequest.streaming")} + {t("chat:apiRequest.streaming")} ), ] case "followup": return [ - , - {t("chat:questions.hasQuestion")}, + , + {t("chat:questions.hasQuestion")}, ] default: return [null, null] } }, [type, isCommandExecuting, message, isMcpServerResponding, apiReqCancelReason, cost, apiRequestFailedMessage, t]) - const headerStyle: React.CSSProperties = { - display: "flex", - alignItems: "center", - gap: "10px", - marginBottom: "10px", - wordBreak: "break-word", - } + const headerClassName = "flex items-center gap-2.5 mb-2.5 break-words" - const pStyle: React.CSSProperties = { - margin: 0, - whiteSpace: "pre-wrap", - wordBreak: "break-word", - overflowWrap: "anywhere", - } + const pClassName = "m-0 whitespace-pre-wrap break-words overflow-wrap-anywhere" const tool = useMemo( () => (message.ask === "tool" ? safeJsonParse(message.text) : null), @@ -351,9 +310,7 @@ export const ChatRowContent = ({ if (tool) { const toolIcon = (name: string) => ( - + ) switch (tool.tool) { @@ -363,11 +320,9 @@ export const ChatRowContent = ({ if (message.type === "ask" && tool.batchDiffs && Array.isArray(tool.batchDiffs)) { return ( <> -
+
{toolIcon("diff")} - - {t("chat:fileOperations.wantsToApplyBatchChanges")} - + {t("chat:fileOperations.wantsToApplyBatchChanges")}
@@ -377,16 +332,13 @@ export const ChatRowContent = ({ // Regular single file diff return ( <> -
+
{tool.isProtected ? ( - + ) : ( toolIcon(tool.tool === "appliedDiff" ? "diff" : "edit") )} - + {tool.isProtected ? t("chat:fileOperations.wantsToEditProtected") : tool.isOutsideWorkspace @@ -408,16 +360,13 @@ export const ChatRowContent = ({ case "insertContent": return ( <> -
+
{tool.isProtected ? ( - + ) : ( toolIcon("insert") )} - + {tool.isProtected ? t("chat:fileOperations.wantsToEditProtected") : tool.isOutsideWorkspace @@ -443,16 +392,13 @@ export const ChatRowContent = ({ case "searchAndReplace": return ( <> -
+
{tool.isProtected ? ( - + ) : ( toolIcon("replace") )} - + {tool.isProtected && message.type === "ask" ? t("chat:fileOperations.wantsToEditProtected") : message.type === "ask" @@ -473,9 +419,9 @@ export const ChatRowContent = ({ ) case "codebaseSearch": { return ( -
+
{toolIcon("search")} - + {tool.path ? ( -
+
{tool.isProtected ? ( - + ) : ( toolIcon("new-file") )} - + {tool.isProtected ? t("chat:fileOperations.wantsToEditProtected") : t("chat:fileOperations.wantsToCreate")} @@ -544,11 +487,9 @@ export const ChatRowContent = ({ if (isBatchRequest) { return ( <> -
+
{toolIcon("files")} - - {t("chat:fileOperations.wantsToReadMultiple")} - + {t("chat:fileOperations.wantsToReadMultiple")}
-
+
{toolIcon("file-code")} - + {message.type === "ask" ? tool.isOutsideWorkspace ? t("chat:fileOperations.wantsToReadOutsideWorkspace") @@ -598,9 +539,9 @@ export const ChatRowContent = ({ case "fetchInstructions": return ( <> -
+
{toolIcon("file-code")} - {t("chat:instructions.wantsToFetch")} + {t("chat:instructions.wantsToFetch")}
-
+
{toolIcon("folder-opened")} - + {message.type === "ask" ? tool.isOutsideWorkspace ? t("chat:directoryOperations.wantsToViewTopLevelOutsideWorkspace") @@ -638,9 +579,9 @@ export const ChatRowContent = ({ case "listFilesRecursive": return ( <> -
+
{toolIcon("folder-opened")} - + {message.type === "ask" ? tool.isOutsideWorkspace ? t("chat:directoryOperations.wantsToViewRecursiveOutsideWorkspace") @@ -662,9 +603,9 @@ export const ChatRowContent = ({ case "listCodeDefinitionNames": return ( <> -
+
{toolIcon("file-code")} - + {message.type === "ask" ? tool.isOutsideWorkspace ? t("chat:directoryOperations.wantsToViewDefinitionsOutsideWorkspace") @@ -686,9 +627,9 @@ export const ChatRowContent = ({ case "searchFiles": return ( <> -
+
{toolIcon("search")} - + {message.type === "ask" ? ( -
+
{toolIcon("symbol-enum")} - + {message.type === "ask" ? ( <> {tool.reason ? ( @@ -767,9 +708,9 @@ export const ChatRowContent = ({ case "newTask": return ( <> -
+
{toolIcon("tasklist")} - + {tool.mode} }} @@ -810,9 +751,9 @@ export const ChatRowContent = ({ case "finishTask": return ( <> -
+
{toolIcon("check-all")} - {t("chat:subtasks.wantsToFinish")} + {t("chat:subtasks.wantsToFinish")}
- {t("chat:diffError.title")} + {t("chat:diffError.title")}
@@ -1019,7 +954,7 @@ export const ChatRowContent = ({ {(((cost === null || cost === undefined) && apiRequestFailedMessage) || apiReqStreamingFailedMessage) && ( <> -

+

{apiRequestFailedMessage || apiReqStreamingFailedMessage} {apiRequestFailedMessage?.toLowerCase().includes("powershell") && ( <> @@ -1135,18 +1070,18 @@ export const ChatRowContent = ({ return ( <> {title && ( -

+
{icon} {title}
)} -

{message.text}

+

{message.text}

) case "completion_result": return ( <> -
+
{icon} {title}
@@ -1209,7 +1144,7 @@ export const ChatRowContent = ({ return ( <> {title && ( -
+
{icon} {title}
@@ -1225,11 +1160,11 @@ export const ChatRowContent = ({ case "mistake_limit_reached": return ( <> -
+
{icon} {title}
-

{message.text}

+

{message.text}

) case "command": @@ -1262,7 +1197,7 @@ export const ChatRowContent = ({ return ( <> -
+
{icon} {title}
@@ -1304,7 +1239,7 @@ export const ChatRowContent = ({ if (message.text) { return (
-
+
{icon} {title}
@@ -1320,7 +1255,7 @@ export const ChatRowContent = ({ return ( <> {title && ( -
+
{icon} {title}
diff --git a/webview-ui/src/components/chat/ChatTextArea.tsx b/webview-ui/src/components/chat/ChatTextArea.tsx index 8a2ab96ee5..30dfa90d6b 100644 --- a/webview-ui/src/components/chat/ChatTextArea.tsx +++ b/webview-ui/src/components/chat/ChatTextArea.tsx @@ -23,7 +23,7 @@ import { SelectDropdown, DropdownOptionType, Button, StandardTooltip } from "@/c import ModeSelector from "./ModeSelector" import { MAX_IMAGES_PER_MESSAGE } from "./ChatView" -import MediaThumbnails from "../common/MediaThumbnails" +import Thumbnails from "../common/Thumbnails" import ContextMenu from "./ContextMenu" import { VolumeX, Pin, Check, Image, WandSparkles, SendHorizontal } from "lucide-react" import { IndexingStatusBadge } from "./IndexingStatusBadge" @@ -1268,9 +1268,9 @@ const ChatTextArea = forwardRef(
{selectedMedia.length > 0 && ( - vscode.postMessage({ type: "selectImages" }), []) const acceptedFileTypes = useMemo(() => { - const modelId = apiConfiguration?.apiModelId - const isGeminiPro = modelId?.includes("gemini-2.5-pro") - const isGeminiFlash = - modelId?.includes("gemini-1.5-flash") || - modelId?.includes("gemini-2.0-flash-001") || - modelId?.includes("gemini-2.5-flash-preview-05-20") || - modelId?.includes("gemini-2.5-flash") || - modelId?.includes("gemini-2.0-flash-lite-preview-02-05") || - modelId?.includes("gemini-2.0-flash-thinking-exp-01-21") || - modelId?.includes("gemini-2.0-flash-thinking-exp-1219") || - modelId?.includes("gemini-2.0-flash-exp") || - modelId?.includes("gemini-2.5-flash-lite-preview-06-17") - - if ((isGeminiPro || isGeminiFlash) && model?.supportsImages) { - return ["png", "jpeg", "webp", "heic", "heif", "mp4", "mov", "avi", "wmv", "flv", "webm"] - } - if (model?.supportsImages) { - return ["png", "jpeg", "webp", "heic", "heif"] - } - return [] - }, [apiConfiguration, model]) + return getAcceptedFileTypes(apiConfiguration?.apiModelId, model?.supportsImages) + }, [apiConfiguration?.apiModelId, model?.supportsImages]) const shouldDisableImages = !model?.supportsImages || sendingDisabled || selectedMedia.length >= MAX_IMAGES_PER_MESSAGE diff --git a/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx b/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx index 3d449f9eaa..715490f8a6 100644 --- a/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx +++ b/webview-ui/src/components/chat/__tests__/ChatTextArea.spec.tsx @@ -5,6 +5,7 @@ import { defaultModeSlug } from "@roo/modes" import { useExtensionState } from "@src/context/ExtensionStateContext" import { vscode } from "@src/utils/vscode" import * as pathMentions from "@src/utils/path-mentions" +import { BASE_IMAGE_FORMATS, VIDEO_FORMATS } from "@src/utils/media-config" import ChatTextArea from "../ChatTextArea" @@ -60,7 +61,7 @@ describe("ChatTextArea", () => { mode: defaultModeSlug, setMode: vi.fn(), modeShortcutText: "(⌘. for next mode)", - acceptedFileTypes: ["png", "jpeg", "gif", "mp4"], + acceptedFileTypes: [...BASE_IMAGE_FORMATS, ...VIDEO_FORMATS], } beforeEach(() => { diff --git a/webview-ui/src/components/common/MediaThumbnails.tsx b/webview-ui/src/components/common/MediaThumbnails.tsx deleted file mode 100644 index 8a3bd4e76f..0000000000 --- a/webview-ui/src/components/common/MediaThumbnails.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import React from "react" -import { FileVideo, X } from "lucide-react" -import { getMimeType } from "../../utils/getMimeType" - -interface MediaThumbnailsProps { - mediaItems: string[] - setMediaItems: React.Dispatch> - style?: React.CSSProperties -} - -const MediaThumbnails: React.FC = ({ mediaItems, setMediaItems, style }) => { - const handleRemoveImage = (index: number) => { - setMediaItems((prevImages) => prevImages.filter((_, i) => i !== index)) - } - - return ( -
- {mediaItems.map((item, index) => { - const mimeType = getMimeType(item) - const isVideo = mimeType?.startsWith("video/") - - return ( -
- {isVideo ? ( -
- -
- ) : ( - {`thumbnail - )} - -
- ) - })} -
- ) -} - -export default MediaThumbnails diff --git a/webview-ui/src/components/common/Thumbnails.tsx b/webview-ui/src/components/common/Thumbnails.tsx index acdf5f4295..d3fb12a8d4 100644 --- a/webview-ui/src/components/common/Thumbnails.tsx +++ b/webview-ui/src/components/common/Thumbnails.tsx @@ -1,6 +1,8 @@ import React, { useState, useRef, useLayoutEffect, memo } from "react" import { useWindowSize } from "react-use" import { vscode } from "@src/utils/vscode" +import { getMimeType, isVideoMimeType } from "../../utils/media" +import { FileVideo } from "lucide-react" interface ThumbnailsProps { images: string[] @@ -46,51 +48,85 @@ const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProp rowGap: 3, ...style, }}> - {images.map((image, index) => ( -
setHoveredIndex(index)} - onMouseLeave={() => setHoveredIndex(null)}> - {`Thumbnail handleImageClick(image)} - /> - {isDeletable && hoveredIndex === index && ( -
handleDelete(index)} - style={{ - position: "absolute", - top: -4, - right: -4, - width: 13, - height: 13, - borderRadius: "50%", - backgroundColor: "var(--vscode-badge-background)", - display: "flex", - justifyContent: "center", - alignItems: "center", - cursor: "pointer", - }}> - { + const mimeType = getMimeType(image) + const isVideo = isVideoMimeType(mimeType) + + return ( +
setHoveredIndex(index)} + onMouseLeave={() => setHoveredIndex(null)}> + {isVideo ? ( +
-
- )} -
- ))} + width: 34, + height: 34, + display: "flex", + alignItems: "center", + justifyContent: "center", + backgroundColor: "var(--vscode-input-background)", + borderRadius: 4, + cursor: "pointer", + }} + onClick={() => handleImageClick(image)} + title={`Video: ${mimeType || "Unknown format"}`}> + +
+ ) : ( + {`Thumbnail handleImageClick(image)} + onError={(e) => { + // Handle image load errors + const target = e.target as HTMLImageElement + target.style.display = "none" + const errorDiv = document.createElement("div") + errorDiv.style.cssText = + "width: 34px; height: 34px; display: flex; align-items: center; justify-content: center; background-color: var(--vscode-input-background); border-radius: 4px; font-size: 10px; color: var(--vscode-errorForeground);" + errorDiv.textContent = "!" + errorDiv.title = "Failed to load image" + target.parentNode?.appendChild(errorDiv) + }} + /> + )} + {isDeletable && hoveredIndex === index && ( +
handleDelete(index)} + style={{ + position: "absolute", + top: -4, + right: -4, + width: 13, + height: 13, + borderRadius: "50%", + backgroundColor: "var(--vscode-badge-background)", + display: "flex", + justifyContent: "center", + alignItems: "center", + cursor: "pointer", + }}> + +
+ )} +
+ ) + })}
) } diff --git a/webview-ui/src/utils/getMimeType.ts b/webview-ui/src/utils/getMimeType.ts deleted file mode 100644 index ff31e5656f..0000000000 --- a/webview-ui/src/utils/getMimeType.ts +++ /dev/null @@ -1,4 +0,0 @@ -export function getMimeType(dataUri: string): string | null { - const match = dataUri.match(/^data:(.*?);/) - return match ? match[1] : null -} diff --git a/webview-ui/src/utils/media-config.ts b/webview-ui/src/utils/media-config.ts new file mode 100644 index 0000000000..45ded84e8c --- /dev/null +++ b/webview-ui/src/utils/media-config.ts @@ -0,0 +1,84 @@ +/** + * Configuration for accepted media file types by model + */ + +export interface MediaConfig { + images: string[] + videos: string[] +} + +// Base image formats supported by most models +export const BASE_IMAGE_FORMATS = ["png", "jpeg", "webp", "heic", "heif"] + +// Video formats supported by Gemini models +export const VIDEO_FORMATS = ["mp4", "mov", "avi", "wmv", "flv", "webm"] + +// Configuration for different model types +export const MEDIA_CONFIG: Record = { + // Gemini Pro and Flash models support both images and videos + gemini_full: { + images: BASE_IMAGE_FORMATS, + videos: VIDEO_FORMATS, + }, + // Default configuration for models that only support images + default: { + images: BASE_IMAGE_FORMATS, + videos: [], + }, + // Configuration for models that don't support any media + none: { + images: [], + videos: [], + }, +} + +/** + * Get accepted file types for a given model + * @param modelId - The model ID + * @param supportsImages - Whether the model supports images + * @returns Array of accepted file extensions + */ +export function getAcceptedFileTypes(modelId: string | undefined, supportsImages: boolean | undefined): string[] { + if (!supportsImages) { + return [] + } + + // Check if it's a Gemini model that supports video + const isGeminiWithVideo = + modelId?.includes("gemini-2.5-pro") || + modelId?.includes("gemini-1.5-flash") || + modelId?.includes("gemini-2.0-flash-001") || + modelId?.includes("gemini-2.5-flash-preview-05-20") || + modelId?.includes("gemini-2.5-flash") || + modelId?.includes("gemini-2.0-flash-lite-preview-02-05") || + modelId?.includes("gemini-2.0-flash-thinking-exp-01-21") || + modelId?.includes("gemini-2.0-flash-thinking-exp-1219") || + modelId?.includes("gemini-2.0-flash-exp") || + modelId?.includes("gemini-2.5-flash-lite-preview-06-17") + + if (isGeminiWithVideo) { + const config = MEDIA_CONFIG.gemini_full + return [...config.images, ...config.videos] + } + + // Default to image-only support + return MEDIA_CONFIG.default.images +} + +/** + * Check if a file type is an image + * @param fileType - The file extension (without dot) + * @returns true if the file type is an image + */ +export function isImageFileType(fileType: string): boolean { + return BASE_IMAGE_FORMATS.includes(fileType.toLowerCase()) +} + +/** + * Check if a file type is a video + * @param fileType - The file extension (without dot) + * @returns true if the file type is a video + */ +export function isVideoFileType(fileType: string): boolean { + return VIDEO_FORMATS.includes(fileType.toLowerCase()) +} diff --git a/webview-ui/src/utils/media.ts b/webview-ui/src/utils/media.ts new file mode 100644 index 0000000000..923e1136f6 --- /dev/null +++ b/webview-ui/src/utils/media.ts @@ -0,0 +1,27 @@ +/** + * Get MIME type from a data URI + * @param dataUri - A data URI string + * @returns The MIME type or null if not found + */ +export function getMimeType(dataUri: string): string | null { + const match = dataUri.match(/^data:(.*?);/) + return match ? match[1] : null +} + +/** + * Check if a MIME type represents a video + * @param mimeType - The MIME type to check + * @returns True if it's a video MIME type + */ +export function isVideoMimeType(mimeType: string | null): boolean { + return mimeType?.startsWith("video/") ?? false +} + +/** + * Check if a MIME type represents an image + * @param mimeType - The MIME type to check + * @returns True if it's an image MIME type + */ +export function isImageMimeType(mimeType: string | null): boolean { + return mimeType?.startsWith("image/") ?? false +}