feat: add voice-to-text input functionality for AI agent interactions

- Add useSpeechRecognition hook with Web Speech API support
- Integrate microphone button into ChatTextArea component
- Add visual feedback for active recording state
- Display interim transcripts during speech input
- Add comprehensive localization support for voice features
- Include error handling for various speech recognition scenarios
- Add comprehensive test coverage for the speech recognition hook

Fixes #9052

webview-ui/src/components/chat/ChatTextArea.tsx

@@ -1,7 +1,7 @@
 import React, { forwardRef, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"
 import { useEvent } from "react-use"
 import DynamicTextArea from "react-textarea-autosize"
-import { VolumeX, Image, WandSparkles, SendHorizontal, MessageSquareX } from "lucide-react"
+import { VolumeX, Image, WandSparkles, SendHorizontal, MessageSquareX, Mic, MicOff } from "lucide-react"
 
 import { mentionRegex, mentionRegexGlobal, commandRegexGlobal, unescapeSpaces } from "@roo/context-mentions"
 import { WebviewMessage } from "@roo/WebviewMessage"
@@ -31,6 +31,7 @@ import { MAX_IMAGES_PER_MESSAGE } from "./ChatView"
 import ContextMenu from "./ContextMenu"
 import { IndexingStatusBadge } from "./IndexingStatusBadge"
 import { usePromptHistory } from "./hooks/usePromptHistory"
+import { useSpeechRecognition } from "./hooks/useSpeechRecognition"
 import { CloudAccountSwitcher } from "../cloud/CloudAccountSwitcher"
 
 interface ChatTextAreaProps {
@@ -214,6 +215,49 @@ export const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 		const [isEnhancingPrompt, setIsEnhancingPrompt] = useState(false)
 		const [isFocused, setIsFocused] = useState(false)
 
+		// Use custom hook for speech recognition
+		const {
+			isListening,
+			isSupported: isSpeechRecognitionSupported,
+			transcript,
+			interimTranscript,
+			error: speechError,
+			toggleListening,
+			clearTranscript,
+		} = useSpeechRecognition()
+
+		// Update input value when transcript changes
+		useEffect(() => {
+			if (transcript && !isListening) {
+				// Append transcript to existing input value with a space if needed
+				const needsSpace = inputValue.length > 0 && !inputValue.endsWith(" ")
+				const newValue = inputValue + (needsSpace ? " " : "") + transcript
+				setInputValue(newValue)
+				// Clear the transcript after using it
+				clearTranscript()
+
+				// Set cursor position to the end
+				const newCursorPosition = newValue.length
+				setCursorPosition(newCursorPosition)
+				setIntendedCursorPosition(newCursorPosition)
+
+				// Focus the textarea
+				setTimeout(() => {
+					if (textAreaRef.current) {
+						textAreaRef.current.focus()
+						textAreaRef.current.setSelectionRange(newCursorPosition, newCursorPosition)
+					}
+				}, 0)
+			}
+		}, [transcript, isListening, inputValue, setInputValue, clearTranscript])
+
+		// Show speech error if any
+		useEffect(() => {
+			if (speechError) {
+				console.error("Speech recognition error:", speechError)
+			}
+		}, [speechError])
+
 		// Use custom hook for prompt history navigation
 		const { handleHistoryNavigation, resetHistoryNavigation, resetOnInputChange } = usePromptHistory({
 			clineMessages,
@@ -1084,6 +1128,19 @@ export const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 								onScroll={() => updateHighlights()}
 							/>
 
+							{/* Show interim transcript overlay when listening */}
+							{isListening && interimTranscript && (
+								<div
+									className="absolute bottom-14 left-2 right-2 z-40 p-2 rounded-md bg-vscode-editor-background border border-vscode-input-border"
+									style={{
+										boxShadow: "0 2px 8px rgba(0, 0, 0, 0.2)",
+									}}>
+									<div className="text-vscode-input-foreground text-sm italic opacity-80">
+										{interimTranscript}
+									</div>
+								</div>
+							)}
+
 							<div className="absolute bottom-2 right-1 z-30 flex flex-col items-center gap-0">
 								<StandardTooltip content={t("chat:addImages")}>
 									<button
@@ -1133,6 +1190,36 @@ export const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
 										<WandSparkles className={cn("w-4 h-4", isEnhancingPrompt && "animate-spin")} />
 									</button>
 								</StandardTooltip>
+								{isSpeechRecognitionSupported && (
+									<StandardTooltip
+										content={isListening ? t("chat:stopListening") : t("chat:startListening")}>
+										<button
+											aria-label={
+												isListening ? t("chat:stopListening") : t("chat:startListening")
+											}
+											onClick={toggleListening}
+											className={cn(
+												"relative inline-flex items-center justify-center",
+												"bg-transparent border-none p-1.5",
+												"rounded-md min-w-[28px] min-h-[28px]",
+												"text-vscode-descriptionForeground hover:text-vscode-foreground",
+												"transition-all duration-150",
+												"cursor-pointer",
+												isListening
+													? "text-vscode-errorForeground opacity-100"
+													: "opacity-50 hover:opacity-100",
+												"hover:bg-[rgba(255,255,255,0.03)] hover:border-[rgba(255,255,255,0.15)]",
+												"focus:outline-none focus-visible:ring-1 focus-visible:ring-vscode-focusBorder",
+												"active:bg-[rgba(255,255,255,0.1)]",
+											)}>
+											{isListening ? (
+												<MicOff className="w-4 h-4 animate-pulse" />
+											) : (
+												<Mic className="w-4 h-4" />
+											)}
+										</button>
+									</StandardTooltip>
+								)}
 								{isEditMode && (
 									<StandardTooltip content={t("chat:cancel.title")}>
 										<button

webview-ui/src/components/chat/hooks/__tests__/useSpeechRecognition.spec.tsx

@@ -0,0 +1,426 @@
+import { renderHook, act } from "@testing-library/react"
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"
+import { useSpeechRecognition } from "../useSpeechRecognition"
+
+// Mock the TranslationContext
+vi.mock("@src/i18n/TranslationContext", () => ({
+	useAppTranslation: () => ({
+		t: (key: string) => key,
+	}),
+}))
+
+// Mock SpeechRecognition API
+class MockSpeechRecognition {
+	continuous = false
+	interimResults = false
+	maxAlternatives = 1
+	lang = ""
+	onstart: ((event: Event) => void) | null = null
+	onend: ((event: Event) => void) | null = null
+	onresult: ((event: any) => void) | null = null
+	onerror: ((event: any) => void) | null = null
+
+	start = vi.fn()
+	stop = vi.fn()
+	abort = vi.fn()
+}
+
+describe("useSpeechRecognition", () => {
+	let mockSpeechRecognition: MockSpeechRecognition
+
+	beforeEach(() => {
+		mockSpeechRecognition = new MockSpeechRecognition()
+
+		// Mock window.SpeechRecognition
+		Object.defineProperty(window, "SpeechRecognition", {
+			writable: true,
+			value: vi.fn().mockImplementation(() => mockSpeechRecognition),
+		})
+
+		Object.defineProperty(window, "webkitSpeechRecognition", {
+			writable: true,
+			value: vi.fn().mockImplementation(() => mockSpeechRecognition),
+		})
+	})
+
+	afterEach(() => {
+		vi.clearAllMocks()
+	})
+
+	it("should initialize with default values", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		expect(result.current.isListening).toBe(false)
+		expect(result.current.isSupported).toBe(true)
+		expect(result.current.transcript).toBe("")
+		expect(result.current.interimTranscript).toBe("")
+		expect(result.current.error).toBe(null)
+	})
+
+	it("should detect when speech recognition is not supported", () => {
+		// Remove SpeechRecognition API
+		Object.defineProperty(window, "SpeechRecognition", {
+			writable: true,
+			value: undefined,
+		})
+		Object.defineProperty(window, "webkitSpeechRecognition", {
+			writable: true,
+			value: undefined,
+		})
+
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		expect(result.current.isSupported).toBe(false)
+	})
+
+	it("should start listening when startListening is called", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		expect(mockSpeechRecognition.start).toHaveBeenCalled()
+	})
+
+	it("should stop listening when stopListening is called", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Start listening first
+		act(() => {
+			result.current.startListening()
+		})
+
+		// Simulate onstart event
+		act(() => {
+			mockSpeechRecognition.onstart?.(new Event("start"))
+		})
+
+		expect(result.current.isListening).toBe(true)
+
+		// Stop listening
+		act(() => {
+			result.current.stopListening()
+		})
+
+		expect(mockSpeechRecognition.stop).toHaveBeenCalled()
+	})
+
+	it("should toggle listening state", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Start listening
+		act(() => {
+			result.current.toggleListening()
+		})
+
+		expect(mockSpeechRecognition.start).toHaveBeenCalled()
+
+		// Simulate onstart event
+		act(() => {
+			mockSpeechRecognition.onstart?.(new Event("start"))
+		})
+
+		expect(result.current.isListening).toBe(true)
+
+		// Stop listening
+		act(() => {
+			result.current.toggleListening()
+		})
+
+		expect(mockSpeechRecognition.stop).toHaveBeenCalled()
+	})
+
+	it("should handle speech recognition results", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Start listening
+		act(() => {
+			result.current.startListening()
+		})
+
+		// Simulate speech recognition result
+		const mockEvent = {
+			resultIndex: 0,
+			results: [
+				{
+					isFinal: true,
+					0: { transcript: "Hello world", confidence: 0.9 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent)
+		})
+
+		expect(result.current.transcript).toBe("Hello world")
+		expect(result.current.interimTranscript).toBe("")
+	})
+
+	it("should handle interim results", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Start listening
+		act(() => {
+			result.current.startListening()
+		})
+
+		// Simulate interim result
+		const mockEvent = {
+			resultIndex: 0,
+			results: [
+				{
+					isFinal: false,
+					0: { transcript: "Hello", confidence: 0.8 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent)
+		})
+
+		expect(result.current.transcript).toBe("")
+		expect(result.current.interimTranscript).toBe("Hello")
+	})
+
+	it("should handle errors", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Start listening
+		act(() => {
+			result.current.startListening()
+		})
+
+		// Simulate error
+		const mockError = {
+			error: "network",
+			message: "Network error",
+		}
+
+		act(() => {
+			mockSpeechRecognition.onerror?.(mockError)
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.network")
+		expect(result.current.isListening).toBe(false)
+	})
+
+	it("should clear transcript", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		// Set some transcript
+		act(() => {
+			result.current.startListening()
+		})
+
+		const mockEvent = {
+			resultIndex: 0,
+			results: [
+				{
+					isFinal: true,
+					0: { transcript: "Test message", confidence: 0.9 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent)
+		})
+
+		expect(result.current.transcript).toBe("Test message")
+
+		// Clear transcript
+		act(() => {
+			result.current.clearTranscript()
+		})
+
+		expect(result.current.transcript).toBe("")
+		expect(result.current.interimTranscript).toBe("")
+		expect(result.current.error).toBe(null)
+	})
+
+	it("should handle no-speech error", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		const mockError = {
+			error: "no-speech",
+		}
+
+		act(() => {
+			mockSpeechRecognition.onerror?.(mockError)
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.noSpeech")
+	})
+
+	it("should handle audio-capture error", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		const mockError = {
+			error: "audio-capture",
+		}
+
+		act(() => {
+			mockSpeechRecognition.onerror?.(mockError)
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.audioCapture")
+	})
+
+	it("should handle not-allowed error", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		const mockError = {
+			error: "not-allowed",
+		}
+
+		act(() => {
+			mockSpeechRecognition.onerror?.(mockError)
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.notAllowed")
+	})
+
+	it("should handle generic error", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		const mockError = {
+			error: "unknown-error",
+		}
+
+		act(() => {
+			mockSpeechRecognition.onerror?.(mockError)
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.generic: unknown-error")
+	})
+
+	it("should set error when trying to start on unsupported browser", () => {
+		// Remove SpeechRecognition API
+		Object.defineProperty(window, "SpeechRecognition", {
+			writable: true,
+			value: undefined,
+		})
+		Object.defineProperty(window, "webkitSpeechRecognition", {
+			writable: true,
+			value: undefined,
+		})
+
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		expect(result.current.error).toBe("chat:voiceToText.errors.notSupported")
+	})
+
+	it("should handle multiple sequential transcripts", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		// First transcript
+		const mockEvent1 = {
+			resultIndex: 0,
+			results: [
+				{
+					isFinal: true,
+					0: { transcript: "First message", confidence: 0.9 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent1)
+		})
+
+		expect(result.current.transcript).toBe("First message")
+
+		// Second transcript
+		const mockEvent2 = {
+			resultIndex: 1,
+			results: [
+				{
+					isFinal: true,
+					0: { transcript: "First message", confidence: 0.9 },
+					length: 1,
+				},
+				{
+					isFinal: true,
+					0: { transcript: " Second message", confidence: 0.9 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent2)
+		})
+
+		expect(result.current.transcript).toBe("First message Second message")
+	})
+
+	it("should combine interim transcript with final on stop", () => {
+		const { result } = renderHook(() => useSpeechRecognition())
+
+		act(() => {
+			result.current.startListening()
+		})
+
+		// Simulate onstart
+		act(() => {
+			mockSpeechRecognition.onstart?.(new Event("start"))
+		})
+
+		// Add interim transcript
+		const mockEvent = {
+			resultIndex: 0,
+			results: [
+				{
+					isFinal: false,
+					0: { transcript: "Interim text", confidence: 0.8 },
+					length: 1,
+				},
+			],
+		}
+
+		act(() => {
+			mockSpeechRecognition.onresult?.(mockEvent)
+		})
+
+		expect(result.current.interimTranscript).toBe("Interim text")
+
+		// Stop listening
+		act(() => {
+			result.current.stopListening()
+		})
+
+		expect(result.current.transcript).toBe("Interim text")
+		expect(result.current.interimTranscript).toBe("")
+	})
+})

webview-ui/src/components/chat/hooks/useSpeechRecognition.ts

@@ -0,0 +1,210 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+import { useAppTranslation } from "@src/i18n/TranslationContext"
+
+// Extend the Window interface to include the Web Speech API
+interface IWindow extends Window {
+	webkitSpeechRecognition: any
+	SpeechRecognition: any
+}
+
+declare const window: IWindow
+
+// Type definitions for the Web Speech API
+interface SpeechRecognitionEvent extends Event {
+	results: SpeechRecognitionResultList
+	resultIndex: number
+}
+
+interface SpeechRecognitionErrorEvent extends Event {
+	error: string
+	message?: string
+}
+
+interface SpeechRecognitionResult {
+	isFinal: boolean
+	[index: number]: SpeechRecognitionAlternative
+}
+
+interface SpeechRecognitionAlternative {
+	transcript: string
+	confidence: number
+}
+
+interface SpeechRecognitionResultList {
+	length: number
+	item(index: number): SpeechRecognitionResult
+	[index: number]: SpeechRecognitionResult
+}
+
+export interface UseSpeechRecognitionReturn {
+	isListening: boolean
+	isSupported: boolean
+	transcript: string
+	interimTranscript: string
+	error: string | null
+	startListening: () => void
+	stopListening: () => void
+	toggleListening: () => void
+	clearTranscript: () => void
+}
+
+export const useSpeechRecognition = (): UseSpeechRecognitionReturn => {
+	const { t } = useAppTranslation()
+	const [isListening, setIsListening] = useState(false)
+	const [transcript, setTranscript] = useState("")
+	const [interimTranscript, setInterimTranscript] = useState("")
+	const [error, setError] = useState<string | null>(null)
+	const recognitionRef = useRef<any>(null)
+	const [isSupported, setIsSupported] = useState(false)
+
+	// Check for browser support
+	useEffect(() => {
+		if (typeof window !== "undefined") {
+			const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition
+			setIsSupported(!!SpeechRecognition)
+		}
+	}, [])
+
+	// Initialize speech recognition
+	useEffect(() => {
+		if (!isSupported) return
+
+		const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition
+		const recognition = new SpeechRecognition()
+
+		// Configure recognition
+		recognition.continuous = true
+		recognition.interimResults = true
+		recognition.maxAlternatives = 1
+
+		// Set language based on user settings (defaulting to browser language)
+		recognition.lang = navigator.language || "en-US"
+
+		// Handle results
+		recognition.onresult = (event: SpeechRecognitionEvent) => {
+			let interimText = ""
+			let finalText = ""
+
+			for (let i = event.resultIndex; i < event.results.length; i++) {
+				const result = event.results[i]
+				const transcriptText = result[0].transcript
+
+				if (result.isFinal) {
+					finalText += transcriptText
+				} else {
+					interimText += transcriptText
+				}
+			}
+
+			if (finalText) {
+				setTranscript((prev) => prev + finalText)
+				setInterimTranscript("")
+			} else {
+				setInterimTranscript(interimText)
+			}
+			setError(null)
+		}
+
+		// Handle errors
+		recognition.onerror = (event: SpeechRecognitionErrorEvent) => {
+			let errorMessage = ""
+
+			switch (event.error) {
+				case "no-speech":
+					errorMessage = t("chat:voiceToText.errors.noSpeech")
+					break
+				case "audio-capture":
+					errorMessage = t("chat:voiceToText.errors.audioCapture")
+					break
+				case "not-allowed":
+					errorMessage = t("chat:voiceToText.errors.notAllowed")
+					break
+				case "network":
+					errorMessage = t("chat:voiceToText.errors.network")
+					break
+				case "aborted":
+					errorMessage = t("chat:voiceToText.errors.aborted")
+					break
+				default:
+					errorMessage = t("chat:voiceToText.errors.generic") + `: ${event.error}`
+			}
+
+			setError(errorMessage)
+			setIsListening(false)
+		}
+
+		// Handle end event
+		recognition.onend = () => {
+			setIsListening(false)
+			setInterimTranscript("")
+		}
+
+		// Handle start event
+		recognition.onstart = () => {
+			setIsListening(true)
+			setError(null)
+		}
+
+		recognitionRef.current = recognition
+
+		// Cleanup
+		return () => {
+			if (recognitionRef.current) {
+				recognitionRef.current.stop()
+			}
+		}
+	}, [isSupported, t])
+
+	const startListening = useCallback(() => {
+		if (!isSupported) {
+			setError(t("chat:voiceToText.errors.notSupported"))
+			return
+		}
+
+		if (recognitionRef.current && !isListening) {
+			try {
+				recognitionRef.current.start()
+			} catch (_err) {
+				// Recognition is already started
+				console.warn("Speech recognition already started")
+			}
+		}
+	}, [isSupported, isListening, t])
+
+	const stopListening = useCallback(() => {
+		if (recognitionRef.current && isListening) {
+			recognitionRef.current.stop()
+			// Combine any interim transcript with the final transcript
+			if (interimTranscript) {
+				setTranscript((prev) => prev + interimTranscript)
+				setInterimTranscript("")
+			}
+		}
+	}, [isListening, interimTranscript])
+
+	const toggleListening = useCallback(() => {
+		if (isListening) {
+			stopListening()
+		} else {
+			startListening()
+		}
+	}, [isListening, startListening, stopListening])
+
+	const clearTranscript = useCallback(() => {
+		setTranscript("")
+		setInterimTranscript("")
+		setError(null)
+	}, [])
+
+	return {
+		isListening,
+		isSupported,
+		transcript,
+		interimTranscript,
+		error,
+		startListening,
+		stopListening,
+		toggleListening,
+		clearTranscript,
+	}
+}

webview-ui/src/i18n/locales/en/chat.json

@@ -421,5 +421,18 @@
 		"problems": "Problems",
 		"terminal": "Terminal",
 		"url": "Paste URL to fetch contents"
+	},
+	"startListening": "Start voice input",
+	"stopListening": "Stop voice input",
+	"voiceToText": {
+		"errors": {
+			"noSpeech": "No speech detected. Please try again.",
+			"audioCapture": "Microphone not found. Please check your microphone.",
+			"notAllowed": "Microphone permission denied. Please allow microphone access.",
+			"network": "Network error. Please check your connection.",
+			"aborted": "Speech recognition aborted.",
+			"generic": "Speech recognition error",
+			"notSupported": "Speech recognition is not supported in your browser."
+		}
 	}
 }