diff --git a/CHANGELOG.md b/CHANGELOG.md index bca24b66dd..4c96bab247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,48 @@ # Roo Code Changelog +## [3.36.2] - 2025-12-04 + +![3.36.2 Release - Dynamic API Settings](/releases/3.36.2-release.png) + +- Restrict GPT-5 tool set to apply_patch for improved compatibility (PR #9853 by @hannesrudolph) +- Add dynamic settings support for Roo models from API, allowing model-specific configurations to be fetched dynamically (PR #9852 by @hannesrudolph) +- Fix: Resolve Chutes provider model fetching issue (PR #9854 by @cte) + +## [3.36.1] - 2025-12-04 + +![3.36.1 Release - Message Management & Stability Improvements](/releases/3.36.1-release.png) + +- Add MessageManager layer for centralized history coordination, fixing message synchronization issues (PR #9842 by @hannesrudolph) +- Fix: Prevent cascading truncation loop by only truncating visible messages (PR #9844 by @hannesrudolph) +- Fix: Handle unknown/invalid native tool calls to prevent extension freeze (PR #9834 by @daniel-lxs) +- Always enable reasoning for models that require it (PR #9836 by @cte) +- ChatView: Smoother stick-to-bottom behavior during streaming (PR #8999 by @hannesrudolph) +- UX: Improved error messages and documentation links (PR #9777 by @brunobergher) +- Fix: Overly round follow-up question suggestions styling (PR #9829 by @brunobergher) +- Add symlink support for slash commands in .roo/commands folder (PR #9838 by @mrubens) +- Ignore input to the execa terminal process for safer command execution (PR #9827 by @mrubens) +- Be safer about large file reads (PR #9843 by @jr) +- Add gpt-5.1-codex-max model to OpenAI provider (PR #9848 by @hannesrudolph) +- Evals UI: Add filtering, bulk delete, tool consolidation, and run notes (PR #9837 by @hannesrudolph) +- Evals UI: Add multi-model launch and UI improvements (PR #9845 by @hannesrudolph) +- Web: New pricing page (PR #9821 by @brunobergher) + +## [3.36.0] - 2025-12-04 + +![3.36.0 Release - Rewind Kangaroo](/releases/3.36.0-release.png) + +- Fix: Restore context when rewinding after condense (#8295 by @hannesrudolph, PR #9665 by @hannesrudolph) +- Add reasoning_details support to Roo provider for enhanced model reasoning visibility (PR #9796 by @app/roomote) +- Default to native tools for all models in the Roo provider for improved performance (PR #9811 by @mrubens) +- Enable search_and_replace for Minimax models (PR #9780 by @mrubens) +- Fix: Resolve Vercel AI Gateway model fetching issues (PR #9791 by @cte) +- Fix: Apply conservative max tokens for Cerebras provider (PR #9804 by @sebastiand-cerebras) +- Fix: Remove omission detection logic to eliminate false positives (#9785 by @Michaelzag, PR #9787 by @app/roomote) +- Refactor: Remove deprecated insert_content tool (PR #9751 by @daniel-lxs) +- Chore: Hide parallel tool calls experiment and disable feature (PR #9798 by @hannesrudolph) +- Update next.js documentation site dependencies (PR #9799 by @jr) +- Fix: Correct download count display on homepage (PR #9807 by @mrubens) + ## [3.35.5] - 2025-12-03 - Feat: Add provider routing selection for OpenRouter embeddings (#9144 by @SannidhyaSah, PR #9693 by @SannidhyaSah) diff --git a/apps/web-evals/src/actions/runs.ts b/apps/web-evals/src/actions/runs.ts index a3fb3feccc..9d213547ce 100644 --- a/apps/web-evals/src/actions/runs.ts +++ b/apps/web-evals/src/actions/runs.ts @@ -13,6 +13,9 @@ import { exerciseLanguages, createRun as _createRun, deleteRun as _deleteRun, + updateRun as _updateRun, + getIncompleteRuns as _getIncompleteRuns, + deleteRunsByIds as _deleteRunsByIds, createTask, getExercisesForLanguage, } from "@roo-code/evals" @@ -20,6 +23,9 @@ import { import { CreateRun } from "@/lib/schemas" import { redisClient } from "@/lib/server/redis" +// Storage base path for eval logs +const EVALS_STORAGE_PATH = "/tmp/evals/runs" + const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals") export async function createRun({ suite, exercises = [], timeout, iterations = 1, ...values }: CreateRun) { @@ -214,3 +220,150 @@ export async function killRun(runId: number): Promise { errors, } } + +export type DeleteIncompleteRunsResult = { + success: boolean + deletedCount: number + deletedRunIds: number[] + storageErrors: string[] +} + +/** + * Delete all incomplete runs (runs without a taskMetricsId/final score). + * Removes both database records and storage folders. + */ +export async function deleteIncompleteRuns(): Promise { + const storageErrors: string[] = [] + + // Get all incomplete runs + const incompleteRuns = await _getIncompleteRuns() + const runIds = incompleteRuns.map((run) => run.id) + + if (runIds.length === 0) { + return { + success: true, + deletedCount: 0, + deletedRunIds: [], + storageErrors: [], + } + } + + // Delete storage folders for each run + for (const runId of runIds) { + const storagePath = path.join(EVALS_STORAGE_PATH, String(runId)) + try { + if (fs.existsSync(storagePath)) { + fs.rmSync(storagePath, { recursive: true, force: true }) + console.log(`Deleted storage folder: ${storagePath}`) + } + } catch (error) { + console.error(`Failed to delete storage folder ${storagePath}:`, error) + storageErrors.push(`Failed to delete storage for run ${runId}`) + } + + // Also try to clear Redis state for any potentially running incomplete runs + try { + const redis = await redisClient() + await redis.del(`heartbeat:${runId}`) + await redis.del(`runners:${runId}`) + } catch (error) { + // Non-critical error, just log it + console.error(`Failed to clear Redis state for run ${runId}:`, error) + } + } + + // Delete from database + await _deleteRunsByIds(runIds) + + revalidatePath("/runs") + + return { + success: true, + deletedCount: runIds.length, + deletedRunIds: runIds, + storageErrors, + } +} + +/** + * Get count of incomplete runs (for UI display) + */ +export async function getIncompleteRunsCount(): Promise { + const incompleteRuns = await _getIncompleteRuns() + return incompleteRuns.length +} + +/** + * Delete all runs older than 30 days. + * Removes both database records and storage folders. + */ +export async function deleteOldRuns(): Promise { + const storageErrors: string[] = [] + + // Get all runs older than 30 days + const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) + const { getRuns } = await import("@roo-code/evals") + const allRuns = await getRuns() + const oldRuns = allRuns.filter((run) => run.createdAt < thirtyDaysAgo) + const runIds = oldRuns.map((run) => run.id) + + if (runIds.length === 0) { + return { + success: true, + deletedCount: 0, + deletedRunIds: [], + storageErrors: [], + } + } + + // Delete storage folders for each run + for (const runId of runIds) { + const storagePath = path.join(EVALS_STORAGE_PATH, String(runId)) + try { + if (fs.existsSync(storagePath)) { + fs.rmSync(storagePath, { recursive: true, force: true }) + console.log(`Deleted storage folder: ${storagePath}`) + } + } catch (error) { + console.error(`Failed to delete storage folder ${storagePath}:`, error) + storageErrors.push(`Failed to delete storage for run ${runId}`) + } + + // Also try to clear Redis state + try { + const redis = await redisClient() + await redis.del(`heartbeat:${runId}`) + await redis.del(`runners:${runId}`) + } catch (error) { + // Non-critical error, just log it + console.error(`Failed to clear Redis state for run ${runId}:`, error) + } + } + + // Delete from database + await _deleteRunsByIds(runIds) + + revalidatePath("/runs") + + return { + success: true, + deletedCount: runIds.length, + deletedRunIds: runIds, + storageErrors, + } +} + +/** + * Update the description of a run. + */ +export async function updateRunDescription(runId: number, description: string | null): Promise<{ success: boolean }> { + try { + await _updateRun(runId, { description }) + revalidatePath("/runs") + revalidatePath(`/runs/${runId}`) + return { success: true } + } catch (error) { + console.error("Failed to update run description:", error) + return { success: false } + } +} diff --git a/apps/web-evals/src/app/runs/new/new-run.tsx b/apps/web-evals/src/app/runs/new/new-run.tsx index 561c3ceb27..be015ac8ca 100644 --- a/apps/web-evals/src/app/runs/new/new-run.tsx +++ b/apps/web-evals/src/app/runs/new/new-run.tsx @@ -7,7 +7,7 @@ import { useQuery } from "@tanstack/react-query" import { useForm, FormProvider } from "react-hook-form" import { zodResolver } from "@hookform/resolvers/zod" import { toast } from "sonner" -import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal, Info } from "lucide-react" +import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal, Info, Plus, Minus } from "lucide-react" import { globalSettingsSchema, @@ -16,7 +16,6 @@ import { getModelId, type ProviderSettings, type GlobalSettings, - type ReasoningEffort, } from "@roo-code/types" import { createRun } from "@/actions/runs" @@ -44,7 +43,6 @@ import { Button, Checkbox, FormControl, - FormDescription, FormField, FormItem, FormLabel, @@ -66,11 +64,6 @@ import { PopoverTrigger, Slider, Label, - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, Tooltip, TooltipContent, TooltipTrigger, @@ -84,21 +77,38 @@ type ImportedSettings = { currentApiConfigName: string } +// Type for a model selection entry +type ModelSelection = { + id: string + model: string + popoverOpen: boolean +} + +// Type for a config selection entry (for import mode) +type ConfigSelection = { + id: string + configName: string + popoverOpen: boolean +} + export function NewRun() { const router = useRouter() const [provider, setModelSource] = useState<"roo" | "openrouter" | "other">("other") - const [modelPopoverOpen, setModelPopoverOpen] = useState(false) const [useNativeToolProtocol, setUseNativeToolProtocol] = useState(true) - const [useMultipleNativeToolCalls, setUseMultipleNativeToolCalls] = useState(false) - const [reasoningEffort, setReasoningEffort] = useState("") const [commandExecutionTimeout, setCommandExecutionTimeout] = useState(20) const [terminalShellIntegrationTimeout, setTerminalShellIntegrationTimeout] = useState(30) // seconds - // State for imported settings with config selection + // State for multiple model selections + const [modelSelections, setModelSelections] = useState([ + { id: crypto.randomUUID(), model: "", popoverOpen: false }, + ]) + + // State for imported settings with multiple config selections const [importedSettings, setImportedSettings] = useState(null) - const [selectedConfigName, setSelectedConfigName] = useState("") - const [configPopoverOpen, setConfigPopoverOpen] = useState(false) + const [configSelections, setConfigSelections] = useState([ + { id: crypto.randomUUID(), configName: "", popoverOpen: false }, + ]) const openRouter = useOpenRouterModels() const rooCodeCloud = useRooCodeCloudModels() @@ -134,7 +144,7 @@ export function NewRun() { formState: { isSubmitting }, } = form - const [model, suite, settings] = watch(["model", "suite", "settings", "concurrency"]) + const [suite, settings] = watch(["suite", "settings", "concurrency"]) // Load settings from localStorage on mount useEffect(() => { @@ -250,6 +260,60 @@ export function NewRun() { [getExercisesForLanguage, selectedExercises], ) + // Add a new model selection + const addModelSelection = useCallback(() => { + setModelSelections((prev) => [...prev, { id: crypto.randomUUID(), model: "", popoverOpen: false }]) + }, []) + + // Remove a model selection + const removeModelSelection = useCallback((id: string) => { + setModelSelections((prev) => prev.filter((s) => s.id !== id)) + }, []) + + // Update a model selection + const updateModelSelection = useCallback( + (id: string, model: string) => { + setModelSelections((prev) => prev.map((s) => (s.id === id ? { ...s, model, popoverOpen: false } : s))) + // Also set the form model field for validation (use first non-empty model) + setValue("model", model) + }, + [setValue], + ) + + // Toggle popover for a model selection + const toggleModelPopover = useCallback((id: string, open: boolean) => { + setModelSelections((prev) => prev.map((s) => (s.id === id ? { ...s, popoverOpen: open } : s))) + }, []) + + // Add a new config selection + const addConfigSelection = useCallback(() => { + setConfigSelections((prev) => [...prev, { id: crypto.randomUUID(), configName: "", popoverOpen: false }]) + }, []) + + // Remove a config selection + const removeConfigSelection = useCallback((id: string) => { + setConfigSelections((prev) => prev.filter((s) => s.id !== id)) + }, []) + + // Update a config selection + const updateConfigSelection = useCallback( + (id: string, configName: string) => { + setConfigSelections((prev) => prev.map((s) => (s.id === id ? { ...s, configName, popoverOpen: false } : s))) + // Also update the form settings for the first config (for validation) + if (importedSettings) { + const providerSettings = importedSettings.apiConfigs[configName] ?? {} + setValue("model", getModelId(providerSettings) ?? "") + setValue("settings", { ...EVALS_SETTINGS, ...providerSettings, ...importedSettings.globalSettings }) + } + }, + [importedSettings, setValue], + ) + + // Toggle popover for a config selection + const toggleConfigPopover = useCallback((id: string, open: boolean) => { + setConfigSelections((prev) => prev.map((s) => (s.id === id ? { ...s, popoverOpen: open } : s))) + }, []) + const onSubmit = useCallback( async (values: CreateRun) => { try { @@ -259,74 +323,104 @@ export function NewRun() { return } - // Build experiments settings - const experimentsSettings = useMultipleNativeToolCalls - ? { experiments: { multipleNativeToolCalls: true } } - : {} + // Determine which selections to use based on provider + const selectionsToLaunch: { model: string; configName?: string }[] = [] - if (provider === "openrouter") { - values.settings = { - ...(values.settings || {}), - apiProvider: "openrouter", - openRouterModelId: model, - toolProtocol: useNativeToolProtocol ? "native" : "xml", - commandExecutionTimeout, - terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, // Convert to ms - ...experimentsSettings, + if (provider === "other") { + // For import mode, use config selections + for (const config of configSelections) { + if (config.configName) { + selectionsToLaunch.push({ model: "", configName: config.configName }) + } } - } else if (provider === "roo") { - values.settings = { - ...(values.settings || {}), - apiProvider: "roo", - apiModelId: model, - toolProtocol: useNativeToolProtocol ? "native" : "xml", - commandExecutionTimeout, - terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, // Convert to ms - ...experimentsSettings, - ...(reasoningEffort - ? { - enableReasoningEffort: true, - reasoningEffort: reasoningEffort as ReasoningEffort, - } - : {}), - } - } else if (provider === "other" && values.settings) { - // For imported settings, merge in experiments and tool protocol - values.settings = { - ...values.settings, - toolProtocol: useNativeToolProtocol ? "native" : "xml", - commandExecutionTimeout, - terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, // Convert to ms - ...experimentsSettings, + } else { + // For openrouter/roo, use model selections + for (const selection of modelSelections) { + if (selection.model) { + selectionsToLaunch.push({ model: selection.model }) + } } } - const { id } = await createRun(values) - router.push(`/runs/${id}`) + if (selectionsToLaunch.length === 0) { + toast.error("Please select at least one model or config") + return + } + + // Show launching toast + const totalRuns = selectionsToLaunch.length + toast.info(totalRuns > 1 ? `Launching ${totalRuns} runs (every 20 seconds)...` : "Launching run...") + + // Launch runs with 20-second delay between each + for (let i = 0; i < selectionsToLaunch.length; i++) { + const selection = selectionsToLaunch[i]! + + // Wait 20 seconds between runs (except for the first one) + if (i > 0) { + await new Promise((resolve) => setTimeout(resolve, 20000)) + } + + const runValues = { ...values } + + if (provider === "openrouter") { + runValues.model = selection.model + runValues.settings = { + ...(runValues.settings || {}), + apiProvider: "openrouter", + openRouterModelId: selection.model, + toolProtocol: useNativeToolProtocol ? "native" : "xml", + commandExecutionTimeout, + terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, + } + } else if (provider === "roo") { + runValues.model = selection.model + runValues.settings = { + ...(runValues.settings || {}), + apiProvider: "roo", + apiModelId: selection.model, + toolProtocol: useNativeToolProtocol ? "native" : "xml", + commandExecutionTimeout, + terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, + } + } else if (provider === "other" && selection.configName && importedSettings) { + const providerSettings = importedSettings.apiConfigs[selection.configName] ?? {} + runValues.model = getModelId(providerSettings) ?? "" + runValues.settings = { + ...EVALS_SETTINGS, + ...providerSettings, + ...importedSettings.globalSettings, + toolProtocol: useNativeToolProtocol ? "native" : "xml", + commandExecutionTimeout, + terminalShellIntegrationTimeout: terminalShellIntegrationTimeout * 1000, + } + } + + try { + await createRun(runValues) + toast.success(`Run ${i + 1}/${totalRuns} launched`) + } catch (e) { + toast.error(`Run ${i + 1} failed: ${e instanceof Error ? e.message : "Unknown error"}`) + } + } + + // Navigate back to main evals UI + router.push("/") } catch (e) { toast.error(e instanceof Error ? e.message : "An unknown error occurred.") } }, [ provider, - model, + modelSelections, + configSelections, + importedSettings, router, useNativeToolProtocol, - useMultipleNativeToolCalls, - reasoningEffort, commandExecutionTimeout, terminalShellIntegrationTimeout, ], ) - const onSelectModel = useCallback( - (model: string) => { - setValue("model", model) - setModelPopoverOpen(false) - }, - [setValue, setModelPopoverOpen], - ) - const onImportSettings = useCallback( async (event: React.ChangeEvent) => { const file = event.target.files?.[0] @@ -355,9 +449,9 @@ export function NewRun() { currentApiConfigName: providerProfiles.currentApiConfigName, }) - // Default to the current config + // Default to the current config for the first selection const defaultConfigName = providerProfiles.currentApiConfigName - setSelectedConfigName(defaultConfigName) + setConfigSelections([{ id: crypto.randomUUID(), configName: defaultConfigName, popoverOpen: false }]) // Apply the default config const providerSettings = providerProfiles.apiConfigs[defaultConfigName] ?? {} @@ -373,22 +467,6 @@ export function NewRun() { [clearErrors, setValue], ) - const onSelectConfig = useCallback( - (configName: string) => { - if (!importedSettings) { - return - } - - setSelectedConfigName(configName) - setConfigPopoverOpen(false) - - const providerSettings = importedSettings.apiConfigs[configName] ?? {} - setValue("model", getModelId(providerSettings) ?? "") - setValue("settings", { ...EVALS_SETTINGS, ...providerSettings, ...importedSettings.globalSettings }) - }, - [importedSettings, setValue], - ) - return ( <> @@ -428,59 +506,91 @@ export function NewRun() { onChange={onImportSettings} /> - {importedSettings && Object.keys(importedSettings.apiConfigs).length > 1 && ( -
- - - - - - - - - - No config found. - - {Object.keys(importedSettings.apiConfigs).map( - (configName) => ( - - {configName} - {configName === - importedSettings.currentApiConfigName && ( - - (default) - - )} - 0 && ( +
+ + {configSelections.map((selection, index) => ( +
+ + toggleConfigPopover(selection.id, open) + }> + + + + + + + + No config found. + + {Object.keys( + importedSettings.apiConfigs, + ).map((configName) => ( + + updateConfigSelection( + selection.id, + configName, + ) + }> + {configName} + {configName === + importedSettings.currentApiConfigName && ( + + (default) + )} - /> - - ), - )} - - - - - + + + ))} + + + + + + {index === configSelections.length - 1 ? ( + + ) : ( + + )} +
+ ))}
)} @@ -501,18 +611,6 @@ export function NewRun() { /> Use Native Tool Calls -
@@ -522,110 +620,103 @@ export function NewRun() { ) : ( <> - - - - - - - + {modelSelections.map((selection, index) => ( +
+ toggleModelPopover(selection.id, open)}> + + + + + + + + No model found. + + {models?.map(({ id, name }) => ( + + updateModelSelection( + selection.id, + id, + ) + }> + {name} + + + ))} + + + + + + {index === modelSelections.length - 1 ? ( + + ) : ( + + )} +
+ ))} + + +
+ +
+
)} + {consolidatedToolColumns.length > 0 && ( + + {taskMetrics?.toolUsage ? ( + (() => { + // Calculate aggregated stats for consolidated tools + let totalAttempts = 0 + let totalFailures = 0 + const breakdown: Array<{ tool: string; attempts: number; rate: string }> = [] + + for (const toolName of consolidatedToolColumns) { + const usage = taskMetrics.toolUsage[toolName as ToolName] + if (usage) { + totalAttempts += usage.attempts + totalFailures += usage.failures + const rate = + usage.attempts > 0 + ? `${Math.round(((usage.attempts - usage.failures) / usage.attempts) * 100)}%` + : "0%" + breakdown.push({ tool: toolName, attempts: usage.attempts, rate }) + } + } + + const consolidatedRate = + totalAttempts > 0 ? ((totalAttempts - totalFailures) / totalAttempts) * 100 : 100 + const rateColor = + consolidatedRate === 100 + ? "text-muted-foreground" + : consolidatedRate >= 80 + ? "text-yellow-500" + : "text-red-500" + + return totalAttempts > 0 ? ( + + +
+ {totalAttempts} + {Math.round(consolidatedRate)}% +
+ + +
+
Consolidated Tools:
+ {breakdown.map(({ tool, attempts, rate }) => ( +
+ {tool}: + + {attempts} ({rate}) + +
+ ))} +
+ + + ) : ( + - + ) + })() + ) : ( + - + )} + + )} {toolColumns.map((toolName) => { const usage = taskMetrics?.toolUsage?.[toolName] const successRate = @@ -166,80 +258,107 @@ export function Run({ run, taskMetrics, toolColumns }: RunProps) { {taskMetrics && formatCurrency(taskMetrics.cost)} {taskMetrics && formatDuration(taskMetrics.duration)} e.stopPropagation()}> - - - - - +
+ {/* Note Icon */} + + + + + + {hasDescription ? ( +
{run.description}
+ ) : ( +
No description. Click to add one.
+ )} + + + + {/* More Actions Menu */} + + + + + +
+ +
View Tasks
+
+ + + {run.settings && ( + setShowSettings(true)}> +
+ +
View Settings
+
+ + )} + {run.taskMetricsId && ( + copyRun()} disabled={isPending || copied}> +
+ {isPending ? ( + <> + + Copying... + + ) : copied ? ( + <> + + Copied! + + ) : ( + <> + + Copy to Production + + )} +
+ + )} + {run.failed > 0 && ( + +
+ {isExportingLogs ? ( + <> + + Exporting... + + ) : ( + <> + + Export Failed Logs + + )} +
+ + )} + { + setDeleteRunId(run.id) + setTimeout(() => continueRef.current?.focus(), 0) + }}>
- -
View Tasks
-
- - - {run.settings && ( - setShowSettings(true)}> -
- -
View Settings
+ +
Delete
 - )} - {run.taskMetricsId && ( - copyRun()} disabled={isPending || copied}> -
- {isPending ? ( - <> - - Copying... - - ) : copied ? ( - <> - - Copied! - - ) : ( - <> - - Copy to Production - - )} -
- - )} - {run.failed > 0 && ( - -
- {isExportingLogs ? ( - <> - - Exporting... - - ) : ( - <> - - Export Failed Logs - - )} -
- - )} - { - setDeleteRunId(run.id) - setTimeout(() => continueRef.current?.focus(), 0) - }}> -
- -
Delete
-
- - - + + +
setDeleteRunId(undefined)}> @@ -268,6 +387,39 @@ export function Run({ run, taskMetrics, toolColumns }: RunProps) { + + {/* Notes/Description Dialog */} + + + + Run Description + +
+