From 2394dcdd976f0324ed1639da2bc1aaad80fe2834 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Fri, 30 Jan 2026 19:49:21 +0000 Subject: [PATCH] fix(vscode-lm): use static vscodeLlmModels definitions for image support - Import vscodeLlmModels from @roo-code/types to check static definitions first - Use pattern matching only for unknown models not in static definitions - Add IMAGE_INCAPABLE_MODEL_PATTERNS for explicit non-vision models - Update IMAGE_CAPABLE_MODEL_PATTERNS to use RegExp for precise matching - Fix inconsistency where older models incorrectly reported supportsImages: true - Update tests to verify static definitions take precedence over pattern matching --- src/api/providers/__tests__/vscode-lm.spec.ts | 154 +++++++++++------- src/api/providers/vscode-lm.ts | 60 +++++-- 2 files changed, 143 insertions(+), 71 deletions(-) diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index a16bb1dd27..c93b72c5bc 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -1,5 +1,5 @@ import type { Mock } from "vitest" -import { checkModelSupportsImages, IMAGE_CAPABLE_MODEL_PREFIXES } from "../vscode-lm" +import { checkModelSupportsImages, IMAGE_CAPABLE_MODEL_PATTERNS, IMAGE_INCAPABLE_MODEL_PATTERNS } from "../vscode-lm" // Mocks must come first, before imports vi.mock("vscode", () => { @@ -540,55 +540,86 @@ describe("VsCodeLmHandler", () => { }) describe("checkModelSupportsImages", () => { - describe("OpenAI GPT models", () => { - it("should return true for all gpt-* models (GitHub Copilot)", () => { - // All GPT models in GitHub Copilot support images - expect(checkModelSupportsImages("gpt", "gpt-4o")).toBe(true) - expect(checkModelSupportsImages("gpt", "gpt-4.1")).toBe(true) - expect(checkModelSupportsImages("gpt", "gpt-5")).toBe(true) - expect(checkModelSupportsImages("gpt", "gpt-5.1")).toBe(true) - expect(checkModelSupportsImages("gpt", "gpt-5.2")).toBe(true) - expect(checkModelSupportsImages("gpt-mini", "gpt-5-mini")).toBe(true) - expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex")).toBe(true) - expect(checkModelSupportsImages("gpt-codex", "gpt-5.2-codex")).toBe(true) - expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex-max")).toBe(true) - expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex-mini")).toBe(true) + describe("static vscodeLlmModels lookup", () => { + it("should return supportsImages from static definitions when model family matches", () => { + // Models in vscodeLlmModels should return their static supportsImages value + expect(checkModelSupportsImages("gpt-3.5-turbo", "gpt-3.5-turbo")).toBe(false) + expect(checkModelSupportsImages("gpt-4", "gpt-4")).toBe(false) + expect(checkModelSupportsImages("gpt-4o-mini", "gpt-4o-mini")).toBe(false) + expect(checkModelSupportsImages("gpt-4o", "gpt-4o")).toBe(true) + expect(checkModelSupportsImages("gpt-4.1", "gpt-4.1")).toBe(true) + expect(checkModelSupportsImages("gpt-5", "gpt-5")).toBe(true) + expect(checkModelSupportsImages("gpt-5-mini", "gpt-5-mini")).toBe(true) + expect(checkModelSupportsImages("o1", "o1")).toBe(false) + expect(checkModelSupportsImages("o3-mini", "o3-mini")).toBe(false) + expect(checkModelSupportsImages("o4-mini", "o4-mini")).toBe(false) }) - it("should return true for o1 and o3 reasoning models", () => { - expect(checkModelSupportsImages("o1", "o1-preview")).toBe(true) - expect(checkModelSupportsImages("o1", "o1-mini")).toBe(true) - expect(checkModelSupportsImages("o3", "o3")).toBe(true) + it("should return supportsImages from static definitions for claude models", () => { + expect(checkModelSupportsImages("claude-3.5-sonnet", "claude-3.5-sonnet")).toBe(true) + expect(checkModelSupportsImages("claude-4-sonnet", "claude-4-sonnet")).toBe(true) + }) + + it("should return supportsImages from static definitions for gemini models", () => { + expect(checkModelSupportsImages("gemini-2.0-flash-001", "gemini-2.0-flash-001")).toBe(true) + expect(checkModelSupportsImages("gemini-2.5-pro", "gemini-2.5-pro")).toBe(true) }) }) - describe("Anthropic Claude models", () => { - it("should return true for all claude-* models (GitHub Copilot)", () => { - // All Claude models in GitHub Copilot support images - expect(checkModelSupportsImages("claude-haiku", "claude-haiku-4.5")).toBe(true) - expect(checkModelSupportsImages("claude-opus", "claude-opus-4.5")).toBe(true) - expect(checkModelSupportsImages("claude-sonnet", "claude-sonnet-4")).toBe(true) - expect(checkModelSupportsImages("claude-sonnet", "claude-sonnet-4.5")).toBe(true) + describe("pattern matching for unknown models", () => { + it("should return true for gpt-4o (but not gpt-4o-mini)", () => { + expect(checkModelSupportsImages("custom", "gpt-4o")).toBe(true) + expect(checkModelSupportsImages("custom", "gpt-4o-mini")).toBe(false) }) - }) - describe("Google Gemini models", () => { - it("should return true for all gemini-* models (GitHub Copilot)", () => { - // All Gemini models in GitHub Copilot support images - expect(checkModelSupportsImages("gemini-pro", "gemini-2.5-pro")).toBe(true) - expect(checkModelSupportsImages("gemini-flash", "gemini-3-flash-preview")).toBe(true) - expect(checkModelSupportsImages("gemini-pro", "gemini-3-pro-preview")).toBe(true) + it("should return true for gpt-4.x and higher versions", () => { + expect(checkModelSupportsImages("custom", "gpt-4.1-preview")).toBe(true) + expect(checkModelSupportsImages("custom", "gpt-4.2")).toBe(true) + }) + + it("should return true for gpt-5 and higher (unknown variants)", () => { + expect(checkModelSupportsImages("custom", "gpt-5-turbo")).toBe(true) + expect(checkModelSupportsImages("custom", "gpt-6")).toBe(true) + }) + + it("should return true for all claude-* models", () => { + expect(checkModelSupportsImages("custom", "claude-haiku-4.5")).toBe(true) + expect(checkModelSupportsImages("custom", "claude-opus-4.5")).toBe(true) + expect(checkModelSupportsImages("custom", "claude-sonnet-4")).toBe(true) + }) + + it("should return true for all gemini-* models", () => { + expect(checkModelSupportsImages("custom", "gemini-2.5-pro")).toBe(true) + expect(checkModelSupportsImages("custom", "gemini-3-flash-preview")).toBe(true) }) }) describe("non-vision models", () => { - it("should return false for grok models (text-only in GitHub Copilot)", () => { - // Grok is the only model family in GitHub Copilot that doesn't support images - expect(checkModelSupportsImages("grok", "grok-code-fast-1")).toBe(false) + it("should return false for gpt-3.5 models", () => { + expect(checkModelSupportsImages("custom", "gpt-3.5-turbo")).toBe(false) + expect(checkModelSupportsImages("custom", "gpt-3.5-turbo-16k")).toBe(false) }) - it("should return false for models with non-matching prefixes", () => { - // Models that don't start with gpt, claude, gemini, o1, or o3 + it("should return false for base gpt-4 and gpt-4-* variants", () => { + expect(checkModelSupportsImages("custom", "gpt-4")).toBe(false) + expect(checkModelSupportsImages("custom", "gpt-4-0125-preview")).toBe(false) + expect(checkModelSupportsImages("custom", "gpt-4-turbo")).toBe(false) + }) + + it("should return false for reasoning models (o1, o3-mini, o4-mini)", () => { + expect(checkModelSupportsImages("custom", "o1")).toBe(false) + expect(checkModelSupportsImages("custom", "o1-preview")).toBe(false) + expect(checkModelSupportsImages("custom", "o1-mini")).toBe(false) + expect(checkModelSupportsImages("custom", "o3-mini")).toBe(false) + expect(checkModelSupportsImages("custom", "o4-mini")).toBe(false) + }) + + it("should return false for grok models", () => { + expect(checkModelSupportsImages("custom", "grok-code-fast-1")).toBe(false) + expect(checkModelSupportsImages("custom", "grok-2")).toBe(false) + }) + + it("should return false for unknown model families", () => { expect(checkModelSupportsImages("mistral", "mistral-large")).toBe(false) expect(checkModelSupportsImages("llama", "llama-3-70b")).toBe(false) expect(checkModelSupportsImages("unknown", "some-random-model")).toBe(false) @@ -596,34 +627,45 @@ describe("checkModelSupportsImages", () => { }) describe("case insensitivity", () => { - it("should match regardless of case", () => { - expect(checkModelSupportsImages("GPT", "GPT-4O")).toBe(true) - expect(checkModelSupportsImages("CLAUDE", "CLAUDE-SONNET-4")).toBe(true) - expect(checkModelSupportsImages("GEMINI", "GEMINI-2.5-PRO")).toBe(true) + it("should match regardless of case for pattern matching", () => { + expect(checkModelSupportsImages("custom", "GPT-4O")).toBe(true) + expect(checkModelSupportsImages("custom", "CLAUDE-SONNET-4")).toBe(true) + expect(checkModelSupportsImages("custom", "GEMINI-2.5-PRO")).toBe(true) }) }) - describe("prefix matching", () => { - it("should only match IDs that start with known prefixes", () => { - // ID must START with the prefix, not just contain it - expect(checkModelSupportsImages("custom", "gpt-4o")).toBe(true) // ID starts with gpt - expect(checkModelSupportsImages("custom", "my-gpt-model")).toBe(false) // gpt not at start + describe("pattern matching edge cases", () => { + it("should only match IDs that start with known patterns", () => { + expect(checkModelSupportsImages("custom", "my-gpt-4o-model")).toBe(false) // gpt not at start expect(checkModelSupportsImages("custom", "not-claude-model")).toBe(false) // claude not at start }) }) }) -describe("IMAGE_CAPABLE_MODEL_PREFIXES", () => { - it("should export the model prefixes array", () => { - expect(Array.isArray(IMAGE_CAPABLE_MODEL_PREFIXES)).toBe(true) - expect(IMAGE_CAPABLE_MODEL_PREFIXES.length).toBeGreaterThan(0) +describe("IMAGE_CAPABLE_MODEL_PATTERNS", () => { + it("should export the model patterns array", () => { + expect(Array.isArray(IMAGE_CAPABLE_MODEL_PATTERNS)).toBe(true) + expect(IMAGE_CAPABLE_MODEL_PATTERNS.length).toBeGreaterThan(0) }) - it("should include key model prefixes", () => { - expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("gpt") - expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("claude") - expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("gemini") - expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("o1") - expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("o3") + it("should contain RegExp patterns for vision-capable models", () => { + // All patterns should be RegExp instances + IMAGE_CAPABLE_MODEL_PATTERNS.forEach((pattern) => { + expect(pattern).toBeInstanceOf(RegExp) + }) + }) +}) + +describe("IMAGE_INCAPABLE_MODEL_PATTERNS", () => { + it("should export the incapable model patterns array", () => { + expect(Array.isArray(IMAGE_INCAPABLE_MODEL_PATTERNS)).toBe(true) + expect(IMAGE_INCAPABLE_MODEL_PATTERNS.length).toBeGreaterThan(0) + }) + + it("should contain RegExp patterns for non-vision models", () => { + // All patterns should be RegExp instances + IMAGE_INCAPABLE_MODEL_PATTERNS.forEach((pattern) => { + expect(pattern).toBeInstanceOf(RegExp) + }) }) }) diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index 953a4fe84c..76f3a798ed 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -2,7 +2,7 @@ import { Anthropic } from "@anthropic-ai/sdk" import * as vscode from "vscode" import OpenAI from "openai" -import { type ModelInfo, openAiModelInfoSaneDefaults } from "@roo-code/types" +import { type ModelInfo, openAiModelInfoSaneDefaults, vscodeLlmModels } from "@roo-code/types" import type { ApiHandlerOptions } from "../../shared/api" import { SELECTOR_SEPARATOR, stringifyVsCodeLmModelSelector } from "../../shared/vsCodeSelectorUtils" @@ -591,33 +591,63 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan } /** - * Model ID prefixes that support image inputs via VS Code Language Model API. + * Model ID patterns that support image inputs via VS Code Language Model API. * These models support the LanguageModelDataPart.image() API introduced in VS Code 1.106+. * - * All GitHub Copilot models with these prefixes support images. - * Only grok-* models don't support images (text only). + * For models not in the static vscodeLlmModels definitions, we use pattern matching + * to determine image support. Only newer model versions support images. * * Source: https://models.dev/api.json (github-copilot provider models) */ -export const IMAGE_CAPABLE_MODEL_PREFIXES = [ - "gpt", // All GPT models (gpt-4o, gpt-4.1, gpt-5, gpt-5.1, gpt-5.2, gpt-5-mini, gpt-5.1-codex, etc.) - "claude", // All Claude models (claude-haiku-4.5, claude-opus-4.5, claude-sonnet-4, claude-sonnet-4.5) - "gemini", // All Gemini models (gemini-2.5-pro, gemini-3-flash-preview, gemini-3-pro-preview) - "o1", // OpenAI o1 reasoning models - "o3", // OpenAI o3 reasoning models +export const IMAGE_CAPABLE_MODEL_PATTERNS = [ + /^gpt-4o$/i, // GPT-4o (omni) supports images, but NOT gpt-4o-mini + /^gpt-4\.[1-9]/i, // GPT-4.1 and higher versions + /^gpt-[5-9]/i, // GPT-5 and higher (gpt-5, gpt-5-mini, gpt-5.1-codex, etc.) + /^claude-/i, // All Claude models support images + /^gemini-/i, // All Gemini models support images +] + +/** + * Model ID patterns that explicitly do NOT support images. + * These patterns are checked before IMAGE_CAPABLE_MODEL_PATTERNS. + */ +export const IMAGE_INCAPABLE_MODEL_PATTERNS = [ + /^gpt-3\.5/i, // GPT-3.5 models don't support images + /^gpt-4$/i, // Base GPT-4 doesn't support images + /^gpt-4-/i, // GPT-4 variants like gpt-4-0125-preview don't support images + /^gpt-4o-mini/i, // GPT-4o-mini doesn't support images + /^o[1-4]-?/i, // Reasoning models (o1, o3-mini, o4-mini) don't support images + /^grok-/i, // Grok models don't support images ] /** * Checks if a model supports image inputs based on its model ID. - * Uses prefix matching against known image-capable model families. + * First checks static vscodeLlmModels definitions for known models, + * then falls back to pattern matching for unknown models. * - * @param _family The model family (unused, kept for API compatibility) + * @param family The model family (used for lookup in static definitions) * @param id The model ID * @returns true if the model supports image inputs */ -export function checkModelSupportsImages(_family: string, id: string): boolean { - const idLower = id.toLowerCase() - return IMAGE_CAPABLE_MODEL_PREFIXES.some((prefix) => idLower.startsWith(prefix)) +export function checkModelSupportsImages(family: string, id: string): boolean { + // First, check if the model exists in static definitions by family or id + const familyInfo = vscodeLlmModels[family as keyof typeof vscodeLlmModels] + if (familyInfo) { + return familyInfo.supportsImages ?? false + } + + const idInfo = vscodeLlmModels[id as keyof typeof vscodeLlmModels] + if (idInfo) { + return idInfo.supportsImages ?? false + } + + // For unknown models, first check if it matches any incapable patterns + if (IMAGE_INCAPABLE_MODEL_PATTERNS.some((pattern) => pattern.test(id))) { + return false + } + + // Then check if it matches any capable patterns + return IMAGE_CAPABLE_MODEL_PATTERNS.some((pattern) => pattern.test(id)) } // Static blacklist of VS Code Language Model IDs that should be excluded from the model list