fix(models): frontier model check matches the model name only, never the provider route (#1280)

Co-authored-by: Ahmed Allam <ahmed39652003@gmail.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-06 11:09:24 -07:00 committed by GitHub
parent ff5c8cc8e4
commit 52b1923347
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 54 additions and 58 deletions

View file

@ -593,17 +593,27 @@ RECOMMENDED_MODEL_NAMES = (
_RECOMMENDED_MODEL_NAME_SET = frozenset(name.lower() for name in RECOMMENDED_MODEL_NAMES)
FRONTIER_MODEL_FAMILIES = (
(("azure", "azure_ai", "bedrock_mantle", "chatgpt", "openai"), ("gpt-5",)),
(
("anthropic", "azure_ai", "bedrock", "claude", "databricks", "snowflake", "vertex_ai"),
("claude-fable-5", "claude-opus-5", "claude-opus-4", "claude-sonnet-5", "claude-sonnet-4"),
),
(("google", "gemini", "vertex_ai"), ("gemini-3",)),
(("deepseek",), ("deepseek-v4", "deepseek-r1", "deepseek-reasoner")),
(("alibaba", "dashscope", "qwen"), ("qwen3.8", "qwen3.7", "qwen3-max")),
(("moonshot", "moonshotai", "kimi"), ("kimi-k3", "kimi-k2.7", "kimi-k2.6")),
(("zai", "z-ai", "zai-org", "zhipuai"), ("glm-5.3", "glm-5.2")),
# Matched against the bare model name only: the route (``openai/``, ``openrouter/``,
# a local gateway, ...) says nothing about the model's quality.
FRONTIER_MODEL_PREFIXES = (
"gpt-5",
"claude-fable-5",
"claude-opus-5",
"claude-opus-4",
"claude-sonnet-5",
"claude-sonnet-4",
"gemini-3",
"deepseek-v4",
"deepseek-r1",
"deepseek-reasoner",
"qwen3.8",
"qwen3.7",
"qwen3-max",
"kimi-k3",
"kimi-k2.7",
"kimi-k2.6",
"glm-5.3",
"glm-5.2",
)
@ -837,11 +847,8 @@ def is_recommended_or_frontier_model(model_name: str) -> bool:
return False
if name in _RECOMMENDED_MODEL_NAME_SET:
return True
provider_name, bare_model_name = _split_model_provider(name)
return any(
_matches_frontier_family(provider_name, bare_model_name, provider_markers, prefixes)
for provider_markers, prefixes in FRONTIER_MODEL_FAMILIES
)
bare_model_name = name.rsplit("/", 1)[-1]
return _matches_model_prefix(bare_model_name, FRONTIER_MODEL_PREFIXES)
def _normalized_model_name(model_name: str) -> str:
@ -853,28 +860,6 @@ def _normalized_model_name(model_name: str) -> str:
return name
def _split_model_provider(model_name: str) -> tuple[str | None, str]:
if "/" not in model_name:
return None, model_name
provider_name, bare_model_name = model_name.rsplit("/", 1)
return provider_name, bare_model_name
def _matches_frontier_family(
provider_name: str | None,
model_name: str,
provider_markers: tuple[str, ...],
model_prefixes: tuple[str, ...],
) -> bool:
if not _matches_model_prefix(model_name, model_prefixes):
return False
if provider_name is None:
return True
return _contains_provider_marker(
provider_name, provider_markers, split_compound_names=True
) or _contains_provider_marker(model_name, provider_markers)
def _matches_model_prefix(model_name: str, model_prefixes: tuple[str, ...]) -> bool:
return any(
candidate.startswith(prefix)
@ -892,16 +877,6 @@ def _model_name_candidates(model_name: str) -> tuple[str, ...]:
return (model_name, *suffixes)
def _contains_provider_marker(
value: str, provider_markers: tuple[str, ...], *, split_compound_names: bool = False
) -> bool:
parts = set(value.replace(".", "/").split("/"))
if split_compound_names:
for separator in ("_", "-"):
parts.update(piece for part in tuple(parts) for piece in part.split(separator))
return any(marker in parts for marker in provider_markers)
def is_known_openai_bare_model(model_name: str) -> bool:
import litellm

View file

@ -192,7 +192,7 @@ class TuiController:
model_warning = ""
if model and not is_recommended_or_frontier_model(model):
model_warning = (
f"{model} is not a recommended frontier model; pentest quality could be degraded"
f"{model} is not a recommended frontier model. Pentest quality could be degraded."
)
state = {
"setup_mode": self.setup_mode,

View file

@ -370,6 +370,17 @@ func TestStartedSnapshotTransitionsToLiveView(t *testing.T) {
}
}
func TestSplashModelWarningRendersTheBackendSentenceOnce(t *testing.T) {
warning := "openai/glm-5.3 is not a recommended frontier model. Pentest quality could be degraded."
got := ansi.Strip(splashModelWarning("openai/glm-5.3", warning))
if got != "⚠ "+warning {
t.Fatalf("splash warning = %q, want %q", got, "⚠ "+warning)
}
if got := ansi.Strip(splashModelWarning("other/model", warning)); got != "⚠ "+warning {
t.Fatalf("splash warning with unrelated model = %q", got)
}
}
func TestSetupStartScreenFitsNarrowTerminal(t *testing.T) {
model := New(nil)
model.width, model.height = 40, 18

View file

@ -411,7 +411,7 @@ func (m Model) splashView() string {
welcome + "\n" + version + "\n" + tagline + "\n\n" +
start.String() + "\n\n" + url
if warn := m.snapshot.ModelWarning; warn != "" {
content += "\n\n" + splashModelWarning(warn)
content += "\n\n" + splashModelWarning(m.snapshot.Model, warn)
}
panel := lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(green).Padding(1, 6).Align(lipgloss.Center).Render(content)
// #splash_screen background is solid black.
@ -419,12 +419,16 @@ func (m Model) splashView() string {
lipgloss.WithWhitespaceBackground(black))
}
// splashModelWarning ports SplashScreen._build_model_warning_text.
func splashModelWarning(model string) string {
// splashModelWarning renders the backend's full warning sentence, with the
// model name highlighted when the sentence leads with it.
func splashModelWarning(model, warning string) string {
yellow := lipgloss.Color("#eab308")
return lipgloss.NewStyle().Bold(true).Foreground(yellow).Render("⚠ ") +
lipgloss.NewStyle().Bold(true).Foreground(render.Cyan).Render(model) +
lipgloss.NewStyle().Foreground(yellow).Render(" is not a recommended frontier model - pentest quality could be degraded")
out := lipgloss.NewStyle().Bold(true).Foreground(yellow).Render("⚠ ")
if model != "" && strings.HasPrefix(warning, model) {
out += lipgloss.NewStyle().Bold(true).Foreground(render.Cyan).Render(model)
warning = strings.TrimPrefix(warning, model)
}
return out + lipgloss.NewStyle().Foreground(yellow).Render(warning)
}
// chatPaneKey identifies everything the bordered trace depends on.

View file

@ -79,6 +79,14 @@ def test_recommended_models_are_matched_case_insensitively() -> None:
"zai/glm-5.3-flash",
"openrouter/z-ai/glm-5.3",
"novita/zai-org/glm-5.2",
"openai/glm-5.3",
"openai/zai-org/glm-5.3",
"hosted_vllm/glm-5.3",
"openai/claude-opus-4-8",
"openai/deepseek-v4-pro",
"custom-ollama/gpt-5-mini-local",
"custom-provider/claude-opus-4-local",
"custom-provider/glm-5.3-local",
],
)
def test_frontier_model_families_are_accepted(model_name: str) -> None:
@ -93,15 +101,13 @@ def test_frontier_model_families_are_accepted(model_name: str) -> None:
"anthropic/claude-3-5-sonnet-latest",
"ollama/llama3.1",
"deepseek/deepseek-chat",
"custom-ollama/gpt-5-mini-local",
"custom-provider/claude-opus-4-local",
"xai/grok-4.5",
"openrouter/x-ai/grok-4",
"mistral/mistral-medium-3-5",
"mistral/magistral-medium-latest",
"zai/glm-4.7",
"openai/glm-4.7",
"openrouter/z-ai/glm-5",
"custom-provider/glm-5.3-local",
],
)
def test_non_frontier_models_are_rejected(model_name: str) -> None: