From 4d1803a72cf368baecff8e72952dd114e6ac7727 Mon Sep 17 00:00:00 2001 From: Ian Date: Tue, 6 Oct 2026 15:11:50 -0400 Subject: [PATCH] feat(llm): detect whether a model accepts image input Add model_supports_images(): OpenRouter models are checked against architecture.input_modalities from OpenRouter's public model list (fetched once per process), others against LiteLLM's supports_vision. Unknown models and lookup failures count as image-capable. Not wired in yet. --- strix/config/models.py | 31 +++++++++++++++++ tests/test_models.py | 78 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/strix/config/models.py b/strix/config/models.py index 4ce6dc86..5a143e70 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -10,6 +10,7 @@ import os import time import uuid from collections.abc import AsyncGenerator +from functools import cache from typing import TYPE_CHECKING, Any, cast from agents import ( @@ -905,6 +906,36 @@ def model_supports_reasoning(model_name: str) -> bool: return bool(entry and entry.get("supports_reasoning")) +_OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" + + +@cache +def _openrouter_input_modalities() -> dict[str, list[str]] | None: + import requests + + try: + response = requests.get(_OPENROUTER_MODELS_URL, timeout=5) + response.raise_for_status() + return { + model["id"].lower(): model["architecture"]["input_modalities"] + for model in response.json()["data"] + } + except Exception: # noqa: BLE001 + logger.warning("Could not fetch OpenRouter model modalities", exc_info=True) + return None + + +def model_supports_images(model_name: str) -> bool: + """Return whether the model accepts image input; unknown models are assumed to.""" + if is_openrouter_model(model_name): + slug = model_name.strip().lower().split("openrouter/", 1)[1] + modalities = (_openrouter_input_modalities() or {}).get(slug) + if modalities is not None: + return "image" in modalities + entry = _catalog_entry(model_name) + return bool(entry.get("supports_vision", True)) if entry else True + + def _bare_openai_name(model_name: str) -> str: name = model_name.strip().lower() for prefix in ("litellm/", "any-llm/", "openai/"): diff --git a/tests/test_models.py b/tests/test_models.py index 82cf8ac4..706450fb 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,8 +2,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any + import litellm import pytest +import requests from agents.extensions.models.litellm_model import LitellmModel from agents.model_settings import ModelSettings from agents.models import _openai_shared @@ -16,6 +19,7 @@ from strix.config.models import ( _NonStreamingModel, _TurnGuardModel, configure_sdk_model_defaults, + model_supports_images, request_timeout_extra_args, resolve_api_type, routes_through_litellm, @@ -26,6 +30,10 @@ from strix.config.settings import Settings from strix.llm.request_log import RequestLoggingModel +if TYPE_CHECKING: + from collections.abc import Iterator + + def test_request_timeout_extra_args_positive() -> None: assert request_timeout_extra_args(300) == {"timeout": 300} assert request_timeout_extra_args(10) == {"timeout": 10} @@ -202,3 +210,73 @@ def test_configure_sdk_api_route_follows_the_given_model( models.configure_sdk_api_route("my-private-model", settings) assert routes == ["responses", "chat_completions"] + + +class _FakeOpenRouterResponse: + def raise_for_status(self) -> None: + pass + + def json(self) -> dict[str, Any]: + return { + "data": [ + {"id": "z-ai/glm-5.3", "architecture": {"input_modalities": ["text"]}}, + {"id": "openai/gpt-5", "architecture": {"input_modalities": ["text", "image"]}}, + ] + } + + +@pytest.fixture +def openrouter_models(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[str]]: + calls: list[str] = [] + + def fake_get(url: str, **_kwargs: Any) -> _FakeOpenRouterResponse: + calls.append(url) + return _FakeOpenRouterResponse() + + monkeypatch.setattr(requests, "get", fake_get) + models._openrouter_input_modalities.cache_clear() + yield calls + models._openrouter_input_modalities.cache_clear() + + +def test_openrouter_image_support_follows_input_modalities(openrouter_models: list[str]) -> None: + assert model_supports_images("openrouter/z-ai/glm-5.3") is False + assert model_supports_images("litellm/openrouter/openai/gpt-5") is True + assert len(openrouter_models) == 1 + + +@pytest.mark.usefixtures("openrouter_models") +def test_openrouter_model_missing_from_listing_falls_back_to_catalog( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "openrouter/acme/text-only", {"supports_vision": False}) + assert model_supports_images("openrouter/acme/text-only") is False + assert model_supports_images("openrouter/acme/unknown") is True + + +def test_openrouter_fetch_failure_fails_open( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + def failing_get(*_args: Any, **_kwargs: Any) -> None: + raise requests.ConnectionError("offline") + + monkeypatch.setattr(requests, "get", failing_get) + models._openrouter_input_modalities.cache_clear() + try: + assert model_supports_images("openrouter/acme/unknown") is True + assert model_supports_images("openrouter/acme/other") is True + finally: + models._openrouter_input_modalities.cache_clear() + assert caplog.text.count("Could not fetch OpenRouter model modalities") == 1 + + +def test_non_openrouter_image_support_uses_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + def unexpected_get(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("non-OpenRouter models must not fetch the OpenRouter listing") + + monkeypatch.setattr(requests, "get", unexpected_get) + monkeypatch.setitem(litellm.model_cost, "acme-text", {"supports_vision": False}) + monkeypatch.setitem(litellm.model_cost, "acme-vision", {"supports_vision": True}) + assert model_supports_images("acme-text") is False + assert model_supports_images("openai/acme-vision") is True + assert model_supports_images("acme-unknown-model") is True