feat(llm): stop sending images for the rest of the run once the model rejects one

strix/agents/factory.py

@@ -18,6 +18,7 @@ from pydantic import ValidationError
 
 from strix.agents.prompt import render_system_prompt
 from strix.config import load_settings
+from strix.config.models import images_allowed
 from strix.tools.agents_graph.tools import (
     agent_finish,
     create_agent,
@@ -375,15 +376,16 @@ def _configure_filesystem_tools(
             )
 
 
-def _make_filesystem_configurator(
-    *, chat_completions: bool, strict_schemas: bool, supports_images: bool = True
-) -> Any:
+def _view_image_enabled(ctx: Any, _agent: Any) -> bool:
+    return images_allowed(ctx.context)
+
+
+def _make_filesystem_configurator(*, chat_completions: bool, strict_schemas: bool) -> Any:
     def configure(toolset: Any) -> None:
         _configure_filesystem_tools(
             toolset, chat_completions=chat_completions, strict_schemas=strict_schemas
         )
-        if not supports_images:
-            toolset.view_image.is_enabled = False
+        toolset.view_image.is_enabled = _view_image_enabled
 
     return configure
 
@@ -739,7 +741,6 @@ def build_strix_agent(
                 configure_tools=_make_filesystem_configurator(
                     chat_completions=chat_completions_tools,
                     strict_schemas=strict_tool_schemas,
-                    supports_images=supports_images,
                 ),
             ),
             Shell(

strix/config/models.py

@@ -7,6 +7,7 @@ import contextlib
 import inspect
 import logging
 import os
+import threading
 import time
 import uuid
 from collections.abc import AsyncGenerator
@@ -911,6 +912,22 @@ def model_supports_images(model_name: str) -> bool:
     return entry is None or bool(entry.get("supports_vision"))
 
 
+# Set once the model rejects an image; one process is one run, so this covers every agent.
+_IMAGES_REJECTED = threading.Event()
+
+
+def mark_images_rejected() -> None:
+    if not _IMAGES_REJECTED.is_set():
+        logger.warning("The model rejected image input; sending no images for the rest of the run")
+        _IMAGES_REJECTED.set()
+
+
+def images_allowed(context: object) -> bool:
+    """Whether a request may carry images: the model should take them and hasn't refused one."""
+    supported = context.get("supports_images", True) if isinstance(context, dict) else True
+    return bool(supported) and not _IMAGES_REJECTED.is_set()
+
+
 def _bare_openai_name(model_name: str) -> str:
     name = model_name.strip().lower()
     for prefix in ("litellm/", "any-llm/", "openai/"):

strix/core/execution.py

@@ -23,6 +23,7 @@ from openai import (
 )
 
 from strix.config import codex
+from strix.config.models import images_allowed, mark_images_rejected
 from strix.core.hooks import (
     BudgetExceededError,
     BudgetPausedError,
@@ -139,7 +140,7 @@ _TEXT_ONLY_IMAGE_TEXT = "[error: this model cannot view images; use `snapshot -i
 
 
 def _with_image_scrub(run_config: RunConfig, context: dict[str, Any]) -> RunConfig:
-    if context.get("supports_images", True):
+    if images_allowed(context):
         return run_config
     # Chain any filter already set; it sees the scrubbed input.
     inner = run_config.call_model_input_filter
@@ -848,7 +849,10 @@ async def _run_cycle(  # noqa: PLR0912, PLR0915
             await coordinator.trigger_budget_stop()
             raise
         except Exception as exc:
-            if image_strips < 3 and session is not None and _is_image_rejection(exc):
+            image_rejected = _is_image_rejection(exc)
+            if image_rejected:
+                mark_images_rejected()
+            if image_strips < 3 and session is not None and image_rejected:
                 try:
                     stripped = await strip_all_images_from_session(session)
                 except Exception:

strix/tools/load_skill/tool.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from agents import RunContextWrapper, function_tool
 
+from strix.config.models import images_allowed
 from strix.skills import load_skills, validate_requested_skills
 
 
@@ -28,8 +29,7 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str:
     err = validate_requested_skills(requested)
     if err:
         return f"load_skill: {err}"
-    context = ctx.context if isinstance(ctx.context, dict) else {}
-    contents = load_skills(requested, supports_images=context.get("supports_images", True))
+    contents = load_skills(requested, supports_images=images_allowed(ctx.context))
     if not contents:
         return "load_skill: no content loaded for requested skills."
     sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()]

strix/tools/mcp/agent_tools.py

@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any
 
 from agents import RunContextWrapper, function_tool
 
+from strix.config.models import images_allowed
 from strix.tools.mcp.client import _errored_tool_output
 from strix.tools.mcp.naming import namespaced_tool_name
 from strix.tools.mcp.registry import MCP_REGISTRY_CONTEXT_KEY, McpRegistry
@@ -288,11 +289,10 @@ async def call_mcp(
             "Call search_mcp_tools, then get_mcp_tool_schema."
         )
     session = await entry.ensure_connected()
-    context = ctx.context if isinstance(ctx.context, dict) else {}
     return await session.dispatch(
         tool,
         arguments or {},
         label=namespaced_tool_name(connection, tool),
         result_transform=entry.result_transform,
-        supports_images=context.get("supports_images", True),
+        supports_images=images_allowed(ctx.context),
     )

tests/test_agent_factory_shell.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import threading
 from types import SimpleNamespace
 from typing import Any, cast
 
@@ -10,7 +11,7 @@ import pytest
 from agents.tool import CustomTool, FunctionTool
 
 from strix.agents import factory
-from strix.config import load_settings
+from strix.config import load_settings, models
 
 
 def _capturing_exec_tool(captured: dict[str, str]) -> FunctionTool:
@@ -117,13 +118,16 @@ def test_function_tools_are_result_bounded() -> None:
     assert getattr(by_name["think"], "_strix_bounded", False) is True
 
 
-@pytest.mark.parametrize(("supports_images", "has_view_image"), [(True, True), (False, False)])
-def test_view_image_is_left_out_for_text_only_models(
-    supports_images: bool, has_view_image: bool
+def test_view_image_is_left_out_once_images_are_not_allowed(
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    agent = factory.build_strix_agent(is_root=True, supports_images=supports_images)
-    filesystem = agent.capabilities[0]
+    monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event())
+    filesystem = factory.build_strix_agent(is_root=True).capabilities[0]
     filesystem.bind(cast("Any", object()))
-
     view_image = next(tool for tool in filesystem.tools() if tool.name == "view_image")
-    assert cast("FunctionTool", view_image).is_enabled is has_view_image
+    is_enabled = cast("Any", cast("FunctionTool", view_image).is_enabled)
+
+    assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is True
+    assert is_enabled(SimpleNamespace(context={"supports_images": False}), None) is False
+    models.mark_images_rejected()
+    assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is False

tests/test_execution_transient_retry.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import Any, cast
+import threading
+from typing import Any
 
 import httpx
 import pytest
@@ -15,7 +16,7 @@ from openai import (
     RateLimitError,
 )
 
-from strix.config import codex
+from strix.config import codex, models
 from strix.core import execution
 from strix.core.agents import AgentCoordinator
 
@@ -129,7 +130,7 @@ async def _run_once(
         coordinator,
         "root",
         input_data="task",
-        run_config=cast("RunConfig", object()),
+        run_config=RunConfig(),
         context={},
         max_turns=5,
         session=session,
@@ -232,6 +233,7 @@ async def _run_with_image_session(
 async def test_run_cycle_strips_images_on_image_rejection(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
+    monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event())
     rejection = APIStatusError(
         "No endpoints found that support image input",
         response=httpx.Response(status_code=404, request=_request()),
@@ -243,3 +245,5 @@ async def test_run_cycle_strips_images_on_image_rejection(
     assert result is streams[1]
     assert attempts == 2
     assert strips == 1
+    # Every later request in the run, from any agent, has its images scrubbed.
+    assert execution._with_image_scrub(RunConfig(), {}).call_model_input_filter is not None