diff --git a/strix/agents/factory.py b/strix/agents/factory.py index 3b9dd616..9b5ec415 100644 --- a/strix/agents/factory.py +++ b/strix/agents/factory.py @@ -18,6 +18,7 @@ from pydantic import ValidationError from strix.agents.prompt import render_system_prompt from strix.config import load_settings +from strix.config.models import images_allowed from strix.tools.agents_graph.tools import ( agent_finish, create_agent, @@ -375,15 +376,16 @@ def _configure_filesystem_tools( ) -def _make_filesystem_configurator( - *, chat_completions: bool, strict_schemas: bool, supports_images: bool = True -) -> Any: +def _view_image_enabled(ctx: Any, _agent: Any) -> bool: + return images_allowed(ctx.context) + + +def _make_filesystem_configurator(*, chat_completions: bool, strict_schemas: bool) -> Any: def configure(toolset: Any) -> None: _configure_filesystem_tools( toolset, chat_completions=chat_completions, strict_schemas=strict_schemas ) - if not supports_images: - toolset.view_image.is_enabled = False + toolset.view_image.is_enabled = _view_image_enabled return configure @@ -739,7 +741,6 @@ def build_strix_agent( configure_tools=_make_filesystem_configurator( chat_completions=chat_completions_tools, strict_schemas=strict_tool_schemas, - supports_images=supports_images, ), ), Shell( diff --git a/strix/config/models.py b/strix/config/models.py index b2d6f9d0..a5f592fc 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -7,6 +7,7 @@ import contextlib import inspect import logging import os +import threading import time import uuid from collections.abc import AsyncGenerator @@ -911,6 +912,22 @@ def model_supports_images(model_name: str) -> bool: return entry is None or bool(entry.get("supports_vision")) +# Set once the model rejects an image; one process is one run, so this covers every agent. +_IMAGES_REJECTED = threading.Event() + + +def mark_images_rejected() -> None: + if not _IMAGES_REJECTED.is_set(): + logger.warning("The model rejected image input; sending no images for the rest of the run") + _IMAGES_REJECTED.set() + + +def images_allowed(context: object) -> bool: + """Whether a request may carry images: the model should take them and hasn't refused one.""" + supported = context.get("supports_images", True) if isinstance(context, dict) else True + return bool(supported) and not _IMAGES_REJECTED.is_set() + + def _bare_openai_name(model_name: str) -> str: name = model_name.strip().lower() for prefix in ("litellm/", "any-llm/", "openai/"): diff --git a/strix/core/execution.py b/strix/core/execution.py index a0b81490..458c1bdc 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -23,6 +23,7 @@ from openai import ( ) from strix.config import codex +from strix.config.models import images_allowed, mark_images_rejected from strix.core.hooks import ( BudgetExceededError, BudgetPausedError, @@ -139,7 +140,7 @@ _TEXT_ONLY_IMAGE_TEXT = "[error: this model cannot view images; use `snapshot -i def _with_image_scrub(run_config: RunConfig, context: dict[str, Any]) -> RunConfig: - if context.get("supports_images", True): + if images_allowed(context): return run_config # Chain any filter already set; it sees the scrubbed input. inner = run_config.call_model_input_filter @@ -848,7 +849,10 @@ async def _run_cycle( # noqa: PLR0912, PLR0915 await coordinator.trigger_budget_stop() raise except Exception as exc: - if image_strips < 3 and session is not None and _is_image_rejection(exc): + image_rejected = _is_image_rejection(exc) + if image_rejected: + mark_images_rejected() + if image_strips < 3 and session is not None and image_rejected: try: stripped = await strip_all_images_from_session(session) except Exception: diff --git a/strix/tools/load_skill/tool.py b/strix/tools/load_skill/tool.py index 83f70f8d..5e9388d3 100644 --- a/strix/tools/load_skill/tool.py +++ b/strix/tools/load_skill/tool.py @@ -4,6 +4,7 @@ from __future__ import annotations from agents import RunContextWrapper, function_tool +from strix.config.models import images_allowed from strix.skills import load_skills, validate_requested_skills @@ -28,8 +29,7 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str: err = validate_requested_skills(requested) if err: return f"load_skill: {err}" - context = ctx.context if isinstance(ctx.context, dict) else {} - contents = load_skills(requested, supports_images=context.get("supports_images", True)) + contents = load_skills(requested, supports_images=images_allowed(ctx.context)) if not contents: return "load_skill: no content loaded for requested skills." sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()] diff --git a/strix/tools/mcp/agent_tools.py b/strix/tools/mcp/agent_tools.py index 7fa2c392..8cdf055e 100644 --- a/strix/tools/mcp/agent_tools.py +++ b/strix/tools/mcp/agent_tools.py @@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any from agents import RunContextWrapper, function_tool +from strix.config.models import images_allowed from strix.tools.mcp.client import _errored_tool_output from strix.tools.mcp.naming import namespaced_tool_name from strix.tools.mcp.registry import MCP_REGISTRY_CONTEXT_KEY, McpRegistry @@ -288,11 +289,10 @@ async def call_mcp( "Call search_mcp_tools, then get_mcp_tool_schema." ) session = await entry.ensure_connected() - context = ctx.context if isinstance(ctx.context, dict) else {} return await session.dispatch( tool, arguments or {}, label=namespaced_tool_name(connection, tool), result_transform=entry.result_transform, - supports_images=context.get("supports_images", True), + supports_images=images_allowed(ctx.context), ) diff --git a/tests/test_agent_factory_shell.py b/tests/test_agent_factory_shell.py index 0109bf52..e090c053 100644 --- a/tests/test_agent_factory_shell.py +++ b/tests/test_agent_factory_shell.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import threading from types import SimpleNamespace from typing import Any, cast @@ -10,7 +11,7 @@ import pytest from agents.tool import CustomTool, FunctionTool from strix.agents import factory -from strix.config import load_settings +from strix.config import load_settings, models def _capturing_exec_tool(captured: dict[str, str]) -> FunctionTool: @@ -117,13 +118,16 @@ def test_function_tools_are_result_bounded() -> None: assert getattr(by_name["think"], "_strix_bounded", False) is True -@pytest.mark.parametrize(("supports_images", "has_view_image"), [(True, True), (False, False)]) -def test_view_image_is_left_out_for_text_only_models( - supports_images: bool, has_view_image: bool +def test_view_image_is_left_out_once_images_are_not_allowed( + monkeypatch: pytest.MonkeyPatch, ) -> None: - agent = factory.build_strix_agent(is_root=True, supports_images=supports_images) - filesystem = agent.capabilities[0] + monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event()) + filesystem = factory.build_strix_agent(is_root=True).capabilities[0] filesystem.bind(cast("Any", object())) - view_image = next(tool for tool in filesystem.tools() if tool.name == "view_image") - assert cast("FunctionTool", view_image).is_enabled is has_view_image + is_enabled = cast("Any", cast("FunctionTool", view_image).is_enabled) + + assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is True + assert is_enabled(SimpleNamespace(context={"supports_images": False}), None) is False + models.mark_images_rejected() + assert is_enabled(SimpleNamespace(context={"supports_images": True}), None) is False diff --git a/tests/test_execution_transient_retry.py b/tests/test_execution_transient_retry.py index fa9f13c5..6e501a78 100644 --- a/tests/test_execution_transient_retry.py +++ b/tests/test_execution_transient_retry.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Any, cast +import threading +from typing import Any import httpx import pytest @@ -15,7 +16,7 @@ from openai import ( RateLimitError, ) -from strix.config import codex +from strix.config import codex, models from strix.core import execution from strix.core.agents import AgentCoordinator @@ -129,7 +130,7 @@ async def _run_once( coordinator, "root", input_data="task", - run_config=cast("RunConfig", object()), + run_config=RunConfig(), context={}, max_turns=5, session=session, @@ -232,6 +233,7 @@ async def _run_with_image_session( async def test_run_cycle_strips_images_on_image_rejection( monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.setattr(models, "_IMAGES_REJECTED", threading.Event()) rejection = APIStatusError( "No endpoints found that support image input", response=httpx.Response(status_code=404, request=_request()), @@ -243,3 +245,5 @@ async def test_run_cycle_strips_images_on_image_rejection( assert result is streams[1] assert attempts == 2 assert strips == 1 + # Every later request in the run, from any agent, has its images scrubbed. + assert execution._with_image_scrub(RunConfig(), {}).call_model_input_filter is not None