diff --git a/strix/interface/tool_components/browser_renderer.py b/strix/interface/tool_components/browser_renderer.py index ce1a550d..9f5089cb 100644 --- a/strix/interface/tool_components/browser_renderer.py +++ b/strix/interface/tool_components/browser_renderer.py @@ -55,18 +55,6 @@ class BrowserRenderer(BaseToolRenderer): text.append(token_value, style=color) return text - @classmethod - def _head(cls) -> Text: - text = Text() - text.append("@ ", style=cls.DIM) - return text - - @classmethod - def _icon(cls, color: str) -> Text: - text = Text() - text.append("◈ ", style=color) - return text - @classmethod def _status_mark(cls, status: str) -> Text: text = Text() @@ -128,7 +116,7 @@ class BrowserRenderer(BaseToolRenderer): "extract": "extracting content", } if action in simple: - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append(simple[action], style=cls.DIM) text.append_text(cls._status_mark(status)) return text @@ -136,14 +124,18 @@ class BrowserRenderer(BaseToolRenderer): # --- launch ---------------------------------------------------- if action == "launch": mode = "local" if args.get("use_local") else "sandboxed" - text = cls._icon(cls.LIFE) + text = Text("◈ ", style=cls.LIFE) text.append("launching browser", style=f"bold {cls.LIFE}") text.append(f" {mode}", style=cls.DIM) + res = result if isinstance(result, dict) else {} + warning = res.get("warning") + if warning: + text.append(f"\n ⚠ {warning}", style=f"italic {cls.DIM}") return text # --- navigate -------------------------------------------------- if action == "open": - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("navigating to ", style=cls.DIM) url = args.get("url", "") if len(url) > 80: @@ -160,7 +152,7 @@ class BrowserRenderer(BaseToolRenderer): "rightclick": "right clicking", "hover": "hovering over", } - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append(labels[action], style=cls.DIM) index = args.get("index") if index is not None: @@ -173,7 +165,7 @@ class BrowserRenderer(BaseToolRenderer): # --- text input ------------------------------------------------ if action in ("type", "input"): label = "typing" if action == "type" else "inputting" - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append(label, style=cls.DIM) t = args.get("text") if t: @@ -185,7 +177,7 @@ class BrowserRenderer(BaseToolRenderer): # --- scroll ---------------------------------------------------- if action == "scroll": d = args.get("direction", "down") - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("scrolling ", style=cls.DIM) text.append(d, style=cls.INTERACT) text.append_text(cls._status_mark(status)) @@ -193,7 +185,7 @@ class BrowserRenderer(BaseToolRenderer): # --- tab switch ------------------------------------------------ if action == "switch": - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("switching to tab ", style=cls.DIM) text.append(str(args.get("tab", "?")), style=f"bold {cls.NAV}") text.append_text(cls._status_mark(status)) @@ -201,7 +193,7 @@ class BrowserRenderer(BaseToolRenderer): # --- keyboard -------------------------------------------------- if action == "keys": - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("pressing ", style=cls.DIM) text.append(args.get("keys", ""), style=f"bold {cls.INTERACT}") text.append_text(cls._status_mark(status)) @@ -209,37 +201,42 @@ class BrowserRenderer(BaseToolRenderer): # --- select ---------------------------------------------------- if action == "select": - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("selecting ", style=cls.DIM) text.append(args.get("value", ""), style=f"bold {cls.INTERACT}") text.append_text(cls._status_mark(status)) + return text # --- eval js --------------------------------------------------- if action == "eval": - text = cls._head() + js = args.get("js") + + text = Text("@ ", style=cls.DIM) text.append("executing javascript", style=cls.DIM) text.append_text(cls._status_mark(status)) - js = args.get("js") if js: text.append("\n") text.append_text(cls._highlight_js(js)) + return text # --- cookies --------------------------------------------------- if action == "cookies": sub = args.get("subcommand", "") - text = cls._head() + + text = Text("@ ", style=cls.DIM) text.append("cookies ", style=cls.DIM) text.append(sub, style=cls.OBSERVE) text.append_text(cls._status_mark(status)) + return text # --- wait ------------------------------------------------------ if action == "wait": sub = args.get("subcommand", "") target = args.get("selector") or args.get("text") or "" - text = cls._head() + text = Text("@ ", style=cls.DIM) if status == "completed": text.append(f"waited for {sub}", style=cls.DIM) text.append_text(cls._status_mark(status)) @@ -253,14 +250,14 @@ class BrowserRenderer(BaseToolRenderer): # --- get ------------------------------------------------------- if action == "get": sub = args.get("subcommand", "") - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("getting ", style=cls.DIM) text.append(sub, style=cls.OBSERVE) text.append_text(cls._status_mark(status)) return text # --- fallback -------------------------------------------------- - text = cls._head() + text = Text("@ ", style=cls.DIM) if action: text.append(action, style=cls.DIM) text.append_text(cls._status_mark(status)) @@ -280,7 +277,7 @@ class BrowserRenderer(BaseToolRenderer): task = args.get("task", "") if status == "running": - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("running task", style=f"bold {cls.EXEC}") if task: text.append("\n ") @@ -298,7 +295,7 @@ class BrowserRenderer(BaseToolRenderer): has_error = "error" in res if has_error: - text = cls._head() + text = Text("@ ", style=cls.DIM) text.append("task failed", style=f"bold {cls.ERR}") if task: text.append("\n ") @@ -309,8 +306,8 @@ class BrowserRenderer(BaseToolRenderer): error_msg = error_msg[:197] + "..." text.append(error_msg, style=cls.ERR) else: - text = cls._head() - text.append("task completed", style=f"bold {cls.OK}") + text = Text("@ ", style=cls.DIM) + text.append("browser task completed", style=f"bold {cls.OK}") if task: text.append("\n ") text.append(task, style=cls.DIM) @@ -325,8 +322,8 @@ class BrowserRenderer(BaseToolRenderer): return text # Unknown status - text = cls._head() - text.append("running task", style=cls.DIM) + text = Text("@ ", style=cls.DIM) + text.append("running browser task", style=cls.DIM) if task: text.append("\n ") text.append(task, style=cls.DIM) diff --git a/strix/tools/browser/browser_actions.py b/strix/tools/browser/browser_actions.py index 84065d12..0726c0f5 100644 --- a/strix/tools/browser/browser_actions.py +++ b/strix/tools/browser/browser_actions.py @@ -15,6 +15,7 @@ from .browser_manager import ( _launch_browser, _launch_local_browser, _reinitialize_after_agent, + llm_supports_vision, ) @@ -209,7 +210,13 @@ async def _run_agent_task( for attempt in range(1, max_attempts + 1): llm = _build_llm() - agent = Agent(task=task, llm=llm, browser=session.browser, flash_mode=True) + agent = Agent( + task=task, + llm=llm, + browser=session.browser, + flash_mode=True, + use_vision=llm.supports_vision, + ) try: result = await asyncio.wait_for(agent.run(), timeout=_TASK_TIMEOUT) @@ -575,26 +582,33 @@ async def browser_actions( if action == "launch": if use_local: session = await _launch_local_browser(agent_id, profile_directory) - return { + result: dict[str, Any] = { "message": "Local browser launched and ready", "mode": "local", "profile_directory": session.profile_directory or "auto", "is_running": True, } - cdp_url, auth_token = _resolve_cdp_url(agent_state) - session = await _launch_browser(cdp_url, agent_id, auth_token) - # Strip auth token from the ws_url before returning — the - # token is a secret and must not leak to the calling agent. - import re + else: + cdp_url, auth_token = _resolve_cdp_url(agent_state) + session = await _launch_browser(cdp_url, agent_id, auth_token) + # Strip auth token from the ws_url before returning — the + # token is a secret and must not leak to the calling agent. + import re - safe_ws = re.sub(r"[?&]token=[^&]+", "", session.ws_url) - return { - "message": "Browser launched and ready", - "mode": "sandboxed", - "cdp_url": session.cdp_url, - "ws_url": safe_ws, - "is_running": True, - } + safe_ws = re.sub(r"[?&]token=[^&]+", "", session.ws_url) + result = { + "message": "Browser launched and ready", + "mode": "sandboxed", + "cdp_url": session.cdp_url, + "ws_url": safe_ws, + "is_running": True, + } + if not llm_supports_vision(): + result["warning"] = ( + "The current model does not support vision — " + "the browser agent will operate without screenshots." + ) + return result if action == "close": await _close_session(agent_id) diff --git a/strix/tools/browser/browser_manager.py b/strix/tools/browser/browser_manager.py index cd342451..92b63086 100644 --- a/strix/tools/browser/browser_manager.py +++ b/strix/tools/browser/browser_manager.py @@ -613,3 +613,16 @@ async def _ensure_healthy_session(session: _BrowserSession, task_num: int) -> st return f"Chromium restarted but reconnection failed: {exc}" return None + + +def llm_supports_vision() -> bool: + """Check whether the configured LLM supports vision/image input.""" + try: + import litellm + + from strix.config.config import resolve_llm_config + + model, _, _ = resolve_llm_config() + return bool(model and litellm.supports_vision(model)) + except Exception: # noqa: BLE001 + return False diff --git a/strix/tools/browser/llm.py b/strix/tools/browser/llm.py index 7efbd98d..bd2ac33f 100644 --- a/strix/tools/browser/llm.py +++ b/strix/tools/browser/llm.py @@ -143,6 +143,7 @@ class ChatLiteLLM(BaseChatModel): # Resolved lazily in __post_init__ _provider_name: str = field(default="", init=False, repr=False) _clean_model: str = field(default="", init=False, repr=False) + _supports_vision: bool = field(default=False, init=False, repr=False) def __post_init__(self) -> None: """Resolve provider info from the model string via litellm.""" @@ -157,12 +158,20 @@ class ChatLiteLLM(BaseChatModel): self._provider_name = "openai" self._clean_model = self.model + try: + import litellm + + self._supports_vision = bool(litellm.supports_vision(self.model)) + except Exception: # noqa: BLE001 + self._supports_vision = False + logger.debug( - "ChatLiteLLM initialized: model=%s, provider=%s, clean=%s, api_base=%s", + "ChatLiteLLM initialized: model=%s, provider=%s, clean=%s, api_base=%s, vision=%s", self.model, self._provider_name, self._clean_model, self.api_base or "(default)", + self._supports_vision, ) @property @@ -173,6 +182,10 @@ class ChatLiteLLM(BaseChatModel): def name(self) -> str: return self._clean_model or self.model + @property + def supports_vision(self) -> bool: + return self._supports_vision + # ------------------------------------------------------------------ # Usage parsing # ------------------------------------------------------------------