scriptoriumadmin/strix
1
0
Fork 0
mirror of https://github.com/usestrix/strix.git synced 2026-09-15 23:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: improve support for non-vision models

Browse source
This commit is contained in:
Shanyu Thibaut Juneja 2026-03-11 17:23:41 -07:00 • committed by STJ
parent f99b7ebd4f
commit 9fc09e86a4
4 changed files with 86 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
strix/interface/tool_components/browser_renderer.py
View file

@ -55,18 +55,6 @@ class BrowserRenderer(BaseToolRenderer):
text.append(token_value, style=color)
return text
@classmethod
def _head(cls) -> Text:
text = Text()
text.append("@ ", style=cls.DIM)
return text
@classmethod
def _icon(cls, color: str) -> Text:
text = Text()
text.append("◈ ", style=color)
return text
@classmethod
def _status_mark(cls, status: str) -> Text:
text = Text()
@ -128,7 +116,7 @@ class BrowserRenderer(BaseToolRenderer):
"extract": "extracting content",
}
if action in simple:
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append(simple[action], style=cls.DIM)
text.append_text(cls._status_mark(status))
return text
@ -136,14 +124,18 @@ class BrowserRenderer(BaseToolRenderer):
# --- launch ----------------------------------------------------
if action == "launch":
mode = "local" if args.get("use_local") else "sandboxed"
text = cls._icon(cls.LIFE)
text = Text("◈ ", style=cls.LIFE)
text.append("launching browser", style=f"bold {cls.LIFE}")
text.append(f" {mode}", style=cls.DIM)
res = result if isinstance(result, dict) else {}
warning = res.get("warning")
if warning:
text.append(f"\n ⚠ {warning}", style=f"italic {cls.DIM}")
return text
# --- navigate --------------------------------------------------
if action == "open":
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("navigating to ", style=cls.DIM)
url = args.get("url", "")
if len(url) > 80:
@ -160,7 +152,7 @@ class BrowserRenderer(BaseToolRenderer):
"rightclick": "right clicking",
"hover": "hovering over",
}
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append(labels[action], style=cls.DIM)
index = args.get("index")
if index is not None:
@ -173,7 +165,7 @@ class BrowserRenderer(BaseToolRenderer):
# --- text input ------------------------------------------------
if action in ("type", "input"):
label = "typing" if action == "type" else "inputting"
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append(label, style=cls.DIM)
t = args.get("text")
if t:
@ -185,7 +177,7 @@ class BrowserRenderer(BaseToolRenderer):
# --- scroll ----------------------------------------------------
if action == "scroll":
d = args.get("direction", "down")
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("scrolling ", style=cls.DIM)
text.append(d, style=cls.INTERACT)
text.append_text(cls._status_mark(status))
@ -193,7 +185,7 @@ class BrowserRenderer(BaseToolRenderer):
# --- tab switch ------------------------------------------------
if action == "switch":
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("switching to tab ", style=cls.DIM)
text.append(str(args.get("tab", "?")), style=f"bold {cls.NAV}")
text.append_text(cls._status_mark(status))
@ -201,7 +193,7 @@ class BrowserRenderer(BaseToolRenderer):
# --- keyboard --------------------------------------------------
if action == "keys":
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("pressing ", style=cls.DIM)
text.append(args.get("keys", ""), style=f"bold {cls.INTERACT}")
text.append_text(cls._status_mark(status))
@ -209,37 +201,42 @@ class BrowserRenderer(BaseToolRenderer):
# --- select ----------------------------------------------------
if action == "select":
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("selecting ", style=cls.DIM)
text.append(args.get("value", ""), style=f"bold {cls.INTERACT}")
text.append_text(cls._status_mark(status))
return text
# --- eval js ---------------------------------------------------
if action == "eval":
text = cls._head()
js = args.get("js")
text = Text("@ ", style=cls.DIM)
text.append("executing javascript", style=cls.DIM)
text.append_text(cls._status_mark(status))
js = args.get("js")
if js:
text.append("\n")
text.append_text(cls._highlight_js(js))
return text
# --- cookies ---------------------------------------------------
if action == "cookies":
sub = args.get("subcommand", "")
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("cookies ", style=cls.DIM)
text.append(sub, style=cls.OBSERVE)
text.append_text(cls._status_mark(status))
return text
# --- wait ------------------------------------------------------
if action == "wait":
sub = args.get("subcommand", "")
target = args.get("selector") or args.get("text") or ""
text = cls._head()
text = Text("@ ", style=cls.DIM)
if status == "completed":
text.append(f"waited for {sub}", style=cls.DIM)
text.append_text(cls._status_mark(status))
@ -253,14 +250,14 @@ class BrowserRenderer(BaseToolRenderer):
# --- get -------------------------------------------------------
if action == "get":
sub = args.get("subcommand", "")
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("getting ", style=cls.DIM)
text.append(sub, style=cls.OBSERVE)
text.append_text(cls._status_mark(status))
return text
# --- fallback --------------------------------------------------
text = cls._head()
text = Text("@ ", style=cls.DIM)
if action:
text.append(action, style=cls.DIM)
text.append_text(cls._status_mark(status))
@ -280,7 +277,7 @@ class BrowserRenderer(BaseToolRenderer):
task = args.get("task", "")
if status == "running":
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("running task", style=f"bold {cls.EXEC}")
if task:
text.append("\n ")
@ -298,7 +295,7 @@ class BrowserRenderer(BaseToolRenderer):
has_error = "error" in res
if has_error:
text = cls._head()
text = Text("@ ", style=cls.DIM)
text.append("task failed", style=f"bold {cls.ERR}")
if task:
text.append("\n ")
@ -309,8 +306,8 @@ class BrowserRenderer(BaseToolRenderer):
error_msg = error_msg[:197] + "..."
text.append(error_msg, style=cls.ERR)
else:
text = cls._head()
text.append("task completed", style=f"bold {cls.OK}")
text = Text("@ ", style=cls.DIM)
text.append("browser task completed", style=f"bold {cls.OK}")
if task:
text.append("\n ")
text.append(task, style=cls.DIM)
@ -325,8 +322,8 @@ class BrowserRenderer(BaseToolRenderer):
return text
# Unknown status
text = cls._head()
text.append("running task", style=cls.DIM)
text = Text("@ ", style=cls.DIM)
text.append("running browser task", style=cls.DIM)
if task:
text.append("\n ")
text.append(task, style=cls.DIM)

44
strix/tools/browser/browser_actions.py
View file

@ -15,6 +15,7 @@ from .browser_manager import (
_launch_browser,
_launch_local_browser,
_reinitialize_after_agent,
llm_supports_vision,
)
@ -209,7 +210,13 @@ async def _run_agent_task(
for attempt in range(1, max_attempts + 1):
llm = _build_llm()
agent = Agent(task=task, llm=llm, browser=session.browser, flash_mode=True)
agent = Agent(
task=task,
llm=llm,
browser=session.browser,
flash_mode=True,
use_vision=llm.supports_vision,
)
try:
result = await asyncio.wait_for(agent.run(), timeout=_TASK_TIMEOUT)
@ -575,26 +582,33 @@ async def browser_actions(
if action == "launch":
if use_local:
session = await _launch_local_browser(agent_id, profile_directory)
return {
result: dict[str, Any] = {
"message": "Local browser launched and ready",
"mode": "local",
"profile_directory": session.profile_directory or "auto",
"is_running": True,
}
cdp_url, auth_token = _resolve_cdp_url(agent_state)
session = await _launch_browser(cdp_url, agent_id, auth_token)
# Strip auth token from the ws_url before returning — the
# token is a secret and must not leak to the calling agent.
import re
else:
cdp_url, auth_token = _resolve_cdp_url(agent_state)
session = await _launch_browser(cdp_url, agent_id, auth_token)
# Strip auth token from the ws_url before returning — the
# token is a secret and must not leak to the calling agent.
import re
safe_ws = re.sub(r"[?&]token=[^&]+", "", session.ws_url)
return {
"message": "Browser launched and ready",
"mode": "sandboxed",
"cdp_url": session.cdp_url,
"ws_url": safe_ws,
"is_running": True,
}
safe_ws = re.sub(r"[?&]token=[^&]+", "", session.ws_url)
result = {
"message": "Browser launched and ready",
"mode": "sandboxed",
"cdp_url": session.cdp_url,
"ws_url": safe_ws,
"is_running": True,
}
if not llm_supports_vision():
result["warning"] = (
"The current model does not support vision — "
"the browser agent will operate without screenshots."
)
return result
if action == "close":
await _close_session(agent_id)

13
strix/tools/browser/browser_manager.py
View file

@ -613,3 +613,16 @@ async def _ensure_healthy_session(session: _BrowserSession, task_num: int) -> st
return f"Chromium restarted but reconnection failed: {exc}"
return None
def llm_supports_vision() -> bool:
"""Check whether the configured LLM supports vision/image input."""
try:
import litellm
from strix.config.config import resolve_llm_config
model, _, _ = resolve_llm_config()
return bool(model and litellm.supports_vision(model))
except Exception: # noqa: BLE001
return False

15
strix/tools/browser/llm.py
View file

@ -143,6 +143,7 @@ class ChatLiteLLM(BaseChatModel):
# Resolved lazily in __post_init__
_provider_name: str = field(default="", init=False, repr=False)
_clean_model: str = field(default="", init=False, repr=False)
_supports_vision: bool = field(default=False, init=False, repr=False)
def __post_init__(self) -> None:
"""Resolve provider info from the model string via litellm."""
@ -157,12 +158,20 @@ class ChatLiteLLM(BaseChatModel):
self._provider_name = "openai"
self._clean_model = self.model
try:
import litellm
self._supports_vision = bool(litellm.supports_vision(self.model))
except Exception: # noqa: BLE001
self._supports_vision = False
logger.debug(
"ChatLiteLLM initialized: model=%s, provider=%s, clean=%s, api_base=%s",
"ChatLiteLLM initialized: model=%s, provider=%s, clean=%s, api_base=%s, vision=%s",
self.model,
self._provider_name,
self._clean_model,
self.api_base or "(default)",
self._supports_vision,
)
@property
@ -173,6 +182,10 @@ class ChatLiteLLM(BaseChatModel):
def name(self) -> str:
return self._clean_model or self.model
@property
def supports_vision(self) -> bool:
return self._supports_vision
# ------------------------------------------------------------------
# Usage parsing
# ------------------------------------------------------------------