diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdx index f1542b752..050431246 100644 --- a/docs/advanced/configuration.mdx +++ b/docs/advanced/configuration.mdx @@ -122,6 +122,11 @@ When remote vars are set, Strix dual-writes telemetry to both local JSONL and th Runtime backend for the sandbox environment. + + Maximum time in seconds to wait for the in-sandbox Caido proxy to become ready. + Increase this on slow or heavily loaded container hosts. + + ## Sandbox Configuration diff --git a/strix/config/settings.py b/strix/config/settings.py index 42a2c97ea..ff774d308 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -110,6 +110,7 @@ class RuntimeSettings(BaseSettings): alias="STRIX_IMAGE", ) backend: str = Field(default="docker", alias="STRIX_RUNTIME_BACKEND") + caido_boot_wait_s: int = Field(default=300, gt=0, alias="STRIX_CAIDO_BOOT_WAIT_S") # Max screenshot/image tool outputs kept live per agent context (0 = none). max_context_images: int = Field(default=3, ge=0, alias="STRIX_MAX_CONTEXT_IMAGES") diff --git a/strix/runtime/caido_bootstrap.py b/strix/runtime/caido_bootstrap.py index 0b9ad5b17..e96181b22 100644 --- a/strix/runtime/caido_bootstrap.py +++ b/strix/runtime/caido_bootstrap.py @@ -13,6 +13,8 @@ import asyncio import contextlib import json import logging +import math +import time from typing import TYPE_CHECKING from caido_sdk_client import Client, TokenAuthOptions @@ -35,16 +37,23 @@ async def _login_as_guest( session: BaseSandboxSession, *, container_url: str, - attempts: int = 10, + wait_timeout_s: float, ) -> str: - """``session.exec`` curl to fetch a guest token; retry until ready. + """``session.exec`` curl to fetch a guest token until the deadline. Caido's GraphQL listener may not be up the instant the container starts. The retry loop also doubles as the Caido readiness probe — no separate TCP healthcheck needed. """ + started_at = time.monotonic() + deadline = started_at + wait_timeout_s last_err: str | None = None - for i in range(1, attempts + 1): + attempts = 0 + + while attempts == 0 or time.monotonic() < deadline: + attempts += 1 + remaining = deadline - time.monotonic() + exec_timeout = max(1, min(15, math.ceil(remaining))) result = await session.exec( "curl", "-fsS", @@ -55,7 +64,7 @@ async def _login_as_guest( "-d", _LOGIN_AS_GUEST_BODY, f"{container_url}/graphql", - timeout=15, + timeout=exec_timeout, ) if result.ok(): try: @@ -74,10 +83,24 @@ async def _login_as_guest( else: stderr = result.stderr.decode("utf-8", errors="replace")[:200] last_err = f"curl exit {result.exit_code}: {stderr}" - logger.debug("loginAsGuest attempt %d/%d failed: %s", i, attempts, last_err) - await asyncio.sleep(min(2.0 * i, 8.0)) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + retry_delay = min(2.0 * attempts, 8.0, remaining) + logger.debug( + "loginAsGuest attempt %d failed after %.1fs; retrying in %.1fs (%.1fs remaining): %s", + attempts, + time.monotonic() - started_at, + retry_delay, + remaining, + last_err, + ) + await asyncio.sleep(retry_delay) - raise RuntimeError(f"loginAsGuest failed after {attempts} attempts: {last_err}") + raise RuntimeError( + f"loginAsGuest not ready after {float(wait_timeout_s):g} seconds " + f"({attempts} attempts): {last_err}" + ) async def bootstrap_caido( @@ -85,11 +108,16 @@ async def bootstrap_caido( *, host_url: str, container_url: str, + wait_timeout_s: float, ) -> Client: """Connect to the in-container Caido sidecar and select a fresh project.""" logger.info("Bootstrapping Caido client (host=%s, container=%s)", host_url, container_url) - access_token = await _login_as_guest(session, container_url=container_url) + access_token = await _login_as_guest( + session, + container_url=container_url, + wait_timeout_s=wait_timeout_s, + ) client = Client(host_url, auth=TokenAuthOptions(token=access_token)) await client.connect() diff --git a/strix/runtime/session_manager.py b/strix/runtime/session_manager.py index 4b61d7351..91b965271 100644 --- a/strix/runtime/session_manager.py +++ b/strix/runtime/session_manager.py @@ -128,7 +128,8 @@ async def create_or_reuse( logger.info("Reusing existing sandbox session for scan %s", scan_id) return cached - backend_name = load_settings().runtime.backend + runtime_settings = load_settings().runtime + backend_name = runtime_settings.backend backend = get_backend(backend_name) if backend_supports_bind_mounts(backend_name): @@ -183,6 +184,7 @@ async def create_or_reuse( session, host_url=host_caido_url, container_url=container_caido_url, + wait_timeout_s=runtime_settings.caido_boot_wait_s, ) bundle = { diff --git a/tests/test_caido_bootstrap.py b/tests/test_caido_bootstrap.py new file mode 100644 index 000000000..5e63e18b3 --- /dev/null +++ b/tests/test_caido_bootstrap.py @@ -0,0 +1,104 @@ +"""Tests for the Caido readiness deadline.""" + +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from strix.runtime import caido_bootstrap + + +if TYPE_CHECKING: + from agents.sandbox.session import BaseSandboxSession + + +@dataclass +class _ExecResult: + success: bool + stdout: str = "" + stderr: bytes = b"connection refused" + exit_code: int = 7 + + def ok(self) -> bool: + return self.success + + +class _Session: + def __init__(self, results: list[_ExecResult]) -> None: + self._results = iter(results) + self.calls: list[tuple[tuple[str, ...], dict[str, Any]]] = [] + + async def exec(self, *args: str, **kwargs: Any) -> _ExecResult: + self.calls.append((args, kwargs)) + return next(self._results) + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self.now + + async def sleep(self, delay: float) -> None: + self.sleeps.append(delay) + self.now += delay + + +def _install_clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + clock = _Clock() + monkeypatch.setattr("strix.runtime.caido_bootstrap.time.monotonic", clock.monotonic) + monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", clock.sleep) + return clock + + +@pytest.mark.asyncio +async def test_login_as_guest_retries_until_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _install_clock(monkeypatch) + session = _Session( + [ + _ExecResult(success=False), + _ExecResult( + success=True, + stdout='{"data":{"loginAsGuest":{"token":{"accessToken":"token"}}}}', + ), + ] + ) + + token = await caido_bootstrap._login_as_guest( + cast("BaseSandboxSession", session), + container_url="http://127.0.0.1:48080", + wait_timeout_s=10, + ) + + assert token == "token" # noqa: S105 - synthetic token fixture + assert clock.sleeps == [2.0] + assert [call[1]["timeout"] for call in session.calls] == [10, 8] + + +@pytest.mark.asyncio +async def test_login_as_guest_stops_at_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _install_clock(monkeypatch) + session = _Session([_ExecResult(success=False), _ExecResult(success=False)]) + + with pytest.raises( + RuntimeError, + match=r"loginAsGuest not ready after 5 seconds \(2 attempts\)", + ): + await caido_bootstrap._login_as_guest( + cast("BaseSandboxSession", session), + container_url="http://127.0.0.1:48080", + wait_timeout_s=5, + ) + + assert clock.sleeps == [2.0, 3.0] + assert [call[1]["timeout"] for call in session.calls] == [5, 3] diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index e83ab1192..e0315db9c 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -33,6 +33,7 @@ _LLM_ENV_KEYS = [ # RuntimeSettings "STRIX_IMAGE", "STRIX_RUNTIME_BACKEND", + "STRIX_CAIDO_BOOT_WAIT_S", # TelemetrySettings "STRIX_TELEMETRY", ] @@ -129,6 +130,12 @@ def test_tool_output_max_bytes_accepts_floor() -> None: assert ContextSettings(STRIX_TOOL_OUTPUT_MAX_BYTES=1024).tool_output_max_bytes == 1024 +def test_runtime_caido_boot_wait_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("STRIX_CAIDO_BOOT_WAIT_S", "900") + + assert loader.load_settings().runtime.caido_boot_wait_s == 900 + + # --------------------------------------------------------------------------- # # _aliases_for # --------------------------------------------------------------------------- #