fix: bound Caido readiness probe by wall-clock deadline, not fixed attempts

_login_as_guest() retried loginAsGuest exactly 10 times with capped
exponential backoff (~68s total), regardless of how long the sandbox
actually took to boot Caido. The sandbox entrypoint chowns a large
toolchain tree before starting caido-cli, which can take minutes on a
loaded host or CI runner, so every scan there failed before a single
LLM call was made.

Replace the fixed attempt count with a wall-clock deadline, configurable
via STRIX_CAIDO_BOOT_WAIT_S (default 180s). Fast hosts are unaffected;
slow/shared hosts and CI runners can raise the budget instead of hitting
a hardcoded ceiling. The error message now reports elapsed time and
attempt count instead of just attempts.

Fixes #1036, #1037
This commit is contained in:
Roshan Aryal 2026-08-11 16:46:33 +12:00
parent 7cc9fa9faa
commit 706040acda
4 changed files with 119 additions and 8 deletions

View file

@ -112,6 +112,12 @@ class RuntimeSettings(BaseSettings):
backend: str = Field(default="docker", alias="STRIX_RUNTIME_BACKEND")
# Max screenshot/image tool outputs kept live per agent context (0 = none).
max_context_images: int = Field(default=3, ge=0, alias="STRIX_MAX_CONTEXT_IMAGES")
# Wall-clock budget for the Caido readiness probe (session_manager ->
# bootstrap_caido -> _login_as_guest). The sandbox entrypoint chowns a
# large toolchain tree before starting caido-cli, which can take minutes
# on a loaded host or CI runner; a fixed attempt count gives up too early
# there. Raise this on slow/shared hosts instead of patching the retry loop.
caido_boot_wait_s: float = Field(default=180.0, gt=0, alias="STRIX_CAIDO_BOOT_WAIT_S")
class TelemetrySettings(BaseSettings):

View file

@ -13,6 +13,7 @@ import asyncio
import contextlib
import json
import logging
import time
from typing import TYPE_CHECKING
from caido_sdk_client import Client, TokenAuthOptions
@ -35,16 +36,23 @@ async def _login_as_guest(
session: BaseSandboxSession,
*,
container_url: str,
attempts: int = 10,
max_wait_s: float = 180.0,
) -> str:
"""``session.exec`` curl to fetch a guest token; retry until ready.
Caido's GraphQL listener may not be up the instant the container
starts. The retry loop also doubles as the Caido readiness probe
no separate TCP healthcheck needed.
starts the sandbox entrypoint chowns a large toolchain tree first,
which can take minutes on a loaded host or CI runner. The retry loop
also doubles as the Caido readiness probe no separate TCP
healthcheck needed so it is bounded by wall-clock time rather than
a fixed attempt count, to accommodate slow boots without making fast
hosts wait needlessly.
"""
deadline = time.monotonic() + max_wait_s
last_err: str | None = None
for i in range(1, attempts + 1):
attempt = 0
while True:
attempt += 1
result = await session.exec(
"curl",
"-fsS",
@ -74,10 +82,16 @@ async def _login_as_guest(
else:
stderr = result.stderr.decode("utf-8", errors="replace")[:200]
last_err = f"curl exit {result.exit_code}: {stderr}"
logger.debug("loginAsGuest attempt %d/%d failed: %s", i, attempts, last_err)
await asyncio.sleep(min(2.0 * i, 8.0))
raise RuntimeError(f"loginAsGuest failed after {attempts} attempts: {last_err}")
remaining = deadline - time.monotonic()
if remaining <= 0:
break
logger.debug("loginAsGuest attempt %d failed: %s", attempt, last_err)
await asyncio.sleep(min(2.0 * attempt, 8.0, remaining))
raise RuntimeError(
f"loginAsGuest failed after {attempt} attempts over {max_wait_s:.0f}s: {last_err}"
)
async def bootstrap_caido(
@ -85,11 +99,14 @@ async def bootstrap_caido(
*,
host_url: str,
container_url: str,
boot_wait_s: float = 180.0,
) -> Client:
"""Connect to the in-container Caido sidecar and select a fresh project."""
logger.info("Bootstrapping Caido client (host=%s, container=%s)", host_url, container_url)
access_token = await _login_as_guest(session, container_url=container_url)
access_token = await _login_as_guest(
session, container_url=container_url, max_wait_s=boot_wait_s
)
client = Client(host_url, auth=TokenAuthOptions(token=access_token))
await client.connect()

View file

@ -183,6 +183,7 @@ async def create_or_reuse(
session,
host_url=host_caido_url,
container_url=container_caido_url,
boot_wait_s=load_settings().runtime.caido_boot_wait_s,
)
bundle = {

View file

@ -0,0 +1,87 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any
import pytest
from strix.runtime.caido_bootstrap import _login_as_guest
@dataclass
class _FakeResult:
exit_code: int
stdout: str = ""
stderr: bytes = b""
def ok(self) -> bool:
return self.exit_code == 0
@dataclass
class _FakeSession:
"""Stands in for BaseSandboxSession; returns queued exec() results."""
results: list[_FakeResult]
sleeps: list[float] = field(default_factory=list)
calls: int = 0
async def exec(self, *_args: Any, **_kwargs: Any) -> _FakeResult:
result = self.results[min(self.calls, len(self.results) - 1)]
self.calls += 1
return result
_FAKE_TOKEN = "guest-token" # noqa: S105 - test fixture, not a real credential
def _token_result() -> _FakeResult:
body = {"data": {"loginAsGuest": {"token": {"accessToken": _FAKE_TOKEN}}}}
return _FakeResult(exit_code=0, stdout=json.dumps(body))
def _refused_result() -> _FakeResult:
return _FakeResult(exit_code=7, stderr=b"curl: (7) Failed to connect")
async def test_login_as_guest_succeeds_once_caido_is_up(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", _no_sleep)
session = _FakeSession(results=[_refused_result(), _refused_result(), _token_result()])
token = await _login_as_guest(session, container_url="http://127.0.0.1:48080", max_wait_s=30)
assert token == _FAKE_TOKEN
assert session.calls == 3
async def test_login_as_guest_respects_configurable_deadline(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A slow-booting sandbox (connection refused throughout) should be given
the full configured budget rather than giving up after a fixed attempt
count, and the raised error should report elapsed time, not attempts.
"""
monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", _no_sleep)
# Fake clock: each read advances by 25s, so a 180s budget survives
# ~7 attempts (the old fixed-10-attempt loop only had ~68s total).
fake_now = [0.0]
def _fake_monotonic() -> float:
fake_now[0] += 25.0
return fake_now[0]
monkeypatch.setattr("strix.runtime.caido_bootstrap.time.monotonic", _fake_monotonic)
session = _FakeSession(results=[_refused_result()])
with pytest.raises(RuntimeError) as exc_info:
await _login_as_guest(session, container_url="http://127.0.0.1:48080", max_wait_s=180)
assert "curl exit 7" in str(exc_info.value)
assert "180s" in str(exc_info.value)
assert session.calls >= 6
async def _no_sleep(_seconds: float) -> None:
"""asyncio.sleep stub so deadline-bound tests run instantly."""