From 6b171b2a3037919bb61878e47e0bf545f7825232 Mon Sep 17 00:00:00 2001 From: Mads Hvelplund Date: Fri, 19 Jun 2026 06:24:53 +0200 Subject: [PATCH] feat: add --mount and large-target pre-flight for local repos (#492) Large local targets were copied into the sandbox file-by-file via the SDK LocalDir entry, which stalls on big repos and could leave /workspace empty. - --mount bind-mounts a host directory read-only at /workspace/ instead of copying it, bypassing the per-file stream. - A size pre-flight (STRIX_MAX_LOCAL_COPY_MB, default 1024) fails fast with a clear message suggesting --mount when a non-mounted local target is too big. --- strix/config/settings.py | 4 ++ strix/core/inputs.py | 3 +- strix/core/runner.py | 2 +- strix/interface/main.py | 39 +++++++++-- strix/interface/utils.py | 80 ++++++++++++++++++++- strix/runtime/backends.py | 2 + strix/runtime/docker_client.py | 20 ++++++ strix/runtime/session_manager.py | 50 ++++++++++--- tests/test_local_sources.py | 116 +++++++++++++++++++++++++++++++ tests/test_session_entries.py | 67 ++++++++++++++++++ 10 files changed, 364 insertions(+), 19 deletions(-) create mode 100644 tests/test_local_sources.py create mode 100644 tests/test_session_entries.py diff --git a/strix/config/settings.py b/strix/config/settings.py index 1458e1ff..ecea8e50 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -47,6 +47,10 @@ class RuntimeSettings(BaseSettings): alias="STRIX_IMAGE", ) backend: str = Field(default="docker", alias="STRIX_RUNTIME_BACKEND") + # Hard cap on a local target's size before we refuse to stream it into the + # sandbox file-by-file (the SDK copies every file individually, which stalls + # on large repos). Above this, the user must bind-mount via ``--mount``. + max_local_copy_mb: int = Field(default=1024, alias="STRIX_MAX_LOCAL_COPY_MB") class TelemetrySettings(BaseSettings): diff --git a/strix/core/inputs.py b/strix/core/inputs.py index b86daa4c..0505ef38 100644 --- a/strix/core/inputs.py +++ b/strix/core/inputs.py @@ -44,7 +44,8 @@ def build_root_task(scan_config: dict[str, Any]) -> str: ) elif ttype == "local_code": path = details.get("target_path", "unknown") - sections["Local Codebases"].append(f"- {path} (available at: {workspace_path})") + suffix = ", read-only mount" if details.get("mount") else "" + sections["Local Codebases"].append(f"- {path} (available at: {workspace_path}{suffix})") elif ttype == "web_application": sections["URLs"].append(f"- {details.get('target_url', '')}") elif ttype == "ip_address": diff --git a/strix/core/runner.py b/strix/core/runner.py index a7371e9f..9ab120bc 100644 --- a/strix/core/runner.py +++ b/strix/core/runner.py @@ -55,7 +55,7 @@ async def run_strix_scan( scan_config: dict[str, Any], scan_id: str | None = None, image: str, - local_sources: list[dict[str, str]] | None = None, + local_sources: list[dict[str, Any]] | None = None, coordinator: AgentCoordinator | None = None, interactive: bool = False, max_turns: int = DEFAULT_MAX_TURNS, diff --git a/strix/interface/main.py b/strix/interface/main.py index b5c56ad4..373b872f 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -33,9 +33,11 @@ from strix.interface.tui import run_tui from strix.interface.utils import ( assign_workspace_subdirs, build_final_stats_text, + build_mount_targets_info, check_docker_connection, clone_repository, collect_local_sources, + find_oversized_local_targets, generate_run_name, image_exists, infer_target_type, @@ -352,6 +354,15 @@ Examples: "Can be specified multiple times for multi-target scans. " "Required for fresh runs; loaded from disk when ``--resume`` is set.", ) + parser.add_argument( + "--mount", + type=str, + action="append", + metavar="PATH", + help="Bind-mount a local directory into the sandbox (read-only) instead of " + "copying it file-by-file. Use this for large repositories that are too big to " + "stream into the container. Can be specified multiple times.", + ) parser.add_argument( "--instruction", type=str, @@ -455,9 +466,9 @@ Examples: args.user_explicit_instruction = args.instruction if args.resume else None if args.resume: - if args.target: + if args.target or args.mount: parser.error( - "Cannot combine --resume with --target. --resume picks up where " + "Cannot combine --resume with --target/--mount. --resume picks up where " "the prior run left off, including the original target list." ) _load_resume_state(args, parser) @@ -470,13 +481,13 @@ Examples: f"or remove --resume to start over with the same targets." ) else: - if not args.target: + if not args.target and not args.mount: parser.error( - "the following arguments are required: -t/--target " + "the following arguments are required: -t/--target or --mount " "(or use --resume to continue a prior scan)" ) args.targets_info = [] - for target in args.target: + for target in args.target or []: try: target_type, target_dict = infer_target_type(target) @@ -491,9 +502,27 @@ Examples: except ValueError: parser.error(f"Invalid target '{target}'") + try: + args.targets_info.extend(build_mount_targets_info(args.mount or [])) + except ValueError as e: + parser.error(str(e)) + assign_workspace_subdirs(args.targets_info) rewrite_localhost_targets(args.targets_info, HOST_GATEWAY_HOSTNAME) + max_copy_bytes = load_settings().runtime.max_local_copy_mb * 1024 * 1024 + oversized = find_oversized_local_targets(args.targets_info, max_copy_bytes) + if oversized: + details = "; ".join( + f"{path} ({size / (1024 * 1024):.0f} MB)" for path, size in oversized + ) + parser.error( + f"Local target too large to stream into the sandbox: {details}. " + f"The limit is {load_settings().runtime.max_local_copy_mb} MB " + "(set STRIX_MAX_LOCAL_COPY_MB to change it). Re-run with " + "--mount to bind-mount the directory instead of copying it." + ) + return args diff --git a/strix/interface/utils.py b/strix/interface/utils.py index ff53a6cd..dfe2f4ec 100644 --- a/strix/interface/utils.py +++ b/strix/interface/utils.py @@ -1185,8 +1185,8 @@ def is_whitebox_scan(targets_info: list[dict[str, Any]]) -> bool: return any(t.get("type") == "local_code" for t in targets_info or []) -def collect_local_sources(targets_info: list[dict[str, Any]]) -> list[dict[str, str]]: - local_sources: list[dict[str, str]] = [] +def collect_local_sources(targets_info: list[dict[str, Any]]) -> list[dict[str, Any]]: + local_sources: list[dict[str, Any]] = [] for target_info in targets_info: details = target_info["details"] @@ -1197,6 +1197,7 @@ def collect_local_sources(targets_info: list[dict[str, Any]]) -> list[dict[str, { "source_path": details["target_path"], "workspace_subdir": workspace_subdir, + "mount": bool(details.get("mount", False)), } ) @@ -1205,12 +1206,87 @@ def collect_local_sources(targets_info: list[dict[str, Any]]) -> list[dict[str, { "source_path": details["cloned_repo_path"], "workspace_subdir": workspace_subdir, + "mount": False, } ) return local_sources +def directory_size_bytes(path: Path) -> int: + """Total size in bytes of regular files under ``path`` (symlinks not followed). + + Best-effort: files that disappear or can't be stat'd mid-walk are skipped. + Used as a cheap (stat-only) pre-flight to estimate the cost of streaming a + local target into the sandbox before we actually try to copy it. + """ + total = 0 + for root, _dirs, files in os.walk(path, followlinks=False): + for name in files: + file_path = os.path.join(root, name) # noqa: PTH118 + try: + if os.path.islink(file_path): # noqa: PTH114 + continue + total += os.path.getsize(file_path) # noqa: PTH202 + except OSError: + continue + return total + + +def find_oversized_local_targets( + targets_info: list[dict[str, Any]], max_bytes: int +) -> list[tuple[str, int]]: + """Return ``(path, size_bytes)`` for non-mounted local targets over ``max_bytes``. + + Mounted targets are bind-mounted rather than copied, so their size is + irrelevant and they are excluded. + """ + oversized: list[tuple[str, int]] = [] + for target in targets_info: + if target.get("type") != "local_code": + continue + details = target.get("details") or {} + if details.get("mount"): + continue + target_path = details.get("target_path") + if not target_path: + continue + size = directory_size_bytes(Path(target_path)) + if size > max_bytes: + oversized.append((target_path, size)) + return oversized + + +def build_mount_targets_info(mount_paths: list[str]) -> list[dict[str, Any]]: + """Build ``targets_info`` entries for ``--mount`` directories. + + Each path must be an existing local directory; it is bind-mounted into the + sandbox (read-only) instead of being copied file-by-file. Raises + ``ValueError`` for a path that does not exist or is not a directory. + """ + targets_info: list[dict[str, Any]] = [] + for raw in mount_paths: + path = Path(raw).expanduser() + try: + resolved = path.resolve() + is_dir = resolved.is_dir() + except (OSError, RuntimeError) as e: + raise ValueError(f"Invalid mount path '{raw}': {e!s}") from e + if not is_dir: + raise ValueError( + f"Mount path '{raw}' is not an existing directory. " + "--mount requires a path to a local directory." + ) + targets_info.append( + { + "type": "local_code", + "details": {"target_path": str(resolved), "mount": True}, + "original": str(resolved), + } + ) + return targets_info + + def _is_localhost_host(host: str) -> bool: host_lower = host.lower().strip("[]") diff --git a/strix/runtime/backends.py b/strix/runtime/backends.py index 9f241a3a..fe6715f8 100644 --- a/strix/runtime/backends.py +++ b/strix/runtime/backends.py @@ -22,6 +22,7 @@ async def _docker_backend( image: str, manifest: Manifest, exposed_ports: tuple[int, ...], + bind_mounts: list[dict[str, Any]] | None = None, ) -> tuple[Any, Any]: """Bring up a session backed by the local Docker daemon. @@ -43,6 +44,7 @@ async def _docker_backend( from strix.runtime.docker_client import StrixDockerSandboxClient client = StrixDockerSandboxClient(docker.from_env()) + client.strix_bind_mounts = bind_mounts or [] options = DockerSandboxClientOptions(image=image, exposed_ports=exposed_ports) session = await client.create(options=options, manifest=manifest) await session.start() diff --git a/strix/runtime/docker_client.py b/strix/runtime/docker_client.py index 2a753834..49cb3e7d 100644 --- a/strix/runtime/docker_client.py +++ b/strix/runtime/docker_client.py @@ -38,6 +38,7 @@ from agents.sandbox.sandboxes.docker import ( from agents.sandbox.session.sandbox_session import SandboxSession from docker import errors as docker_errors # type: ignore[import-untyped, unused-ignore] from docker.models.containers import Container # type: ignore[import-untyped, unused-ignore] +from docker.types import Mount as DockerSDKMount # type: ignore[import-untyped, unused-ignore] from docker.utils import parse_repository_tag # type: ignore[import-untyped, unused-ignore] @@ -45,6 +46,10 @@ logger = logging.getLogger(__name__) class StrixDockerSandboxClient(DockerSandboxClient): + # Host directories to bind-mount into the container, set by the docker + # backend before ``create()``. Each item is ``{source, target, read_only}``. + strix_bind_mounts: list[dict[str, Any]] + async def _create_container( self, image: str, @@ -111,6 +116,21 @@ class StrixDockerSandboxClient(DockerSandboxClient): extra_hosts = create_kwargs.setdefault("extra_hosts", {}) extra_hosts["host.docker.internal"] = "host-gateway" + # Strix injection: host bind mounts (e.g. large repos passed via --mount) + # that bypass the SDK's file-by-file LocalDir copy. + bind_mounts = getattr(self, "strix_bind_mounts", ()) + if bind_mounts: + mounts = create_kwargs.setdefault("mounts", []) + for spec in bind_mounts: + mounts.append( + DockerSDKMount( + target=spec["target"], + source=spec["source"], + type="bind", + read_only=spec.get("read_only", True), + ) + ) + logger.debug( "Creating sandbox container: image=%s caps=%s exposed_ports=%s", image, diff --git a/strix/runtime/session_manager.py b/strix/runtime/session_manager.py index 6f3e2733..a19495cc 100644 --- a/strix/runtime/session_manager.py +++ b/strix/runtime/session_manager.py @@ -23,30 +23,59 @@ _CONTAINER_CAIDO_PORT = 48080 _SESSION_CACHE: dict[str, dict[str, Any]] = {} +# Manifest root inside the container; entry keys hang off this path. +_WORKSPACE_ROOT = "/workspace" + + +def build_session_entries( + local_sources: list[dict[str, Any]], +) -> tuple[dict[str | Path, BaseEntry], list[dict[str, Any]]]: + """Split local sources into copied manifest entries and host bind mounts. + + Sources flagged ``mount`` are bind-mounted read-only at + ``/workspace/`` (not added to the manifest, so the SDK + does not stream them in file-by-file). Every other source becomes a + ``LocalDir`` entry copied into the container as before. + """ + entries: dict[str | Path, BaseEntry] = {} + bind_mounts: list[dict[str, Any]] = [] + for src in local_sources: + ws_subdir = src.get("workspace_subdir") or "" + host_path = src.get("source_path") or "" + if not ws_subdir or not host_path: + continue + resolved = Path(host_path).expanduser().resolve() + if src.get("mount"): + bind_mounts.append( + { + "source": str(resolved), + "target": f"{_WORKSPACE_ROOT}/{ws_subdir}", + "read_only": True, + } + ) + else: + entries[ws_subdir] = LocalDir(src=resolved) + return entries, bind_mounts + async def create_or_reuse( scan_id: str, *, image: str, - local_sources: list[dict[str, str]], + local_sources: list[dict[str, Any]], ) -> dict[str, Any]: """Return the existing session bundle for ``scan_id`` or create a new one. - Each ``local_sources`` entry mounts its host ``source_path`` at - ``/workspace/`` inside the container. + Each ``local_sources`` entry exposes its host ``source_path`` at + ``/workspace/`` inside the container — copied in, or + bind-mounted read-only when the entry is flagged ``mount``. """ cached = _SESSION_CACHE.get(scan_id) if cached is not None: logger.info("Reusing existing sandbox session for scan %s", scan_id) return cached - entries: dict[str | Path, BaseEntry] = {} - for src in local_sources: - ws_subdir = src.get("workspace_subdir") or "" - host_path = src.get("source_path") or "" - if not ws_subdir or not host_path: - continue - entries[ws_subdir] = LocalDir(src=Path(host_path).expanduser().resolve()) + entries, bind_mounts = build_session_entries(local_sources) # Caido runs as an in-container sidecar; HTTP(S) traffic from any # process started via ``session.exec`` (the SDK's Shell tool, etc.) @@ -81,6 +110,7 @@ async def create_or_reuse( image=image, manifest=manifest, exposed_ports=(_CONTAINER_CAIDO_PORT,), + bind_mounts=bind_mounts, ) caido_endpoint = await session.resolve_exposed_port(_CONTAINER_CAIDO_PORT) diff --git a/tests/test_local_sources.py b/tests/test_local_sources.py new file mode 100644 index 00000000..af6e5a17 --- /dev/null +++ b/tests/test_local_sources.py @@ -0,0 +1,116 @@ +"""Tests for local-source sizing and ``--mount`` target helpers in interface.utils.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + + +if TYPE_CHECKING: + from pathlib import Path + +from strix.interface.utils import ( + build_mount_targets_info, + collect_local_sources, + directory_size_bytes, + find_oversized_local_targets, +) + + +def _write_file(path: Path, size: int) -> None: + path.write_bytes(b"x" * size) + + +def _local_target(target_path: str, *, mount: bool = False) -> dict[str, Any]: + details: dict[str, Any] = {"target_path": target_path, "workspace_subdir": "repo"} + if mount: + details["mount"] = True + return {"type": "local_code", "details": details, "original": target_path} + + +def test_directory_size_empty_dir_is_zero(tmp_path: Path) -> None: + assert directory_size_bytes(tmp_path) == 0 + + +def test_directory_size_sums_flat_and_nested_files(tmp_path: Path) -> None: + _write_file(tmp_path / "a.txt", 100) + nested = tmp_path / "sub" / "deep" + nested.mkdir(parents=True) + _write_file(nested / "b.txt", 250) + assert directory_size_bytes(tmp_path) == 350 + + +def test_directory_size_skips_symlinks(tmp_path: Path) -> None: + _write_file(tmp_path / "real.txt", 100) + (tmp_path / "link.txt").symlink_to(tmp_path / "real.txt") + # The symlink target is counted once via the real file, not doubled. + assert directory_size_bytes(tmp_path) == 100 + + +def test_find_oversized_returns_nothing_under_limit(tmp_path: Path) -> None: + _write_file(tmp_path / "a.txt", 100) + targets = [_local_target(str(tmp_path))] + assert find_oversized_local_targets(targets, max_bytes=1000) == [] + + +def test_find_oversized_returns_target_over_limit(tmp_path: Path) -> None: + _write_file(tmp_path / "big.bin", 500) + targets = [_local_target(str(tmp_path))] + result = find_oversized_local_targets(targets, max_bytes=100) + assert result == [(str(tmp_path), 500)] + + +def test_find_oversized_ignores_mounted_targets(tmp_path: Path) -> None: + _write_file(tmp_path / "big.bin", 500) + targets = [_local_target(str(tmp_path), mount=True)] + assert find_oversized_local_targets(targets, max_bytes=100) == [] + + +def test_find_oversized_ignores_non_local_targets() -> None: + targets = [{"type": "web_application", "details": {"target_url": "https://x"}}] + assert find_oversized_local_targets(targets, max_bytes=0) == [] + + +def test_collect_local_sources_propagates_mount_flag() -> None: + copied = _local_target("/copied") + copied["details"]["workspace_subdir"] = "copied" + mounted = _local_target("/mounted", mount=True) + mounted["details"]["workspace_subdir"] = "mounted" + + sources = collect_local_sources([copied, mounted]) + + by_path = {s["source_path"]: s for s in sources} + assert by_path["/copied"]["mount"] is False + assert by_path["/mounted"]["mount"] is True + + +def test_collect_local_sources_repository_is_never_mounted() -> None: + repo = { + "type": "repository", + "details": {"cloned_repo_path": "/clone", "workspace_subdir": "clone"}, + } + sources = collect_local_sources([repo]) + assert sources == [{"source_path": "/clone", "workspace_subdir": "clone", "mount": False}] + + +def test_build_mount_targets_info_for_valid_dir(tmp_path: Path) -> None: + result = build_mount_targets_info([str(tmp_path)]) + assert len(result) == 1 + entry = result[0] + assert entry["type"] == "local_code" + assert entry["details"]["mount"] is True + assert entry["details"]["target_path"] == str(tmp_path.resolve()) + + +def test_build_mount_targets_info_rejects_missing_path(tmp_path: Path) -> None: + missing = tmp_path / "does-not-exist" + with pytest.raises(ValueError, match="not an existing directory"): + build_mount_targets_info([str(missing)]) + + +def test_build_mount_targets_info_rejects_file(tmp_path: Path) -> None: + file_path = tmp_path / "a-file.txt" + _write_file(file_path, 10) + with pytest.raises(ValueError, match="not an existing directory"): + build_mount_targets_info([str(file_path)]) diff --git a/tests/test_session_entries.py b/tests/test_session_entries.py new file mode 100644 index 00000000..8288c1d0 --- /dev/null +++ b/tests/test_session_entries.py @@ -0,0 +1,67 @@ +"""Tests for build_session_entries: splitting copied vs bind-mounted sources.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from agents.sandbox.entries import LocalDir + +from strix.runtime.session_manager import build_session_entries + + +if TYPE_CHECKING: + from pathlib import Path + + +def _source(subdir: str, path: str, *, mount: bool = False) -> dict[str, Any]: + return {"source_path": path, "workspace_subdir": subdir, "mount": mount} + + +def test_copied_source_becomes_localdir_entry(tmp_path: Path) -> None: + entries, bind_mounts = build_session_entries([_source("repo", str(tmp_path))]) + + assert bind_mounts == [] + assert isinstance(entries["repo"], LocalDir) + assert entries["repo"].src == tmp_path.resolve() + + +def test_mounted_source_becomes_bind_mount(tmp_path: Path) -> None: + entries, bind_mounts = build_session_entries([_source("repo", str(tmp_path), mount=True)]) + + assert entries == {} + assert bind_mounts == [ + { + "source": str(tmp_path.resolve()), + "target": "/workspace/repo", + "read_only": True, + } + ] + + +def test_mixed_sources_split_correctly(tmp_path: Path) -> None: + copied = tmp_path / "copied" + mounted = tmp_path / "mounted" + copied.mkdir() + mounted.mkdir() + + entries, bind_mounts = build_session_entries( + [ + _source("copied", str(copied)), + _source("mounted", str(mounted), mount=True), + ] + ) + + assert list(entries) == ["copied"] + assert isinstance(entries["copied"], LocalDir) + assert [m["target"] for m in bind_mounts] == ["/workspace/mounted"] + + +def test_incomplete_sources_are_skipped() -> None: + entries, bind_mounts = build_session_entries( + [ + {"source_path": "", "workspace_subdir": "x"}, + {"source_path": "/p", "workspace_subdir": ""}, + ] + ) + assert entries == {} + assert bind_mounts == []