mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
Merge pull request #640 from fabro-sh/codex/workspace-glob-semantics
Unify workspace glob semantics across sandboxes and artifacts
This commit is contained in:
Bryan Helmkamp
GitHub
commit
e2df6e68c7
33 changed files with 1313 additions and 1095 deletions
3
Cargo.lock
generated
3
Cargo.lock
generated
|
|
@ -2965,7 +2965,6 @@ dependencies = [
|
|||
"futures",
|
||||
"futures-util",
|
||||
"git2",
|
||||
"glob",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"httpmock",
|
||||
|
|
@ -3272,6 +3271,7 @@ dependencies = [
|
|||
"console 0.15.11",
|
||||
"dirs",
|
||||
"fabro-static",
|
||||
"glob",
|
||||
"insta",
|
||||
"open",
|
||||
"rand 0.9.4",
|
||||
|
|
@ -3280,6 +3280,7 @@ dependencies = [
|
|||
"shlex",
|
||||
"tempfile",
|
||||
"termimad",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ regex = "1"
|
|||
semver = "1"
|
||||
aho-corasick = "1"
|
||||
globset = "0.4"
|
||||
glob = "0.3"
|
||||
dirs = "6"
|
||||
mac_address = "1"
|
||||
md5 = "0.7"
|
||||
|
|
|
|||
|
|
@ -206,7 +206,7 @@ function ArtifactsPanel({ snapshot }: { snapshot: WorkflowSettings }) {
|
|||
const include = getArray(artifacts, "include");
|
||||
return (
|
||||
<Panel title="Artifacts">
|
||||
<Row title="Include" help="Globs collected from the sandbox at run end.">
|
||||
<Row title="Include" help="Workspace-relative globs collected after each stage.">
|
||||
{include && include.length > 0 ? (
|
||||
<GlobList globs={include.filter((e): e is string => typeof e === "string")} />
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -315,7 +315,7 @@ After startup, the agent sees 22 Playwright tools including:
|
|||
- `mcp__playwright__browser_type`
|
||||
- `mcp__playwright__browser_fill_form`
|
||||
|
||||
The agent uses `browser_snapshot` (accessibility tree) for structured page understanding and `browser_take_screenshot` to save visual captures. Screenshots saved to `screenshots/` are automatically collected as [artifacts](/execution/run-configuration#assets).
|
||||
The agent uses `browser_snapshot` (accessibility tree) for structured page understanding and `browser_take_screenshot` to save visual captures. Screenshots saved to `screenshots/` can be collected as [artifacts](/execution/run-configuration#runartifacts) by including `screenshots/**`.
|
||||
|
||||
<Note>
|
||||
When using Playwright MCP with the sandbox transport, call the `browser_install` tool first to ensure the Playwright browser binaries are available inside the sandbox.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ title: "Outputs & Artifacts"
|
|||
description: "How Fabro captures agent responses, tracks file changes, and collects test assets"
|
||||
---
|
||||
|
||||
When an agent or prompt node finishes, Fabro captures its response text and produces an **outcome** that feeds into context, transition logic, and downstream nodes. Fabro also tracks every file change per stage, offloads large outputs into content-addressed blob storage, and automatically collects test artifacts like screenshots and reports.
|
||||
When an agent or prompt node finishes, Fabro captures its response text and produces an **outcome** that feeds into context, transition logic, and downstream nodes. Fabro also tracks every file change per stage, offloads large outputs into content-addressed blob storage, and can collect configured artifacts such as screenshots and reports.
|
||||
|
||||
## Response capture
|
||||
|
||||
|
|
@ -246,51 +246,34 @@ For local sandboxes, syncing is a no-op since the agent can already access the h
|
|||
|
||||
## Automatic asset capture
|
||||
|
||||
After each node executes a command, Fabro automatically scans the sandbox for test artifacts — screenshots, videos, reports, and traces — and copies any new or changed files to the run's directory. This happens without any agent or workflow configuration.
|
||||
When `[run.artifacts]` contains include patterns, Fabro scans the sandbox after each stage and captures matching files. Artifact collection is opt-in; no scan runs when the include list is empty.
|
||||
|
||||
### How asset capture works
|
||||
|
||||
1. **Before** the command runs, Fabro takes a baseline snapshot of known artifact paths in the sandbox
|
||||
2. **After** the command completes, Fabro re-scans and diffs against the baseline
|
||||
3. Files that are new or modified since the command started are downloaded to the stage's artifact directory
|
||||
1. Fabro compiles and validates the configured workspace-relative globs.
|
||||
2. The sandbox provider enumerates regular files and their sizes without recursing through symlinks below the workspace root.
|
||||
3. Fabro applies the globs to normalized relative paths, enforces its collection limits, and downloads the selected files.
|
||||
|
||||
Only files modified after the command started are collected. Files that match the baseline fingerprint (same size and mtime) are skipped. Individual files over 10 MB and total collections over 50 MB are also skipped.
|
||||
Each scan represents the post-stage workspace state; Fabro does not depend on filesystem modification timestamps. The same path and content hash is recorded only once per run, even when it still matches after later stages. Individual files over 10 MB are skipped, and each collection is limited to 100 files and 50 MB total.
|
||||
|
||||
### What gets captured
|
||||
|
||||
Fabro looks for files inside these directories:
|
||||
|
||||
| Directory | Typical contents |
|
||||
|---|---|
|
||||
| `playwright-report/` | Playwright HTML reports |
|
||||
| `test-results/` | Playwright screenshots, videos, and traces |
|
||||
| `cypress/videos/` | Cypress test recordings |
|
||||
| `cypress/screenshots/` | Cypress failure screenshots |
|
||||
|
||||
And files matching these filename patterns anywhere in the tree:
|
||||
|
||||
| Pattern | Typical contents |
|
||||
|---|---|
|
||||
| `junit*.xml` | JUnit XML test reports |
|
||||
| `*.trace.zip` | Playwright trace archives |
|
||||
|
||||
Tool caches and dependency directories (`node_modules`, `.cache/ms-playwright`, `.yarn/cache`, etc.) are excluded from scanning.
|
||||
|
||||
### Asset storage layout
|
||||
|
||||
Collected assets are written to the run's directory, organized by node and retry attempt:
|
||||
Configure the paths your workflow produces:
|
||||
|
||||
```toml title="run.toml"
|
||||
[run.artifacts]
|
||||
include = [
|
||||
"test-results/**",
|
||||
"playwright-report/**",
|
||||
"**/*.trace.zip",
|
||||
".ai/reports/*.md",
|
||||
".ai/plans/????-??-??-*.md",
|
||||
]
|
||||
```
|
||||
~/.fabro/scratch/{run_id}/
|
||||
cache/
|
||||
artifacts/
|
||||
files/
|
||||
{node_slug}/
|
||||
retry_1/
|
||||
test-results/
|
||||
screenshot.png
|
||||
video.webm
|
||||
```
|
||||
|
||||
Patterns are rooted at the sandbox working directory. `*` and `?` stay within one path segment, while `**` crosses directories. See [`[run.artifacts]`](/execution/run-configuration#runartifacts) for the complete matching contract.
|
||||
|
||||
Fabro prunes dependency, cache, and build directories including `.git`, `node_modules`, `target`, `.venv`, `.cache`, and `dist`.
|
||||
|
||||
## Observability
|
||||
|
||||
|
|
|
|||
|
|
@ -117,7 +117,9 @@ Finds files matching a glob pattern.
|
|||
| `pattern` | string | yes | Glob pattern to match files (e.g. `**/*.rs`) |
|
||||
| `path` | string | no | Directory to search in (default: working directory) |
|
||||
|
||||
Returns matching file paths, one per line.
|
||||
Returns matching file paths, one per line, sorted lexicographically by their path relative to the search root.
|
||||
|
||||
Patterns are case-sensitive and relative to `path`: `*` and `?` stay within one path segment, bracket expressions such as `[abc]` match one character, and `**` crosses directories when used as a complete segment. Leading dots are matched normally. For example, `*.rs` searches only the root of `path`, and `**/*.rs` searches recursively. Patterns must use `/`, be relative, and cannot contain a backslash or `..` segment.
|
||||
|
||||
<Note>
|
||||
Unlike `grep`, `glob` does **not** mark files as read for the read-before-write guardrail. To modify a file found via glob, the agent must read it first.
|
||||
|
|
|
|||
23
docs/public/changelog/2026-07-25.mdx
Normal file
23
docs/public/changelog/2026-07-25.mdx
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
title: "Consistent workspace globs"
|
||||
date: "2026-07-25"
|
||||
---
|
||||
|
||||
<Warning>
|
||||
**Glob semantics changed.** Artifact patterns are now rooted at the sandbox working directory, and the agent's `glob` tool roots patterns at its `path` argument. Both use the same path-segment semantics.
|
||||
|
||||
- Replace `*.trace.zip` with `**/*.trace.zip` when traces may be nested.
|
||||
- Replace `test-results/**` with `**/test-results/**` only when `test-results` may appear below any directory instead of at the workspace root.
|
||||
- Absolute patterns, backslashes, and patterns containing a `..` segment are now rejected.
|
||||
- Agent `glob` results are now sorted lexicographically; local results are no longer ordered by modification time.
|
||||
</Warning>
|
||||
|
||||
## One glob language in every sandbox
|
||||
|
||||
Fabro now compiles workspace globs once and applies them to normalized relative paths in Rust. Local, Docker, and Daytona sandboxes only enumerate files; provider commands and APIs no longer interpret user patterns.
|
||||
|
||||
`*` and `?` stay within one path segment, bracket expressions match one character, and a complete `**` segment crosses directories. Matching is case-sensitive, leading dots are ordinary characters, and `/` is the separator everywhere. Artifact capture and the agent `glob` tool therefore agree on patterns such as `.ai/reports/*.md`, `.ai/reports/**/*.md`, and `.ai/plans/????-??-??-*.md`.
|
||||
|
||||
Artifact traversal also uses structured provider metadata for file-size limits and prunes dependency, cache, and build directories before matching.
|
||||
|
||||
Artifact collection now treats each post-stage workspace state as authoritative instead of relying on modification timestamps. Fabro records the same path and content hash only once per run and captures the path again when its content changes.
|
||||
|
|
@ -295,6 +295,7 @@
|
|||
"group": "July 2026",
|
||||
"icon": "clock-rotate-left",
|
||||
"pages": [
|
||||
"changelog/2026-07-25",
|
||||
"changelog/2026-07-24",
|
||||
"changelog/2026-07-23",
|
||||
"changelog/2026-07-22",
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ repo_name = "fabro"
|
|||
repo_url = "https://github.com/fabro-sh/fabro"
|
||||
|
||||
[run.artifacts]
|
||||
include = ["test-results/**", "playwright-report/**"]
|
||||
include = ["test-results/**", "playwright-report/**", "**/*.trace.zip"]
|
||||
|
||||
[run.agent.mcps.playwright]
|
||||
type = "sandbox"
|
||||
|
|
@ -416,15 +416,30 @@ Configure automatic collection of test artifacts (Playwright reports, JUnit XML,
|
|||
|
||||
```toml title="run.toml"
|
||||
[run.artifacts]
|
||||
include = ["test-results/**", "playwright-report/**", "*.trace.zip"]
|
||||
include = ["test-results/**", "playwright-report/**", "**/*.trace.zip"]
|
||||
```
|
||||
|
||||
| Field | Description |
|
||||
|---|---|
|
||||
| `include` | Glob patterns for files to collect as assets. Matched against the working directory after each stage completes. |
|
||||
| `include` | Workspace-relative glob patterns for regular files to collect as assets after each stage. |
|
||||
|
||||
Artifact collection is opt-in — when no `[run.artifacts]` section is present, no file scanning occurs.
|
||||
|
||||
Artifact globs use `/` as the separator and have the same semantics in every sandbox:
|
||||
|
||||
- `*` and `?` match within one path segment.
|
||||
- Bracket expressions such as `[abc]` and `[!abc]` match one character.
|
||||
- `**` matches across directories when used as a complete segment.
|
||||
- Leading dots are matched normally.
|
||||
- Patterns use `/`, are relative to the sandbox working directory, and are case-sensitive. Backslashes are invalid.
|
||||
- Absolute patterns and patterns containing a `..` segment are invalid.
|
||||
|
||||
For example, `.ai/reports/*.md` matches direct Markdown children of `.ai/reports`, while `.ai/reports/**/*.md` also matches nested reports. `*.trace.zip` matches only the working-directory root; use `**/*.trace.zip` to match at any depth. To collect date-named implementation plans, use `.ai/plans/????-??-??-*.md`.
|
||||
|
||||
Each collection reflects the post-stage workspace state rather than filesystem modification timestamps. A path with unchanged content is recorded only once per run; if its content changes, Fabro captures the new version.
|
||||
|
||||
Fabro resolves the configured workspace root but does not recurse through symlinks below it while collecting artifacts. Dependency, cache, and build directories such as `.git`, `node_modules`, `target`, `.venv`, `.cache`, and `dist` are pruned. A collection is limited to 100 files, 10 MB per file, and 50 MB total.
|
||||
|
||||
### `[run.agent]`
|
||||
|
||||
Configure workflow agent behavior that is not tied to a single stage.
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ pub trait Sandbox: Send + Sync {
|
|||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<ExecResult, String>;
|
||||
async fn grep(&self, pattern: &str, path: &str, options: &GrepOptions) -> Result<Vec<String>, String>;
|
||||
async fn walk_files(&self, base: &str, relative_start: &str, options: &WalkOptions) -> Result<Vec<SandboxFile>, String>;
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> Result<Vec<String>, String>;
|
||||
async fn initialize(&self) -> Result<(), String>;
|
||||
async fn cleanup(&self) -> Result<(), String>;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ use fabro_types::{
|
|||
SystemActorKind, WorkflowSettings, parse_blob_ref,
|
||||
};
|
||||
use fabro_util::version::FABRO_VERSION;
|
||||
use fabro_util::workspace_glob::{WorkspaceGlob, WorkspaceGlobError};
|
||||
use fabro_workflow::command_log::{command_log_path, read_json_string_blob, read_log_slice};
|
||||
use fabro_workflow::run_status::RunStatus;
|
||||
use fabro_workflow::{Error as WorkflowError, operations};
|
||||
|
|
@ -902,13 +903,31 @@ async fn snapshot_run_variables(
|
|||
state.stores.variables.value_map().await
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum RunVariableSubstitutionError {
|
||||
#[error(transparent)]
|
||||
Interpolation(#[from] ResolveError),
|
||||
|
||||
#[error("run.artifacts.include[{index}]: {source}")]
|
||||
ArtifactGlob {
|
||||
index: usize,
|
||||
#[source]
|
||||
source: WorkspaceGlobError,
|
||||
},
|
||||
}
|
||||
|
||||
fn substitute_run_variables(
|
||||
variables: &HashMap<String, String>,
|
||||
settings: &mut WorkflowSettings,
|
||||
) -> Result<(), ResolveError> {
|
||||
) -> Result<(), RunVariableSubstitutionError> {
|
||||
settings
|
||||
.run
|
||||
.substitute_variables(|name| variables.get(name).cloned())
|
||||
.substitute_variables(|name| variables.get(name).cloned())?;
|
||||
for (index, pattern) in settings.run.artifacts.include.iter().enumerate() {
|
||||
WorkspaceGlob::try_new(pattern)
|
||||
.map_err(|source| RunVariableSubstitutionError::ArtifactGlob { index, source })?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_run_status(
|
||||
|
|
@ -1213,3 +1232,26 @@ fn build_command_log_response(
|
|||
})
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rejects_artifact_glob_that_becomes_unsafe_after_interpolation() {
|
||||
let variables = HashMap::from([("PATTERN".to_string(), "../outside/**".to_string())]);
|
||||
let mut settings = WorkflowSettings::default();
|
||||
settings.run.artifacts.include = vec!["{{ vars.PATTERN }}".to_string()];
|
||||
|
||||
let error = substitute_run_variables(&variables, &mut settings)
|
||||
.expect_err("interpolated parent traversal should be rejected");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
RunVariableSubstitutionError::ArtifactGlob {
|
||||
index: 0,
|
||||
source: WorkspaceGlobError::ParentTraversal { .. },
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ pub trait Sandbox: Send + Sync {
|
|||
async fn write_file(&self, path: &str, content: &str) -> Result<(), String>;
|
||||
async fn exec_command(&self, command: &str, timeout_ms: u64, ...) -> Result<ExecResult, String>;
|
||||
async fn grep(&self, pattern: &str, path: &str, options: &GrepOptions) -> Result<Vec<String>, String>;
|
||||
async fn walk_files(&self, base: &str, relative_start: &str, options: &WalkOptions) -> Result<Vec<SandboxFile>, String>;
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> Result<Vec<String>, String>;
|
||||
// ... plus delete_file, file_exists, list_directory, initialize, cleanup, platform info
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,21 +31,23 @@ exact match and unique unless replace_all is true. If the edit fails with 'old_s
|
|||
re-read the file and take the exact text from the fresh output rather than guessing again. \
|
||||
Preserve existing indentation.";
|
||||
|
||||
const GLOB_DESCRIPTION: &str = "Find files by name using a glob pattern, most recently modified \
|
||||
first.
|
||||
const GLOB_DESCRIPTION: &str = "Find files by search-root-relative path using a glob pattern. \
|
||||
Results are sorted lexicographically by relative path.
|
||||
|
||||
Use this instead of `find` or recursive `ls` through Bash. Prefer patterns with a literal anchor \
|
||||
— an extension or a subdirectory — over bare wildcards.
|
||||
|
||||
Good patterns:
|
||||
- `*.rs` — an extension at any depth below the search root
|
||||
- `*.rs` — direct children of the search root
|
||||
- `**/*.rs` — files at any depth below the search root
|
||||
- `src/*.rs` — directly inside `src/`, not recursive
|
||||
- `src/**/*.rs` — recursive walk under a subdirectory
|
||||
- `{src,tests}/**/*.rs` — brace expansion works
|
||||
- `src/[lm]ib.rs` — a bracket expression matches one character
|
||||
|
||||
Avoid recursing into dependency or build output (`node_modules/**`, `target/**`): those produce \
|
||||
thousands of matches and waste context. Narrow to a specific subpath instead. Results are files, \
|
||||
so to locate a directory, glob for something inside it.";
|
||||
so to locate a directory, glob for something inside it. Patterns must use `/`, be relative, and \
|
||||
cannot contain a `..` segment.";
|
||||
|
||||
pub struct KimiProfile {
|
||||
base: BaseProfile,
|
||||
|
|
@ -345,7 +347,8 @@ mod tests {
|
|||
// Grep must not promise ripgrep syntax: fabro falls back to POSIX grep.
|
||||
let grep = describe("Grep");
|
||||
assert!(grep.contains("POSIX"), "{grep}");
|
||||
assert!(describe("Glob").contains("most recently modified"));
|
||||
assert!(describe("Glob").contains("sorted lexicographically"));
|
||||
assert!(describe("Glob").contains("`*.rs` — direct children"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
// `crate::delegate_sandbox!` invocations continue to work.
|
||||
pub use fabro_sandbox::{
|
||||
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, RefreshOutcome,
|
||||
Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle,
|
||||
StdioProcessTermination, delegate_sandbox, format_lines_numbered, shell_quote,
|
||||
Sandbox, SandboxEvent, SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess,
|
||||
StdioProcessHandle, StdioProcessTermination, WalkOptions, delegate_sandbox,
|
||||
format_lines_numbered, shell_quote,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -489,11 +489,11 @@ pub fn make_glob_tool() -> RegisteredTool {
|
|||
RegisteredTool {
|
||||
definition: ToolDefinition {
|
||||
name: "glob".into(),
|
||||
description: "Find files by file names using a glob pattern. Use path to choose the search root. Prefer this over shell find or ls when locating repository files.".into(),
|
||||
description: "Find files by search-root-relative path using a glob pattern. Use path to choose the search root. `*` stays within one path segment and `**` searches recursively. Prefer this over shell find or ls when locating repository files.".into(),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {"type": "string", "description": "Glob pattern to match files"},
|
||||
"pattern": {"type": "string", "description": "Glob pattern relative to the search root"},
|
||||
"path": {"type": "string", "description": "Directory to search in (default: working directory)"}
|
||||
},
|
||||
"required": ["pattern"]
|
||||
|
|
@ -870,7 +870,8 @@ mod tests {
|
|||
assert!(description("shell").contains("timeout_ms"));
|
||||
assert!(description("grep").contains("regex"));
|
||||
assert!(description("grep").contains("glob_filter"));
|
||||
assert!(description("glob").contains("file names"));
|
||||
assert!(description("glob").contains("search-root-relative"));
|
||||
assert!(description("glob").contains("`**` searches recursively"));
|
||||
assert!(description("web_fetch").contains("http:// or https://"));
|
||||
assert!(description("web_fetch").contains("prompt"));
|
||||
|
||||
|
|
|
|||
|
|
@ -39,9 +39,6 @@ fabro-static.workspace = true
|
|||
fabro-util = { path = "../../foundation/fabro-util" }
|
||||
fabro-redact.workspace = true
|
||||
|
||||
# local
|
||||
glob = { version = "0.3" }
|
||||
|
||||
futures = { workspace = true }
|
||||
|
||||
# docker
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use crate::sandbox;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum CloneDecision {
|
||||
EmptyWorkspace {
|
||||
|
|
@ -32,9 +34,9 @@ pub(crate) fn github_repo_layout(
|
|||
})?;
|
||||
let workspace_root = trim_root(workspace_root);
|
||||
let repos_root = trim_root(repos_root);
|
||||
let repos_owner_path = join_remote_path(repos_root, &owner);
|
||||
let primary_repo_path = join_remote_path(&repos_owner_path, &repo);
|
||||
let primary_repo_link = join_remote_path(workspace_root, &repo);
|
||||
let repos_owner_path = sandbox::join_sandbox_path(repos_root, &owner);
|
||||
let primary_repo_path = sandbox::join_sandbox_path(&repos_owner_path, &repo);
|
||||
let primary_repo_link = sandbox::join_sandbox_path(workspace_root, &repo);
|
||||
|
||||
Ok(GitHubRepoLayout {
|
||||
owner,
|
||||
|
|
@ -51,14 +53,6 @@ fn trim_root(root: &str) -> &str {
|
|||
if trimmed.is_empty() { "/" } else { trimmed }
|
||||
}
|
||||
|
||||
fn join_remote_path(root: &str, name: &str) -> String {
|
||||
if root == "/" {
|
||||
format!("/{name}")
|
||||
} else {
|
||||
format!("{root}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum EmptyWorkspaceReason {
|
||||
SkipClone,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -27,12 +27,13 @@ use tokio_util::sync::CancellationToken;
|
|||
use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason};
|
||||
use crate::redact::redact_auth_url;
|
||||
use crate::sandbox::{
|
||||
BASH_ENV_VAR, BASH_PROBE_MARKER, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, REMOTE_BASH,
|
||||
RefreshOutcome, optional_timeout, resolve_path, validate_bash_probe,
|
||||
self, BASH_ENV_VAR, BASH_PROBE_MARKER, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, REMOTE_BASH,
|
||||
REMOTE_WALK_TIMEOUT_MS, RefreshOutcome, optional_timeout, resolve_path, validate_bash_probe,
|
||||
};
|
||||
use crate::{
|
||||
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox,
|
||||
SandboxEvent, SandboxEventCallback, StdioProcess, glob_match, managed_labels, shell_quote,
|
||||
SandboxEvent, SandboxEventCallback, SandboxFile, StdioProcess, WalkOptions, managed_labels,
|
||||
shell_quote,
|
||||
};
|
||||
|
||||
/// Remediation shown when a Daytona sandbox has no usable Bash.
|
||||
|
|
@ -682,49 +683,6 @@ impl DaytonaSandbox {
|
|||
})
|
||||
}
|
||||
|
||||
async fn list_files_recursive(&self, root: &str) -> crate::Result<Vec<String>> {
|
||||
let sandbox = self.sandbox()?;
|
||||
let fs_svc = sandbox
|
||||
.fs()
|
||||
.await
|
||||
.map_err(|e| crate::Error::context("Failed to get Daytona fs service", e))?;
|
||||
let mut candidates = Vec::new();
|
||||
let mut stack = vec![root.to_string()];
|
||||
let mut visited_dirs = HashSet::new();
|
||||
|
||||
while let Some(dir) = stack.pop() {
|
||||
if !visited_dirs.insert(dir.clone()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let entries = match fs_svc.list_files(&dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(daytona_sdk::DaytonaError::NotFound { .. }) => continue,
|
||||
Err(err) => {
|
||||
return Err(crate::Error::context(
|
||||
format!("Failed to list Daytona directory {dir}"),
|
||||
err,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
for entry in entries {
|
||||
if entry.name.is_empty() || entry.name == "." || entry.name == ".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
let child_path = glob_match::join_path(&dir, &entry.name);
|
||||
if entry.is_dir {
|
||||
stack.push(child_path);
|
||||
} else {
|
||||
candidates.push(child_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
/// Read-only access to the SDK sandbox once initialized. Returns `None`
|
||||
/// before `initialize()` or `reconnect()` has populated the cell.
|
||||
pub fn sandbox_handle(&self) -> Option<&daytona_sdk::Sandbox> {
|
||||
|
|
@ -2081,22 +2039,26 @@ impl Sandbox for DaytonaSandbox {
|
|||
Ok(result.stdout.lines().map(String::from).collect())
|
||||
}
|
||||
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result<Vec<String>> {
|
||||
let base = path.map_or_else(
|
||||
|| self.working_directory().to_string(),
|
||||
|p| self.resolve_path(p),
|
||||
);
|
||||
async fn walk_files(
|
||||
&self,
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
if options.excludes_relative_path(relative_start) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let traversal_root = glob_match::traversal_root(&base, pattern);
|
||||
let matcher = glob_match::GlobMatcher::new(&base, pattern)?;
|
||||
let mut matches = self
|
||||
.list_files_recursive(&traversal_root)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|path| matcher.matches(path))
|
||||
.collect::<Vec<_>>();
|
||||
matches.sort();
|
||||
Ok(matches)
|
||||
let base = self.resolve_path(base);
|
||||
let command = sandbox::build_remote_walk_command(&base, relative_start, options);
|
||||
let result = self
|
||||
.exec_command(&command, REMOTE_WALK_TIMEOUT_MS, None, None, None)
|
||||
.await?;
|
||||
if !result.is_success() {
|
||||
return Err(crate::Error::exec("recursive file traversal", result));
|
||||
}
|
||||
|
||||
sandbox::parse_remote_walk_output(&base, relative_start, &result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,14 +29,15 @@ use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason};
|
|||
use crate::managed_labels::{self, MANAGED_LABEL, RUN_ID_LABEL};
|
||||
use crate::redact::redact_auth_url;
|
||||
use crate::sandbox::{
|
||||
BASH_ENV_VAR, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, REMOTE_BASH, RefreshOutcome,
|
||||
StdioProcessControl, optional_timeout, resolve_path, validate_bash_probe,
|
||||
self, BASH_ENV_VAR, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, REMOTE_BASH,
|
||||
REMOTE_WALK_TIMEOUT_MS, RefreshOutcome, StdioProcessControl, optional_timeout, resolve_path,
|
||||
validate_bash_probe,
|
||||
};
|
||||
use crate::{
|
||||
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
|
||||
ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector,
|
||||
StdioProcess, StdioProcessHandle, StdioProcessTermination, format_lines_numbered, glob_match,
|
||||
shell_quote,
|
||||
ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, SandboxFile,
|
||||
StderrCollector, StdioProcess, StdioProcessHandle, StdioProcessTermination, WalkOptions,
|
||||
format_lines_numbered, shell_quote,
|
||||
};
|
||||
|
||||
const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for every command, with no `sh` fallback; use an \
|
||||
|
|
@ -1881,34 +1882,26 @@ impl Sandbox for DockerSandbox {
|
|||
.collect())
|
||||
}
|
||||
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result<Vec<String>> {
|
||||
let base_dir = path.map_or_else(
|
||||
|| self.working_directory().to_string(),
|
||||
|path| self.resolve_container_path(path),
|
||||
);
|
||||
let traversal_root = glob_match::traversal_root(&base_dir, pattern);
|
||||
let quoted_root = shell_quote(&traversal_root);
|
||||
let command = format!("if [ -e {quoted_root} ]; then find {quoted_root} -type f; fi");
|
||||
let result = self
|
||||
.docker_exec_shell(&command, 30_000, None, None, None)
|
||||
.await?;
|
||||
if !result.is_success() {
|
||||
return Err(crate::Error::message(format!(
|
||||
"glob failed (exit {}): {}",
|
||||
result.display_exit_code(),
|
||||
result.stderr
|
||||
)));
|
||||
async fn walk_files(
|
||||
&self,
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
if options.excludes_relative_path(relative_start) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let matcher = glob_match::GlobMatcher::new(&base_dir, pattern)?;
|
||||
let mut matches = result
|
||||
.stdout
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty() && matcher.matches(line))
|
||||
.map(String::from)
|
||||
.collect::<Vec<_>>();
|
||||
matches.sort();
|
||||
Ok(matches)
|
||||
let base = self.resolve_container_path(base);
|
||||
let command = sandbox::build_remote_walk_command(&base, relative_start, options);
|
||||
let result = self
|
||||
.docker_exec_shell(&command, REMOTE_WALK_TIMEOUT_MS, None, None, None)
|
||||
.await?;
|
||||
if !result.is_success() {
|
||||
return Err(crate::Error::exec("recursive file traversal", result));
|
||||
}
|
||||
|
||||
sandbox::parse_remote_walk_output(&base, relative_start, &result.stdout)
|
||||
}
|
||||
|
||||
fn working_directory(&self) -> &str {
|
||||
|
|
@ -2036,6 +2029,33 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::sandbox::{BASH_PROBE_MARKER, bash_probe_passed};
|
||||
|
||||
#[test]
|
||||
fn remote_walk_command_only_uses_find_for_traversal() {
|
||||
let command = sandbox::build_remote_walk_command("/workspace", ".ai", &WalkOptions {
|
||||
excluded_directory_names: vec!["target".to_string(), "node_modules".to_string()],
|
||||
});
|
||||
|
||||
assert!(command.contains("[ ! -L /workspace/.ai ]"));
|
||||
assert!(command.contains("find -H /workspace/.ai"));
|
||||
assert!(command.contains("-name target"));
|
||||
assert!(command.contains("-name node_modules"));
|
||||
assert!(command.contains("-printf '%s\\0%P\\0'"));
|
||||
assert!(!command.contains("*.md"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_walk_output_is_relative_to_the_declared_base() {
|
||||
let files =
|
||||
sandbox::parse_remote_walk_output("/workspace", ".ai/reports", "12\0result.md\0")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(files, vec![SandboxFile {
|
||||
path: "/workspace/.ai/reports/result.md".to_string(),
|
||||
relative_path: ".ai/reports/result.md".to_string(),
|
||||
size: 12,
|
||||
}]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_run_container_idle_command_uses_non_login_bash() {
|
||||
let config = container_config(&DockerSandboxOptions::default(), None);
|
||||
|
|
|
|||
|
|
@ -1,243 +0,0 @@
|
|||
use std::path::Path;
|
||||
|
||||
pub(crate) struct GlobMatcher {
|
||||
pattern: glob::Pattern,
|
||||
}
|
||||
|
||||
const MATCH_OPTIONS: glob::MatchOptions = glob::MatchOptions {
|
||||
case_sensitive: true,
|
||||
require_literal_separator: true,
|
||||
require_literal_leading_dot: false,
|
||||
};
|
||||
|
||||
impl GlobMatcher {
|
||||
pub(crate) fn new(base: &str, pattern: &str) -> crate::Result<Self> {
|
||||
let full_pattern = full_pattern(base, pattern);
|
||||
let pattern = glob::Pattern::new(&full_pattern)
|
||||
.map_err(|err| crate::Error::context("Invalid glob pattern", err))?;
|
||||
Ok(Self { pattern })
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, path: &str) -> bool {
|
||||
self.pattern.matches_with(path, MATCH_OPTIONS)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn traversal_root(base: &str, pattern: &str) -> String {
|
||||
let full_pattern = full_pattern(base, pattern);
|
||||
literal_traversal_root(&full_pattern)
|
||||
}
|
||||
|
||||
pub(crate) fn join_path(base: &str, path: &str) -> String {
|
||||
if path.is_empty() {
|
||||
return base.to_string();
|
||||
}
|
||||
if is_absolute(path) {
|
||||
return path.to_string();
|
||||
}
|
||||
if base.is_empty() {
|
||||
return path.to_string();
|
||||
}
|
||||
if base == "/" {
|
||||
return format!("/{path}");
|
||||
}
|
||||
format!("{}/{}", base.trim_end_matches('/'), path)
|
||||
}
|
||||
|
||||
fn full_pattern(base: &str, pattern: &str) -> String {
|
||||
if is_absolute(pattern) {
|
||||
pattern.to_string()
|
||||
} else {
|
||||
join_path(base, pattern)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_absolute(path: &str) -> bool {
|
||||
path.starts_with('/') || Path::new(path).is_absolute()
|
||||
}
|
||||
|
||||
fn literal_traversal_root(pattern: &str) -> String {
|
||||
let absolute = pattern.starts_with('/');
|
||||
let mut literal_segments = Vec::new();
|
||||
let mut saw_meta = false;
|
||||
|
||||
for segment in pattern.split('/').filter(|segment| !segment.is_empty()) {
|
||||
if has_glob_meta(segment) {
|
||||
saw_meta = true;
|
||||
break;
|
||||
}
|
||||
literal_segments.push(segment);
|
||||
}
|
||||
|
||||
if saw_meta {
|
||||
return build_path(absolute, &literal_segments);
|
||||
}
|
||||
|
||||
parent_path(&build_path(absolute, &literal_segments))
|
||||
}
|
||||
|
||||
fn has_glob_meta(segment: &str) -> bool {
|
||||
segment
|
||||
.chars()
|
||||
.any(|character| matches!(character, '*' | '?' | '['))
|
||||
}
|
||||
|
||||
fn build_path(absolute: bool, segments: &[&str]) -> String {
|
||||
if segments.is_empty() {
|
||||
return if absolute {
|
||||
"/".to_string()
|
||||
} else {
|
||||
".".to_string()
|
||||
};
|
||||
}
|
||||
|
||||
let joined = segments.join("/");
|
||||
if absolute {
|
||||
format!("/{joined}")
|
||||
} else {
|
||||
joined
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_path(path: &str) -> String {
|
||||
if path == "/" {
|
||||
return "/".to_string();
|
||||
}
|
||||
|
||||
let trimmed = path.trim_end_matches('/');
|
||||
if trimmed.is_empty() {
|
||||
return ".".to_string();
|
||||
}
|
||||
|
||||
match trimmed.rfind('/') {
|
||||
Some(0) => "/".to_string(),
|
||||
Some(index) => trimmed[..index].to_string(),
|
||||
None => ".".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "glob matcher tests stage filesystem fixtures with sync std::fs writes"
|
||||
)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use super::GlobMatcher;
|
||||
|
||||
fn path_string(path: &Path) -> String {
|
||||
path.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
fn match_glob(
|
||||
base: &str,
|
||||
pattern: &str,
|
||||
candidate_paths: &[String],
|
||||
) -> crate::Result<Vec<String>> {
|
||||
let matcher = GlobMatcher::new(base, pattern)?;
|
||||
Ok(candidate_paths
|
||||
.iter()
|
||||
.filter(|path| matcher.matches(path))
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skill_pattern_matches_exactly_one_directory_level() {
|
||||
let candidates = vec![
|
||||
"/workspace/SKILL.md".to_string(),
|
||||
"/workspace/a/SKILL.md".to_string(),
|
||||
"/workspace/a/b/SKILL.md".to_string(),
|
||||
"/workspace/a/README.md".to_string(),
|
||||
];
|
||||
|
||||
let results = match_glob("/workspace", "*/SKILL.md", &candidates).unwrap();
|
||||
|
||||
assert_eq!(results, vec!["/workspace/a/SKILL.md"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn star_matches_only_top_level_files_under_base() {
|
||||
let candidates = vec![
|
||||
"/workspace/a.rs".to_string(),
|
||||
"/workspace/src/lib.rs".to_string(),
|
||||
"/workspace/b.txt".to_string(),
|
||||
];
|
||||
|
||||
let results = match_glob("/workspace", "*.rs", &candidates).unwrap();
|
||||
|
||||
assert_eq!(results, vec!["/workspace/a.rs"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_glob_matches_files_at_any_depth() {
|
||||
let candidates = vec![
|
||||
"/workspace/a.rs".to_string(),
|
||||
"/workspace/src/lib.rs".to_string(),
|
||||
"/workspace/src/nested/main.rs".to_string(),
|
||||
"/workspace/src/nested/readme.md".to_string(),
|
||||
];
|
||||
|
||||
let results = match_glob("/workspace", "**/*.rs", &candidates).unwrap();
|
||||
|
||||
assert_eq!(results, vec![
|
||||
"/workspace/a.rs",
|
||||
"/workspace/src/lib.rs",
|
||||
"/workspace/src/nested/main.rs",
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matcher_matches_glob_crate_on_fixture_patterns() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
let src = root.join("src");
|
||||
let nested = src.join("nested");
|
||||
let skills = root.join("skills");
|
||||
let skill_a = skills.join("a");
|
||||
let skill_b = skill_a.join("b");
|
||||
std::fs::create_dir_all(&nested).unwrap();
|
||||
std::fs::create_dir_all(&skill_b).unwrap();
|
||||
std::fs::write(root.join("a.rs"), "").unwrap();
|
||||
std::fs::write(root.join("b.txt"), "").unwrap();
|
||||
std::fs::write(src.join("lib.rs"), "").unwrap();
|
||||
std::fs::write(nested.join("main.rs"), "").unwrap();
|
||||
std::fs::write(skills.join("SKILL.md"), "").unwrap();
|
||||
std::fs::write(skill_a.join("SKILL.md"), "").unwrap();
|
||||
std::fs::write(skill_b.join("SKILL.md"), "").unwrap();
|
||||
|
||||
let candidates = [
|
||||
root.join("a.rs"),
|
||||
root.join("b.txt"),
|
||||
src.join("lib.rs"),
|
||||
nested.join("main.rs"),
|
||||
skills.join("SKILL.md"),
|
||||
skill_a.join("SKILL.md"),
|
||||
skill_b.join("SKILL.md"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|path| path_string(&path))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for (base, pattern) in [
|
||||
(root, "*.rs"),
|
||||
(root, "**/*.rs"),
|
||||
(root, "src/*.rs"),
|
||||
(skills.as_path(), "*/SKILL.md"),
|
||||
] {
|
||||
let full_pattern = format!("{}/{pattern}", base.display());
|
||||
let mut expected = glob::glob(&full_pattern)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.map(|path| path_string(&path))
|
||||
.collect::<Vec<_>>();
|
||||
expected.sort();
|
||||
|
||||
let mut actual = match_glob(&path_string(base), pattern, &candidates).unwrap();
|
||||
actual.sort();
|
||||
|
||||
assert_eq!(actual, expected, "pattern {full_pattern}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,6 @@ pub mod config;
|
|||
pub mod error;
|
||||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub mod from_environment;
|
||||
mod glob_match;
|
||||
pub mod provider;
|
||||
pub mod sandbox;
|
||||
pub mod sandbox_spec;
|
||||
|
|
@ -54,9 +53,9 @@ pub use reconnect::{reconnect, reconnect_for_run, reconnect_for_run_with_callbac
|
|||
pub use sandbox::{
|
||||
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
|
||||
ExecStreamingResult, GitRunInfo, GitSetupIntent, GrepOptions, RefreshOutcome, Sandbox,
|
||||
SandboxEvent, SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle,
|
||||
StdioProcessTermination, format_lines_numbered, git_push_via_exec, redacted_output_tail,
|
||||
setup_git_via_exec, shell_quote,
|
||||
SandboxEvent, SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess,
|
||||
StdioProcessHandle, StdioProcessTermination, WalkOptions, format_lines_numbered,
|
||||
git_push_via_exec, redacted_output_tail, setup_git_via_exec, shell_quote,
|
||||
};
|
||||
pub use sandbox_spec::SandboxSpec;
|
||||
pub use terminal::{TerminalSession, TerminalSize, open_terminal_for_run};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use fabro_static::EnvVars;
|
||||
|
|
@ -18,8 +18,8 @@ use crate::sandbox::{
|
|||
};
|
||||
use crate::{
|
||||
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
|
||||
ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector,
|
||||
StdioProcess, StdioProcessHandle, StdioProcessTermination, glob_match,
|
||||
ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, SandboxFile,
|
||||
StderrCollector, StdioProcess, StdioProcessHandle, StdioProcessTermination, WalkOptions,
|
||||
};
|
||||
|
||||
/// Remediation shown when the worker has no usable Bash.
|
||||
|
|
@ -726,22 +726,14 @@ impl Sandbox for LocalSandbox {
|
|||
Ok(results)
|
||||
}
|
||||
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result<Vec<String>> {
|
||||
let base_dir =
|
||||
path.map_or_else(|| self.working_directory.clone(), |p| self.resolve_path(p));
|
||||
let base = base_dir.to_string_lossy().into_owned();
|
||||
let traversal_root = PathBuf::from(glob_match::traversal_root(&base, pattern));
|
||||
let matcher = glob_match::GlobMatcher::new(&base, pattern)?;
|
||||
let mut matches = collect_local_files(traversal_root)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|(path, _)| matcher.matches(path))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Sort by mtime (newest first) using the metadata collected during traversal.
|
||||
matches.sort_by_key(|(_, modified)| std::cmp::Reverse(*modified));
|
||||
|
||||
Ok(matches.into_iter().map(|(path, _)| path).collect())
|
||||
async fn walk_files(
|
||||
&self,
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
let base = self.resolve_path(base);
|
||||
walk_local_files(&base, relative_start, options).await
|
||||
}
|
||||
|
||||
async fn download_file_to_local(
|
||||
|
|
@ -981,29 +973,64 @@ where
|
|||
}
|
||||
}
|
||||
|
||||
async fn collect_local_files(root: PathBuf) -> crate::Result<Vec<(String, SystemTime)>> {
|
||||
let mut files = Vec::new();
|
||||
let mut stack = vec![root];
|
||||
async fn walk_local_files(
|
||||
base: &Path,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
if options.excludes_relative_path(relative_start) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
while let Some(path) = stack.pop() {
|
||||
let metadata = match fs::symlink_metadata(&path).await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => {
|
||||
return Err(crate::Error::context(
|
||||
format!("Failed to stat {}", path.display()),
|
||||
err,
|
||||
));
|
||||
}
|
||||
let Some((root, root_metadata)) = resolve_local_walk_root(base, relative_start).await? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut files = Vec::new();
|
||||
let mut stack = vec![(root, Some(root_metadata))];
|
||||
|
||||
while let Some((path, known_metadata)) = stack.pop() {
|
||||
let metadata = match known_metadata {
|
||||
Some(metadata) => metadata,
|
||||
None => match fs::symlink_metadata(&path).await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => {
|
||||
return Err(crate::Error::context(
|
||||
format!("Failed to stat {}", path.display()),
|
||||
err,
|
||||
));
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_file() {
|
||||
files.push((
|
||||
path.to_string_lossy().into_owned(),
|
||||
metadata.modified().unwrap_or(UNIX_EPOCH),
|
||||
));
|
||||
let relative_path = path.strip_prefix(base).map_err(|error| {
|
||||
crate::Error::context(
|
||||
format!(
|
||||
"Failed to make {} relative to {}",
|
||||
path.display(),
|
||||
base.display()
|
||||
),
|
||||
error,
|
||||
)
|
||||
})?;
|
||||
files.push(SandboxFile {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
relative_path: relative_path
|
||||
.to_string_lossy()
|
||||
.replace(std::path::MAIN_SEPARATOR, "/"),
|
||||
size: metadata.len(),
|
||||
});
|
||||
} else if file_type.is_dir() {
|
||||
if path != base
|
||||
&& path
|
||||
.file_name()
|
||||
.and_then(std::ffi::OsStr::to_str)
|
||||
.is_some_and(|file_name| options.excludes_name(file_name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut entries = match fs::read_dir(&path).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
|
|
@ -1020,7 +1047,7 @@ async fn collect_local_files(root: PathBuf) -> crate::Result<Vec<(String, System
|
|||
err,
|
||||
)
|
||||
})? {
|
||||
stack.push(entry.path());
|
||||
stack.push((entry.path(), None));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1028,6 +1055,58 @@ async fn collect_local_files(root: PathBuf) -> crate::Result<Vec<(String, System
|
|||
Ok(files)
|
||||
}
|
||||
|
||||
async fn resolve_local_walk_root(
|
||||
base: &Path,
|
||||
relative_start: &str,
|
||||
) -> crate::Result<Option<(PathBuf, std::fs::Metadata)>> {
|
||||
if relative_start.is_empty() {
|
||||
return match fs::metadata(base).await {
|
||||
Ok(metadata) => Ok(Some((base.to_path_buf(), metadata))),
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
|
||||
) =>
|
||||
{
|
||||
Ok(None)
|
||||
}
|
||||
Err(error) => Err(crate::Error::context(
|
||||
format!("Failed to stat traversal base {}", base.display()),
|
||||
error,
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
let mut root = base.to_path_buf();
|
||||
let mut metadata = None;
|
||||
for segment in relative_start.split('/') {
|
||||
root.push(segment);
|
||||
let component_metadata = match fs::symlink_metadata(&root).await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
|
||||
) =>
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(crate::Error::context(
|
||||
format!("Failed to stat traversal root component {}", root.display()),
|
||||
error,
|
||||
));
|
||||
}
|
||||
};
|
||||
if component_metadata.file_type().is_symlink() {
|
||||
return Ok(None);
|
||||
}
|
||||
metadata = Some(component_metadata);
|
||||
}
|
||||
|
||||
Ok(metadata.map(|metadata| (root, metadata)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
|
|
@ -1810,8 +1889,7 @@ mod tests {
|
|||
std::fs::write(dir.join("src/nested/readme.md"), "").unwrap();
|
||||
|
||||
let env = LocalSandbox::new(dir.clone());
|
||||
let mut results = env.glob("**/*.rs", None).await.unwrap();
|
||||
results.sort();
|
||||
let results = env.glob("**/*.rs", None).await.unwrap();
|
||||
|
||||
assert_eq!(results, vec![
|
||||
dir.join("a.rs").to_string_lossy().into_owned(),
|
||||
|
|
@ -1859,6 +1937,64 @@ mod tests {
|
|||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn glob_follows_the_declared_symlinked_search_root_only() {
|
||||
let parent = temp_dir();
|
||||
let workspace = parent.join("workspace");
|
||||
let outside = parent.join("outside");
|
||||
let search_root = parent.join("workspace-link");
|
||||
std::fs::create_dir_all(&workspace).unwrap();
|
||||
std::fs::create_dir_all(&outside).unwrap();
|
||||
std::fs::write(workspace.join("README.md"), "").unwrap();
|
||||
std::fs::write(outside.join("outside.md"), "").unwrap();
|
||||
std::os::unix::fs::symlink(&outside, workspace.join("linked")).unwrap();
|
||||
std::os::unix::fs::symlink(&workspace, &search_root).unwrap();
|
||||
|
||||
let env = LocalSandbox::new(search_root.clone());
|
||||
let results = env.glob("**/*.md", None).await.unwrap();
|
||||
|
||||
assert_eq!(results, vec![
|
||||
search_root.join("README.md").to_string_lossy().into_owned()
|
||||
]);
|
||||
std::fs::remove_dir_all(&parent).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_files_returns_relative_paths_and_prunes_directories() {
|
||||
let dir = temp_dir();
|
||||
std::fs::create_dir_all(dir.join(".ai/reports")).unwrap();
|
||||
std::fs::create_dir_all(dir.join(".ai/target")).unwrap();
|
||||
std::fs::write(dir.join(".ai/reports/result.md"), "report").unwrap();
|
||||
std::fs::write(dir.join(".ai/reports/empty.md"), "").unwrap();
|
||||
std::fs::write(dir.join(".ai/target/ignored.md"), "ignored").unwrap();
|
||||
|
||||
let sandbox = LocalSandbox::new(dir.clone());
|
||||
let files = sandbox
|
||||
.walk_files(sandbox.working_directory(), ".ai", &WalkOptions {
|
||||
excluded_directory_names: vec!["target".to_string()],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut file_metadata = files
|
||||
.iter()
|
||||
.map(|file| (file.relative_path.as_str(), file.size))
|
||||
.collect::<Vec<_>>();
|
||||
file_metadata.sort_unstable();
|
||||
assert_eq!(file_metadata, vec![
|
||||
(".ai/reports/empty.md", 0),
|
||||
(".ai/reports/result.md", 6),
|
||||
]);
|
||||
let root = dir.to_string_lossy();
|
||||
assert!(
|
||||
files
|
||||
.iter()
|
||||
.all(|file| file.path.starts_with(root.as_ref()))
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_sandbox_download_file_to_local() {
|
||||
let dir = temp_dir();
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use std::time::Duration;
|
|||
use async_trait::async_trait;
|
||||
use fabro_types::{CommandOutputStream, CommandTermination};
|
||||
use fabro_util::shell;
|
||||
use fabro_util::workspace_glob::WorkspaceGlob;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
|
||||
use tokio::sync::Mutex as TokioMutex;
|
||||
|
|
@ -28,6 +29,10 @@ pub(crate) const BASH_PROBE_TIMEOUT_MS: u64 = 10_000;
|
|||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub(crate) const REMOTE_BASH: &str = "/bin/bash";
|
||||
|
||||
/// Timeout for provider-neutral remote file traversal.
|
||||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub(crate) const REMOTE_WALK_TIMEOUT_MS: u64 = 30_000;
|
||||
|
||||
/// Environment variable Bash consults for non-interactive startup source.
|
||||
///
|
||||
/// Sandbox providers must remove or blank this before invoking `bash -c`;
|
||||
|
|
@ -214,6 +219,17 @@ macro_rules! delegate_sandbox {
|
|||
self.$field.glob(pattern, path).await
|
||||
}
|
||||
|
||||
async fn walk_files(
|
||||
&self,
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
options: &$crate::WalkOptions,
|
||||
) -> $crate::Result<Vec<$crate::SandboxFile>> {
|
||||
self.$field
|
||||
.walk_files(base, relative_start, options)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn download_file_to_local(
|
||||
&self,
|
||||
remote_path: &str,
|
||||
|
|
@ -896,6 +912,39 @@ pub struct DirEntry {
|
|||
pub size: Option<u64>,
|
||||
}
|
||||
|
||||
/// A regular file discovered inside a sandbox.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SandboxFile {
|
||||
/// Provider-resolved path accepted by sandbox filesystem operations.
|
||||
pub path: String,
|
||||
/// `/`-separated path relative to the requested traversal base.
|
||||
pub relative_path: String,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// Provider-neutral controls for recursive file traversal.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct WalkOptions {
|
||||
/// Directory basenames that providers must prune at every depth.
|
||||
pub excluded_directory_names: Vec<String>,
|
||||
}
|
||||
|
||||
impl WalkOptions {
|
||||
#[must_use]
|
||||
pub fn excludes_name(&self, name: &str) -> bool {
|
||||
self.excluded_directory_names
|
||||
.iter()
|
||||
.any(|excluded| excluded == name)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn excludes_relative_path(&self, relative_path: &str) -> bool {
|
||||
relative_path
|
||||
.split('/')
|
||||
.any(|segment| self.excludes_name(segment))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct GrepOptions {
|
||||
pub glob_filter: Option<String>,
|
||||
|
|
@ -1031,7 +1080,41 @@ pub trait Sandbox: Send + Sync {
|
|||
path: &str,
|
||||
options: &GrepOptions,
|
||||
) -> crate::Result<Vec<String>>;
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result<Vec<String>>;
|
||||
|
||||
/// Recursively enumerate regular files below a caller-declared base.
|
||||
///
|
||||
/// `relative_start` is a normalized literal directory path relative to
|
||||
/// `base`; it is an optimization boundary, not a matching expression.
|
||||
/// Every returned `relative_path` remains relative to `base`.
|
||||
/// Implementations resolve `base` itself but must not recurse through
|
||||
/// symlinks encountered in `relative_start` or below it.
|
||||
///
|
||||
/// Production providers that support filesystem search must override this.
|
||||
async fn walk_files(
|
||||
&self,
|
||||
_base: &str,
|
||||
_relative_start: &str,
|
||||
_options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
Err(crate::Error::message(
|
||||
"recursive file traversal is not supported by this sandbox",
|
||||
))
|
||||
}
|
||||
|
||||
/// Match a workspace-relative glob using provider-independent semantics.
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result<Vec<String>> {
|
||||
let glob = WorkspaceGlob::try_new(pattern)
|
||||
.map_err(|error| crate::Error::context("Invalid glob pattern", error))?;
|
||||
let base = path.unwrap_or_else(|| self.working_directory());
|
||||
let mut files = self
|
||||
.walk_files(base, glob.traversal_root(), &WalkOptions::default())
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|file| glob.is_match(&file.relative_path))
|
||||
.collect::<Vec<_>>();
|
||||
files.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
|
||||
Ok(files.into_iter().map(|file| file.path).collect())
|
||||
}
|
||||
/// Copy a file from the sandbox to a local filesystem path.
|
||||
/// Handles binary files correctly across all sandbox types.
|
||||
async fn download_file_to_local(
|
||||
|
|
@ -1143,10 +1226,95 @@ pub(crate) fn resolve_path(path: &str, working_dir: &str) -> String {
|
|||
if std::path::Path::new(path).is_absolute() {
|
||||
path.to_string()
|
||||
} else {
|
||||
format!("{working_dir}/{path}")
|
||||
join_sandbox_path(working_dir, path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub(crate) fn join_sandbox_path(base: &str, relative_path: &str) -> String {
|
||||
if relative_path.is_empty() {
|
||||
return base.to_string();
|
||||
}
|
||||
if base.is_empty() {
|
||||
return relative_path.to_string();
|
||||
}
|
||||
if base == "/" {
|
||||
return format!("/{relative_path}");
|
||||
}
|
||||
format!("{}/{relative_path}", base.trim_end_matches('/'))
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub(crate) fn build_remote_walk_command(
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> String {
|
||||
let traversal_root = join_sandbox_path(base, relative_start);
|
||||
let quoted_root = shell_quote(&traversal_root);
|
||||
let mut command = format!("if [ -e {quoted_root} ]");
|
||||
let mut component_path = base.to_string();
|
||||
for segment in relative_start
|
||||
.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
{
|
||||
component_path = join_sandbox_path(&component_path, segment);
|
||||
let _ = write!(command, " && [ ! -L {} ]", shell_quote(&component_path));
|
||||
}
|
||||
let _ = write!(command, "; then find -H {quoted_root}");
|
||||
|
||||
if !options.excluded_directory_names.is_empty() {
|
||||
command.push_str(" \\( -type d \\(");
|
||||
for (index, directory_name) in options.excluded_directory_names.iter().enumerate() {
|
||||
if index > 0 {
|
||||
command.push_str(" -o");
|
||||
}
|
||||
let _ = write!(command, " -name {}", shell_quote(directory_name));
|
||||
}
|
||||
command.push_str(" \\) -prune \\) -o");
|
||||
}
|
||||
|
||||
command.push_str(" -not -type l -type f -printf '%s\\0%P\\0'; fi");
|
||||
command
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "docker", feature = "daytona"))]
|
||||
pub(crate) fn parse_remote_walk_output(
|
||||
base: &str,
|
||||
relative_start: &str,
|
||||
output: &str,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
let mut fields = output.split('\0');
|
||||
let mut files = Vec::new();
|
||||
|
||||
while let Some(size) = fields.next() {
|
||||
if size.is_empty() {
|
||||
break;
|
||||
}
|
||||
let relative_to_start = fields.next().ok_or_else(|| {
|
||||
crate::Error::message("Malformed recursive file traversal output: missing path")
|
||||
})?;
|
||||
let size = size.parse::<u64>().map_err(|error| {
|
||||
crate::Error::context(
|
||||
format!("Malformed recursive file traversal size {size:?}"),
|
||||
error,
|
||||
)
|
||||
})?;
|
||||
let relative_path = if relative_to_start.is_empty() {
|
||||
relative_start.to_string()
|
||||
} else {
|
||||
join_sandbox_path(relative_start, relative_to_start)
|
||||
};
|
||||
files.push(SandboxFile {
|
||||
path: join_sandbox_path(base, &relative_path),
|
||||
relative_path,
|
||||
size,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Shell-quote a string using `shlex::try_quote`, with a fallback for edge
|
||||
/// cases. Re-exported from [`fabro_util::shell::shell_quote`] so sandbox code
|
||||
/// and the config resolve layer share one audited implementation.
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ use tokio_util::sync::CancellationToken;
|
|||
use crate::sandbox::{self, StdioProcessControl};
|
||||
use crate::{
|
||||
DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult, GrepOptions, Sandbox, SandboxEvent,
|
||||
SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle,
|
||||
StdioProcessTermination,
|
||||
SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess, StdioProcessHandle,
|
||||
StdioProcessTermination, WalkOptions,
|
||||
};
|
||||
|
||||
// --- MockSandbox ---
|
||||
|
|
@ -47,6 +47,10 @@ pub struct MockSandbox {
|
|||
/// Fails `exec_command` and `exec_command_streaming` before any process
|
||||
/// runs, so callers see a transport error rather than an `ExecResult`.
|
||||
pub exec_error: Option<String>,
|
||||
/// Files returned by `walk_files`, before traversal-root and exclusion
|
||||
/// filtering.
|
||||
pub walk_files: Vec<SandboxFile>,
|
||||
pub walk_files_error: Option<String>,
|
||||
/// Reported by `exec_command_streaming`. Set to `false` to model a
|
||||
/// provider that cannot separate stdout from stderr.
|
||||
pub streams_separated: bool,
|
||||
|
|
@ -83,6 +87,18 @@ impl MockSandbox {
|
|||
.lock()
|
||||
.expect("stdio_process lock poisoned") = Some(process);
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_walk_files(mut self, files: Vec<SandboxFile>) -> Self {
|
||||
self.walk_files = files;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_walk_files_error(mut self, error: impl Into<String>) -> Self {
|
||||
self.walk_files_error = Some(error.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl MockSandbox {
|
||||
|
|
@ -123,6 +139,8 @@ impl Default for MockSandbox {
|
|||
stdio_process_error: None,
|
||||
stdio_process: Mutex::new(None),
|
||||
exec_error: None,
|
||||
walk_files: Vec::new(),
|
||||
walk_files_error: None,
|
||||
streams_separated: true,
|
||||
}
|
||||
}
|
||||
|
|
@ -343,6 +361,38 @@ impl Sandbox for MockSandbox {
|
|||
Ok(self.glob_results.clone())
|
||||
}
|
||||
|
||||
async fn walk_files(
|
||||
&self,
|
||||
_base: &str,
|
||||
relative_start: &str,
|
||||
options: &WalkOptions,
|
||||
) -> crate::Result<Vec<SandboxFile>> {
|
||||
if let Some(error) = &self.walk_files_error {
|
||||
return Err(crate::Error::message(error.clone()));
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.walk_files
|
||||
.iter()
|
||||
.filter(|file| {
|
||||
relative_start.is_empty()
|
||||
|| file.relative_path == relative_start
|
||||
|| file
|
||||
.relative_path
|
||||
.strip_prefix(relative_start)
|
||||
.is_some_and(|suffix| suffix.starts_with('/'))
|
||||
})
|
||||
.filter(|file| {
|
||||
let parent = file
|
||||
.relative_path
|
||||
.rsplit_once('/')
|
||||
.map_or("", |(parent, _)| parent);
|
||||
!options.excludes_relative_path(parent)
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn download_file_to_local(
|
||||
&self,
|
||||
remote_path: &str,
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
use std::path::Path;
|
||||
|
||||
use fabro_agent::Sandbox;
|
||||
use fabro_sandbox::shell_quote;
|
||||
use fabro_sandbox::{SandboxFile, WalkOptions};
|
||||
use fabro_types::ArtifactUpload;
|
||||
use fabro_util::workspace_glob::WorkspaceGlobSet;
|
||||
use futures::{StreamExt as _, TryStreamExt as _, stream};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::fs;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// A file discovered by the find command.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DiscoveredFile {
|
||||
pub relative_path: String,
|
||||
pub size: u64,
|
||||
pub mtime_epoch_secs: f64,
|
||||
}
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tracing::warn;
|
||||
|
||||
/// Summary of an artifact collection run.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
|
|
@ -26,7 +21,7 @@ pub struct ArtifactCollectionSummary {
|
|||
pub captured_assets: Vec<ArtifactUpload>,
|
||||
}
|
||||
|
||||
/// Directories to exclude from the find search and checkpoint commits.
|
||||
/// Directories to exclude from artifact traversal and checkpoint commits.
|
||||
pub const EXCLUDE_DIRS: &[&str] = &[
|
||||
".git",
|
||||
"node_modules",
|
||||
|
|
@ -53,289 +48,143 @@ const MAX_FILE_SIZE: u64 = 10 * 1024 * 1024;
|
|||
/// Maximum total size for all collected files (50 MB).
|
||||
const MAX_TOTAL_SIZE: u64 = 50 * 1024 * 1024;
|
||||
|
||||
/// Build a platform-aware find command to discover artifact files matching the
|
||||
/// given globs.
|
||||
///
|
||||
/// Globs without `/` are treated as filename patterns (`-name`).
|
||||
/// Globs with `/` are treated as directory patterns: the trailing `/**` (if
|
||||
/// any) is stripped and the remainder is matched via `-path '*/{dir}/*'`.
|
||||
pub fn build_find_command(root: &str, platform: &str, globs: &[String]) -> String {
|
||||
let mut cmd = format!("find -H {}", shell_quote(root));
|
||||
|
||||
// Prune excluded directories
|
||||
let prune_parts: Vec<String> = EXCLUDE_DIRS
|
||||
.iter()
|
||||
.map(|d| format!("-name {}", shell_quote(d)))
|
||||
.collect();
|
||||
cmd.push_str(" \\( ");
|
||||
cmd.push_str(&prune_parts.join(" -o "));
|
||||
cmd.push_str(" \\) -prune -o");
|
||||
|
||||
// Match conditions: not a symlink, is a file, matches user globs
|
||||
cmd.push_str(" -not -type l -type f \\(");
|
||||
|
||||
let mut conditions: Vec<String> = Vec::new();
|
||||
for glob in globs {
|
||||
if glob.contains('/') {
|
||||
// Directory-style glob: strip trailing /** and match as path
|
||||
let dir = glob.trim_end_matches("/**").trim_end_matches("/*");
|
||||
conditions.push(format!(" -path {}", shell_quote(&format!("*/{dir}/*"))));
|
||||
} else {
|
||||
// Filename glob
|
||||
conditions.push(format!(" -name {}", shell_quote(glob)));
|
||||
}
|
||||
}
|
||||
cmd.push_str(&conditions.join(" -o"));
|
||||
cmd.push_str(" \\)");
|
||||
|
||||
// Platform-specific output format
|
||||
match platform {
|
||||
"darwin" => {
|
||||
cmd.push_str(" -exec stat -f '%z %m' {} \\; -print");
|
||||
}
|
||||
_ => {
|
||||
// Linux: use -printf for size, mtime, and relative path
|
||||
cmd.push_str(" -printf '%s\\t%T@\\t%P\\n'");
|
||||
}
|
||||
}
|
||||
|
||||
cmd
|
||||
}
|
||||
|
||||
/// Parse the output of the find command into discovered files.
|
||||
pub fn parse_find_output(output: &str, platform: &str) -> Vec<DiscoveredFile> {
|
||||
match platform {
|
||||
"darwin" => parse_find_output_darwin(output),
|
||||
_ => parse_find_output_linux(output),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_find_output_linux(output: &str) -> Vec<DiscoveredFile> {
|
||||
let mut files = Vec::new();
|
||||
for line in output.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let parts: Vec<&str> = line.splitn(3, '\t').collect();
|
||||
if parts.len() != 3 {
|
||||
continue;
|
||||
}
|
||||
let Ok(size) = parts[0].parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mtime) = parts[1].parse::<f64>() else {
|
||||
continue;
|
||||
};
|
||||
let path = parts[2].to_string();
|
||||
if path.is_empty() {
|
||||
continue;
|
||||
}
|
||||
files.push(DiscoveredFile {
|
||||
relative_path: path,
|
||||
size,
|
||||
mtime_epoch_secs: mtime,
|
||||
});
|
||||
}
|
||||
files
|
||||
}
|
||||
|
||||
fn parse_find_output_darwin(output: &str) -> Vec<DiscoveredFile> {
|
||||
let mut files = Vec::new();
|
||||
let lines: Vec<&str> = output.lines().collect();
|
||||
// Darwin output comes in pairs: "size mtime" then "path"
|
||||
let mut i = 0;
|
||||
while i + 1 < lines.len() {
|
||||
let stat_line = lines[i].trim();
|
||||
let path_line = lines[i + 1].trim();
|
||||
i += 2;
|
||||
|
||||
if stat_line.is_empty() || path_line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let stat_parts: Vec<&str> = stat_line.splitn(2, ' ').collect();
|
||||
if stat_parts.len() != 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Ok(size) = stat_parts[0].parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mtime) = stat_parts[1].parse::<f64>() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
files.push(DiscoveredFile {
|
||||
relative_path: path_line.to_string(),
|
||||
size,
|
||||
mtime_epoch_secs: mtime,
|
||||
});
|
||||
}
|
||||
files
|
||||
}
|
||||
/// Independent traversal roots may run concurrently, but remote providers
|
||||
/// should not receive an unbounded burst of file-walk operations.
|
||||
const MAX_CONCURRENT_ARTIFACT_WALKS: usize = 4;
|
||||
|
||||
/// Select which files should be collected based on size budgets.
|
||||
pub fn select_files_to_collect(
|
||||
discovered: &[DiscoveredFile],
|
||||
_command_start_epoch: f64,
|
||||
) -> Vec<DiscoveredFile> {
|
||||
let mut candidates: Vec<DiscoveredFile> = discovered
|
||||
.iter()
|
||||
.filter(|f| {
|
||||
// Skip oversized files
|
||||
if f.size > MAX_FILE_SIZE {
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
})
|
||||
.cloned()
|
||||
pub fn select_files_to_collect(discovered: Vec<SandboxFile>) -> Vec<SandboxFile> {
|
||||
let mut candidates: Vec<SandboxFile> = discovered
|
||||
.into_iter()
|
||||
.filter(|file| file.size <= MAX_FILE_SIZE)
|
||||
.collect();
|
||||
|
||||
// Sort by size ascending (smallest first)
|
||||
candidates.sort_by_key(|f| f.size);
|
||||
candidates.sort_by(|left, right| {
|
||||
left.size
|
||||
.cmp(&right.size)
|
||||
.then_with(|| left.relative_path.cmp(&right.relative_path))
|
||||
});
|
||||
|
||||
// Enforce total budget and count limit
|
||||
let mut total: u64 = 0;
|
||||
let mut total = 0;
|
||||
let mut selected = Vec::new();
|
||||
for f in candidates {
|
||||
if selected.len() >= MAX_FILE_COUNT {
|
||||
for file in candidates {
|
||||
if selected.len() >= MAX_FILE_COUNT || total + file.size > MAX_TOTAL_SIZE {
|
||||
break;
|
||||
}
|
||||
if total + f.size > MAX_TOTAL_SIZE {
|
||||
break;
|
||||
}
|
||||
total += f.size;
|
||||
selected.push(f);
|
||||
total += file.size;
|
||||
selected.push(file);
|
||||
}
|
||||
|
||||
selected
|
||||
}
|
||||
|
||||
/// Timeout for the find command (30 seconds).
|
||||
const FIND_TIMEOUT_MS: u64 = 30_000;
|
||||
|
||||
/// Normalize discovered file paths to be relative to the working directory.
|
||||
/// On darwin, find outputs absolute paths; on linux, `-printf '%P'` gives
|
||||
/// relative paths. This strips the working directory prefix and leading `./` to
|
||||
/// ensure consistent relative paths.
|
||||
fn normalize_paths(discovered: Vec<DiscoveredFile>, root: &str) -> Vec<DiscoveredFile> {
|
||||
let root_with_slash = if root.ends_with('/') {
|
||||
root.to_string()
|
||||
} else {
|
||||
format!("{root}/")
|
||||
};
|
||||
discovered
|
||||
.into_iter()
|
||||
.map(|mut f| {
|
||||
if let Some(stripped) = f.relative_path.strip_prefix(&root_with_slash) {
|
||||
f.relative_path = stripped.to_string();
|
||||
} else if let Some(stripped) = f.relative_path.strip_prefix(root) {
|
||||
f.relative_path = stripped.strip_prefix('/').unwrap_or(stripped).to_string();
|
||||
}
|
||||
if let Some(stripped) = f.relative_path.strip_prefix("./") {
|
||||
f.relative_path = stripped.to_string();
|
||||
}
|
||||
f
|
||||
})
|
||||
.filter(|f| !f.relative_path.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn compute_artifact_info(
|
||||
relative_path: &str,
|
||||
local_path: &Path,
|
||||
) -> std::result::Result<ArtifactUpload, String> {
|
||||
) -> std::result::Result<Option<ArtifactUpload>, String> {
|
||||
let mime = mime_guess::from_path(relative_path)
|
||||
.first_or_octet_stream()
|
||||
.to_string();
|
||||
let data = fs::read(local_path)
|
||||
let file = fs::File::open(local_path)
|
||||
.await
|
||||
.map_err(|e| format!("failed to read {}: {e}", local_path.display()))?;
|
||||
.map_err(|error| format!("failed to open {}: {error}", local_path.display()))?;
|
||||
let mut data = Vec::new();
|
||||
file.take(MAX_FILE_SIZE + 1)
|
||||
.read_to_end(&mut data)
|
||||
.await
|
||||
.map_err(|error| format!("failed to read {}: {error}", local_path.display()))?;
|
||||
let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX);
|
||||
if bytes > MAX_FILE_SIZE {
|
||||
return Ok(None);
|
||||
}
|
||||
let content_md5 = format!("{:x}", md5::compute(&data));
|
||||
let content_sha256 = hex::encode(Sha256::digest(&data));
|
||||
Ok(ArtifactUpload {
|
||||
Ok(Some(ArtifactUpload {
|
||||
path: relative_path.to_string(),
|
||||
mime,
|
||||
content_md5,
|
||||
content_sha256,
|
||||
bytes,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// Collect artifact files matching the configured globs that were created
|
||||
/// during this stage.
|
||||
/// Collect artifact files matching the configured workspace globs.
|
||||
pub async fn collect_artifacts(
|
||||
sandbox: &dyn Sandbox,
|
||||
artifact_capture_dir: &Path,
|
||||
globs: &[String],
|
||||
command_start_epoch: f64,
|
||||
globs: &WorkspaceGlobSet,
|
||||
) -> Result<ArtifactCollectionSummary, String> {
|
||||
let root = sandbox.working_directory();
|
||||
let platform = sandbox.platform();
|
||||
let cmd = build_find_command(root, platform, globs);
|
||||
|
||||
debug!(cmd = cmd.as_str(), "Collecting artifacts");
|
||||
let result = sandbox
|
||||
.exec_command(&cmd, FIND_TIMEOUT_MS, None, None, None)
|
||||
.await
|
||||
.map_err(|e| e.display_with_causes())?;
|
||||
if !result.is_success() {
|
||||
let stderr = result.stderr.trim();
|
||||
let status = format!(
|
||||
"exit code {}, termination {}",
|
||||
result.display_exit_code(),
|
||||
result.termination.as_str()
|
||||
);
|
||||
return if stderr.is_empty() {
|
||||
Err(format!("find command failed ({status})"))
|
||||
} else {
|
||||
Err(format!("find command failed ({status}): {stderr}"))
|
||||
};
|
||||
}
|
||||
|
||||
let discovered = parse_find_output(&result.stdout, platform);
|
||||
let discovered = normalize_paths(discovered, root);
|
||||
let walk_options = WalkOptions {
|
||||
excluded_directory_names: EXCLUDE_DIRS
|
||||
.iter()
|
||||
.map(|directory| (*directory).to_string())
|
||||
.collect(),
|
||||
};
|
||||
let walk_options = &walk_options;
|
||||
let traversal_roots = globs
|
||||
.traversal_roots()
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>();
|
||||
let walks = stream::iter(traversal_roots)
|
||||
.map(|traversal_root| async move {
|
||||
sandbox
|
||||
.walk_files(sandbox.working_directory(), &traversal_root, walk_options)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"artifact file traversal failed below {traversal_root:?}: {}",
|
||||
error.display_with_causes()
|
||||
)
|
||||
})
|
||||
})
|
||||
.buffer_unordered(MAX_CONCURRENT_ARTIFACT_WALKS)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
let discovered = walks
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|file| globs.is_match(&file.relative_path))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let total_discovered = discovered.len();
|
||||
let to_collect = select_files_to_collect(&discovered, command_start_epoch);
|
||||
let files_skipped = total_discovered - to_collect.len();
|
||||
let to_collect = select_files_to_collect(discovered);
|
||||
let mut files_skipped = total_discovered - to_collect.len();
|
||||
|
||||
let mut files_copied: usize = 0;
|
||||
let mut files_copied = 0;
|
||||
let mut total_bytes: u64 = 0;
|
||||
let mut download_errors: usize = 0;
|
||||
let mut hash_errors: usize = 0;
|
||||
let mut captured_assets: Vec<ArtifactUpload> = Vec::new();
|
||||
let mut download_errors = 0;
|
||||
let mut hash_errors = 0;
|
||||
let mut captured_assets = Vec::new();
|
||||
|
||||
for file in &to_collect {
|
||||
let dest = artifact_capture_dir.join(&file.relative_path);
|
||||
match sandbox
|
||||
.download_file_to_local(&file.relative_path, &dest)
|
||||
.await
|
||||
{
|
||||
match sandbox.download_file_to_local(&file.path, &dest).await {
|
||||
Ok(()) => match compute_artifact_info(&file.relative_path, &dest).await {
|
||||
Ok(info) => {
|
||||
Ok(Some(info)) if total_bytes.saturating_add(info.bytes) <= MAX_TOTAL_SIZE => {
|
||||
files_copied += 1;
|
||||
total_bytes += info.bytes;
|
||||
captured_assets.push(info);
|
||||
}
|
||||
Err(e) => {
|
||||
Ok(Some(_) | None) => {
|
||||
let _ = fs::remove_file(&dest).await;
|
||||
files_skipped += 1;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
path = file.relative_path.as_str(),
|
||||
error = e.as_str(),
|
||||
error = error.as_str(),
|
||||
"Asset hash failed"
|
||||
);
|
||||
let _ = fs::remove_file(&dest).await;
|
||||
hash_errors += 1;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let error = e.display_with_causes();
|
||||
Err(error) => {
|
||||
let rendered = error.display_with_causes();
|
||||
warn!(
|
||||
path = file.relative_path.as_str(),
|
||||
error = error.as_str(),
|
||||
error = rendered.as_str(),
|
||||
"Asset download failed"
|
||||
);
|
||||
download_errors += 1;
|
||||
|
|
@ -356,306 +205,133 @@ pub async fn collect_artifacts(
|
|||
#[cfg(test)]
|
||||
#[expect(clippy::disallowed_methods, reason = "tests write fixtures to disk")]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use fabro_agent::sandbox::ExecResult;
|
||||
use fabro_types::CommandTermination;
|
||||
use fabro_sandbox::test_support::MockSandbox;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Minimal mock sandbox for artifact_snapshot tests.
|
||||
struct AssetMockSandbox {
|
||||
files: HashMap<String, String>,
|
||||
exec_result: ExecResult,
|
||||
working_dir: &'static str,
|
||||
platform_str: &'static str,
|
||||
}
|
||||
|
||||
impl AssetMockSandbox {
|
||||
fn new(files: HashMap<String, String>, exec_stdout: &str, platform: &'static str) -> Self {
|
||||
Self {
|
||||
files,
|
||||
exec_result: ExecResult {
|
||||
stdout: exec_stdout.to_string(),
|
||||
stderr: String::new(),
|
||||
exit_code: Some(0),
|
||||
termination: CommandTermination::Exited,
|
||||
duration_ms: 10,
|
||||
},
|
||||
working_dir: "/home/test",
|
||||
platform_str: platform,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_exec_result(mut self, exec_result: ExecResult) -> Self {
|
||||
self.exec_result = exec_result;
|
||||
self
|
||||
fn sandbox_file(relative_path: &str, size: u64) -> SandboxFile {
|
||||
SandboxFile {
|
||||
path: format!("/home/test/{relative_path}"),
|
||||
relative_path: relative_path.to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Sandbox for AssetMockSandbox {
|
||||
async fn read_file_bytes(&self, _: &str) -> fabro_sandbox::Result<Vec<u8>> {
|
||||
Err("not implemented".into())
|
||||
fn asset_sandbox(contents: HashMap<String, String>) -> MockSandbox {
|
||||
let mut files = HashMap::new();
|
||||
let mut discovered = Vec::new();
|
||||
for (relative_path, content) in contents {
|
||||
let file = sandbox_file(&relative_path, content.len() as u64);
|
||||
files.insert(file.path.clone(), content);
|
||||
discovered.push(file);
|
||||
}
|
||||
async fn write_file(&self, _: &str, _: &str) -> fabro_sandbox::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_file(&self, _: &str) -> fabro_sandbox::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn file_exists(&self, _: &str) -> fabro_sandbox::Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
async fn list_directory(
|
||||
&self,
|
||||
_: &str,
|
||||
_: Option<usize>,
|
||||
) -> fabro_sandbox::Result<Vec<fabro_agent::sandbox::DirEntry>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn exec_command(
|
||||
&self,
|
||||
_: &str,
|
||||
_: u64,
|
||||
_: Option<&str>,
|
||||
_: Option<&std::collections::HashMap<String, String>>,
|
||||
_: Option<tokio_util::sync::CancellationToken>,
|
||||
) -> fabro_sandbox::Result<ExecResult> {
|
||||
Ok(self.exec_result.clone())
|
||||
}
|
||||
async fn grep(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: &fabro_agent::sandbox::GrepOptions,
|
||||
) -> fabro_sandbox::Result<Vec<String>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn glob(&self, _: &str, _: Option<&str>) -> fabro_sandbox::Result<Vec<String>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn download_file_to_local(
|
||||
&self,
|
||||
remote_path: &str,
|
||||
local_path: &std::path::Path,
|
||||
) -> fabro_sandbox::Result<()> {
|
||||
let content = self.files.get(remote_path).ok_or_else(|| {
|
||||
fabro_sandbox::Error::message(format!("File not found: {remote_path}"))
|
||||
})?;
|
||||
if let Some(parent) = local_path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.await
|
||||
.map_err(|e| fabro_sandbox::Error::context("Failed to create dirs", e))?;
|
||||
}
|
||||
fs::write(local_path, content.as_bytes())
|
||||
.await
|
||||
.map_err(|e| fabro_sandbox::Error::context("Failed to write", e))?;
|
||||
Ok(())
|
||||
}
|
||||
async fn upload_file_from_local(
|
||||
&self,
|
||||
_local_path: &std::path::Path,
|
||||
_remote_path: &str,
|
||||
) -> fabro_sandbox::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn initialize(&self) -> fabro_sandbox::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn cleanup(&self) -> fabro_sandbox::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn working_directory(&self) -> &str {
|
||||
self.working_dir
|
||||
}
|
||||
fn platform(&self) -> &str {
|
||||
self.platform_str
|
||||
}
|
||||
fn os_version(&self) -> String {
|
||||
"Linux 6.1.0".into()
|
||||
|
||||
MockSandbox {
|
||||
files,
|
||||
..MockSandbox::linux()
|
||||
}
|
||||
.with_walk_files(discovered)
|
||||
}
|
||||
|
||||
fn workspace_globs(patterns: &[&str]) -> WorkspaceGlobSet {
|
||||
WorkspaceGlobSet::try_new(patterns).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_find_output_linux() {
|
||||
let output = "1024\t1709312400.0\ttest-results/r.xml\n";
|
||||
let files = parse_find_output(output, "linux");
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(files[0].relative_path, "test-results/r.xml");
|
||||
assert_eq!(files[0].size, 1024);
|
||||
assert!((files[0].mtime_epoch_secs - 1_709_312_400.0).abs() < 0.01);
|
||||
}
|
||||
fn select_files_skips_oversized_files() {
|
||||
let selected = select_files_to_collect(vec![sandbox_file("huge.xml", MAX_FILE_SIZE + 1)]);
|
||||
|
||||
#[test]
|
||||
fn parse_find_output_darwin() {
|
||||
let output = "1024 1709312400\n/tmp/test/test-results/r.xml\n";
|
||||
let files = parse_find_output(output, "darwin");
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(files[0].relative_path, "/tmp/test/test-results/r.xml");
|
||||
assert_eq!(files[0].size, 1024);
|
||||
assert!((files[0].mtime_epoch_secs - 1_709_312_400.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_find_output_skips_malformed_lines() {
|
||||
let output = "not-a-number\t1709312400.0\tfile.xml\n\
|
||||
1024\t1709312400.0\ttest-results/good.xml\n\
|
||||
incomplete\n";
|
||||
let files = parse_find_output(output, "linux");
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(files[0].relative_path, "test-results/good.xml");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_files_collects_old_mtime() {
|
||||
let discovered = vec![DiscoveredFile {
|
||||
relative_path: "test-results/old.xml".to_string(),
|
||||
size: 1024,
|
||||
mtime_epoch_secs: 500.0,
|
||||
}];
|
||||
let selected = select_files_to_collect(&discovered, 1000.0);
|
||||
assert_eq!(selected.len(), 1);
|
||||
assert_eq!(selected[0].relative_path, "test-results/old.xml");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_files_skips_oversized() {
|
||||
let discovered = vec![DiscoveredFile {
|
||||
relative_path: "test-results/huge.xml".to_string(),
|
||||
size: MAX_FILE_SIZE + 1,
|
||||
mtime_epoch_secs: 2000.0,
|
||||
}];
|
||||
let selected = select_files_to_collect(&discovered, 1000.0);
|
||||
assert_eq!(selected.len(), 0);
|
||||
assert!(selected.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_files_sorts_smallest_first() {
|
||||
let discovered = vec![
|
||||
DiscoveredFile {
|
||||
relative_path: "a.xml".to_string(),
|
||||
size: 3000,
|
||||
mtime_epoch_secs: 2000.0,
|
||||
},
|
||||
DiscoveredFile {
|
||||
relative_path: "b.xml".to_string(),
|
||||
size: 1000,
|
||||
mtime_epoch_secs: 2000.0,
|
||||
},
|
||||
DiscoveredFile {
|
||||
relative_path: "c.xml".to_string(),
|
||||
size: 2000,
|
||||
mtime_epoch_secs: 2000.0,
|
||||
},
|
||||
sandbox_file("a.xml", 3000),
|
||||
sandbox_file("b.xml", 1000),
|
||||
sandbox_file("c.xml", 2000),
|
||||
];
|
||||
let selected = select_files_to_collect(&discovered, 1000.0);
|
||||
assert_eq!(selected.len(), 3);
|
||||
assert_eq!(selected[0].size, 1000);
|
||||
assert_eq!(selected[1].size, 2000);
|
||||
assert_eq!(selected[2].size, 3000);
|
||||
|
||||
let selected = select_files_to_collect(discovered);
|
||||
|
||||
assert_eq!(
|
||||
selected
|
||||
.iter()
|
||||
.map(|file| file.relative_path.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["b.xml", "c.xml", "a.xml"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_files_enforces_total_budget() {
|
||||
let discovered: Vec<DiscoveredFile> = (0..6)
|
||||
.map(|i| DiscoveredFile {
|
||||
relative_path: format!("file{i}.xml"),
|
||||
size: 9 * 1024 * 1024, // 9 MB each
|
||||
mtime_epoch_secs: 2000.0,
|
||||
})
|
||||
.collect();
|
||||
let selected = select_files_to_collect(&discovered, 1000.0);
|
||||
// 50 MB budget / 9 MB each = 5 fit (45 MB), 6th would be 54 MB
|
||||
let discovered = (0..6)
|
||||
.map(|index| sandbox_file(&format!("file{index}.xml"), 9 * 1024 * 1024))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let selected = select_files_to_collect(discovered);
|
||||
|
||||
assert_eq!(selected.len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_find_command_filename_glob() {
|
||||
let globs = vec!["*.trace.zip".to_string()];
|
||||
let cmd = build_find_command("/workspace", "linux", &globs);
|
||||
assert!(cmd.contains("-name '*.trace.zip'"));
|
||||
assert!(cmd.contains("-printf"));
|
||||
assert!(cmd.contains("-prune"));
|
||||
assert!(cmd.contains("node_modules"));
|
||||
}
|
||||
fn select_files_enforces_count_limit() {
|
||||
let discovered = (0..150)
|
||||
.map(|index| sandbox_file(&format!("file{index}.txt"), 100))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
#[test]
|
||||
fn build_find_command_directory_glob() {
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let cmd = build_find_command("/workspace", "linux", &globs);
|
||||
assert!(cmd.contains("-path '*/test-results/*'"));
|
||||
}
|
||||
let selected = select_files_to_collect(discovered);
|
||||
|
||||
#[test]
|
||||
fn build_find_command_mixed_globs() {
|
||||
let globs = vec![
|
||||
"test-results/**".to_string(),
|
||||
"playwright-report/**".to_string(),
|
||||
"*.trace.zip".to_string(),
|
||||
];
|
||||
let cmd = build_find_command("/workspace", "linux", &globs);
|
||||
assert!(cmd.contains("-path '*/test-results/*'"));
|
||||
assert!(cmd.contains("-path '*/playwright-report/*'"));
|
||||
assert!(cmd.contains("-name '*.trace.zip'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_find_command_shell_quotes_root_and_globs() {
|
||||
let globs = vec!["test result's/**".to_string(), "*.trace zip".to_string()];
|
||||
let cmd = build_find_command("/workspace with spaces", "linux", &globs);
|
||||
assert!(cmd.starts_with(&format!(
|
||||
"find -H {}",
|
||||
shell_quote("/workspace with spaces")
|
||||
)));
|
||||
assert!(cmd.contains(&format!("-path {}", shell_quote("*/test result's/*"))));
|
||||
assert!(cmd.contains(&format!("-name {}", shell_quote("*.trace zip"))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_find_command_darwin() {
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let cmd = build_find_command("/workspace", "darwin", &globs);
|
||||
assert!(cmd.contains("-exec stat -f"));
|
||||
assert!(!cmd.contains("-printf"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_paths_strips_root_prefix() {
|
||||
let files = vec![
|
||||
DiscoveredFile {
|
||||
relative_path: "/workspace/test-results/r.xml".to_string(),
|
||||
size: 100,
|
||||
mtime_epoch_secs: 1000.0,
|
||||
},
|
||||
DiscoveredFile {
|
||||
relative_path: "./test-results/s.xml".to_string(),
|
||||
size: 200,
|
||||
mtime_epoch_secs: 1000.0,
|
||||
},
|
||||
DiscoveredFile {
|
||||
relative_path: "test-results/t.xml".to_string(),
|
||||
size: 300,
|
||||
mtime_epoch_secs: 1000.0,
|
||||
},
|
||||
];
|
||||
let normalized = normalize_paths(files, "/workspace");
|
||||
assert_eq!(normalized[0].relative_path, "test-results/r.xml");
|
||||
assert_eq!(normalized[1].relative_path, "test-results/s.xml");
|
||||
assert_eq!(normalized[2].relative_path, "test-results/t.xml");
|
||||
assert_eq!(selected.len(), MAX_FILE_COUNT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_assets_downloads_and_writes_manifest() {
|
||||
async fn collect_artifacts_matches_workspace_relative_paths() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let contents = HashMap::from([
|
||||
(".ai/reports/summary.md".to_string(), "summary".to_string()),
|
||||
(
|
||||
".ai/reports/nested/ignored.md".to_string(),
|
||||
"nested".to_string(),
|
||||
),
|
||||
(
|
||||
".ai/plans/2026-07-25-globbing.md".to_string(),
|
||||
"plan".to_string(),
|
||||
),
|
||||
(".ai/plans/DRAFTING.md".to_string(), "drafting".to_string()),
|
||||
("README.md".to_string(), "readme".to_string()),
|
||||
]);
|
||||
let sandbox = asset_sandbox(contents);
|
||||
let globs = workspace_globs(&[".ai/reports/*.md", ".ai/plans/????-??-??-*.md"]);
|
||||
|
||||
let mut files = HashMap::new();
|
||||
files.insert("test-results/r.xml".to_string(), "<test/>".to_string());
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mock = AssetMockSandbox::new(files, "1024\t2000.0\ttest-results/r.xml\n", "linux");
|
||||
assert_eq!(summary.files_copied, 2);
|
||||
assert_eq!(
|
||||
summary
|
||||
.captured_assets
|
||||
.iter()
|
||||
.map(|asset| asset.path.as_str())
|
||||
.collect::<BTreeSet<_>>(),
|
||||
BTreeSet::from([".ai/plans/2026-07-25-globbing.md", ".ai/reports/summary.md",])
|
||||
);
|
||||
assert!(!stage_dir.path().join("manifest.json").exists());
|
||||
}
|
||||
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let summary = collect_artifacts(&mock, stage_dir.path(), &globs, 1000.0)
|
||||
#[tokio::test]
|
||||
async fn collect_artifacts_preserves_content_metadata() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let sandbox = asset_sandbox(HashMap::from([(
|
||||
"test-results/r.xml".to_string(),
|
||||
"<test/>".to_string(),
|
||||
)]));
|
||||
let globs = workspace_globs(&["test-results/**"]);
|
||||
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -673,30 +349,28 @@ mod tests {
|
|||
asset.content_sha256,
|
||||
"28e51ddac37391b99c2b9053f1122d0bf84b02365e6fd8c6e8667378bd00f436"
|
||||
);
|
||||
|
||||
// Check that the file was written to the stage dir
|
||||
let dest = stage_dir.path().join("test-results/r.xml");
|
||||
assert!(dest.exists());
|
||||
let content = std::fs::read_to_string(&dest).unwrap();
|
||||
assert_eq!(content, "<test/>");
|
||||
|
||||
// No manifest is written; the durable artifact record lives elsewhere.
|
||||
let manifest = stage_dir.path().join("manifest.json");
|
||||
assert!(!manifest.exists());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(stage_dir.path().join("test-results/r.xml")).unwrap(),
|
||||
"<test/>"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_assets_collects_old_mtime_files() {
|
||||
async fn collect_artifacts_downloads_provider_resolved_paths() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let file = SandboxFile {
|
||||
path: "provider-object:report-1".to_string(),
|
||||
relative_path: "test-results/r.xml".to_string(),
|
||||
size: 7,
|
||||
};
|
||||
let sandbox = MockSandbox {
|
||||
files: HashMap::from([(file.path.clone(), "<test/>".to_string())]),
|
||||
..MockSandbox::linux()
|
||||
}
|
||||
.with_walk_files(vec![file]);
|
||||
let globs = workspace_globs(&["test-results/**"]);
|
||||
|
||||
let mut files = HashMap::new();
|
||||
files.insert("test-results/r.xml".to_string(), "<test/>".to_string());
|
||||
|
||||
// File mtime (500.0) is before command_start_epoch (1000.0)
|
||||
let mock = AssetMockSandbox::new(files, "1024\t500.0\ttest-results/r.xml\n", "linux");
|
||||
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let summary = collect_artifacts(&mock, stage_dir.path(), &globs, 1000.0)
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -705,39 +379,89 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_assets_returns_error_when_find_fails() {
|
||||
async fn collect_artifacts_rechecks_downloaded_file_size() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let mock =
|
||||
AssetMockSandbox::new(HashMap::new(), "", "linux").with_exec_result(ExecResult {
|
||||
stdout: String::new(),
|
||||
stderr: "find: /workspace: Permission denied\n".to_string(),
|
||||
exit_code: Some(1),
|
||||
termination: CommandTermination::Exited,
|
||||
duration_ms: 10,
|
||||
});
|
||||
let content = "x".repeat(usize::try_from(MAX_FILE_SIZE + 1).unwrap());
|
||||
let file = sandbox_file("test-results/grew.bin", 1);
|
||||
let sandbox = MockSandbox {
|
||||
files: HashMap::from([(file.path.clone(), content)]),
|
||||
..MockSandbox::linux()
|
||||
}
|
||||
.with_walk_files(vec![file]);
|
||||
let globs = workspace_globs(&["test-results/**"]);
|
||||
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let error = collect_artifacts(&mock, stage_dir.path(), &globs, 1000.0)
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.expect_err("nonzero find exit should fail artifact collection");
|
||||
.unwrap();
|
||||
|
||||
assert!(error.contains("find command failed"), "{error}");
|
||||
assert!(error.contains("Permission denied"), "{error}");
|
||||
assert_eq!(summary.files_copied, 0);
|
||||
assert_eq!(summary.files_skipped, 1);
|
||||
assert!(summary.captured_assets.is_empty());
|
||||
assert!(!stage_dir.path().join("test-results/grew.bin").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_assets_non_fatal_on_download_error() {
|
||||
async fn collect_artifacts_prunes_dependency_and_build_directories() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let sandbox = asset_sandbox(HashMap::from([
|
||||
(".ai/reports/keep.md".to_string(), "keep".to_string()),
|
||||
("target/report.md".to_string(), "target".to_string()),
|
||||
(
|
||||
"nested/node_modules/report.md".to_string(),
|
||||
"dependency".to_string(),
|
||||
),
|
||||
]));
|
||||
let globs = workspace_globs(&["**/*.md"]);
|
||||
|
||||
// Don't add the file to the mock files map — download will fail
|
||||
let mock = AssetMockSandbox::new(
|
||||
HashMap::new(),
|
||||
"100\t2000.0\ttest-results/missing.xml\n200\t2000.0\ttest-results/also-missing.xml\n",
|
||||
"linux",
|
||||
);
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let globs = vec!["test-results/**".to_string()];
|
||||
let summary = collect_artifacts(&mock, stage_dir.path(), &globs, 1000.0)
|
||||
assert_eq!(summary.files_copied, 1);
|
||||
assert_eq!(summary.captured_assets[0].path, ".ai/reports/keep.md");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_artifacts_deduplicates_overlapping_patterns() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let sandbox = asset_sandbox(HashMap::from([(
|
||||
".ai/reports/summary.md".to_string(),
|
||||
"summary".to_string(),
|
||||
)]));
|
||||
let globs = workspace_globs(&[".ai/**/*.md", ".ai/reports/*.md"]);
|
||||
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(summary.files_copied, 1);
|
||||
assert_eq!(summary.captured_assets.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_artifacts_reports_traversal_errors() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let sandbox = asset_sandbox(HashMap::new()).with_walk_files_error("permission denied");
|
||||
let globs = workspace_globs(&["test-results/**"]);
|
||||
|
||||
let error = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.expect_err("failed traversal should fail artifact collection");
|
||||
|
||||
assert!(error.contains("artifact file traversal failed"), "{error}");
|
||||
assert!(error.contains("permission denied"), "{error}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_artifacts_keeps_download_errors_non_fatal() {
|
||||
let stage_dir = tempfile::tempdir().unwrap();
|
||||
let sandbox = asset_sandbox(HashMap::new()).with_walk_files(vec![
|
||||
sandbox_file("test-results/missing.xml", 100),
|
||||
sandbox_file("test-results/also-missing.xml", 200),
|
||||
]);
|
||||
let globs = workspace_globs(&["test-results/**"]);
|
||||
|
||||
let summary = collect_artifacts(&sandbox, stage_dir.path(), &globs)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -745,37 +469,4 @@ mod tests {
|
|||
assert_eq!(summary.download_errors, 2);
|
||||
assert_eq!(summary.hash_errors, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_files_enforces_count_limit() {
|
||||
// Create 150 small, recent files — should be capped at MAX_FILE_COUNT (100)
|
||||
let discovered: Vec<DiscoveredFile> = (0..150)
|
||||
.map(|i| DiscoveredFile {
|
||||
relative_path: format!("file{i}.txt"),
|
||||
size: 100, // tiny files, well within total budget
|
||||
mtime_epoch_secs: 2000.0,
|
||||
})
|
||||
.collect();
|
||||
let selected = select_files_to_collect(&discovered, 1000.0);
|
||||
assert_eq!(selected.len(), MAX_FILE_COUNT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_find_command_excludes_venv() {
|
||||
let globs = vec!["*.xml".to_string()];
|
||||
let cmd = build_find_command("/workspace", "linux", &globs);
|
||||
assert!(cmd.contains(".venv"), "expected .venv in prune clause");
|
||||
assert!(cmd.contains("venv"), "expected venv in prune clause");
|
||||
assert!(cmd.contains(".cache"), "expected .cache in prune clause");
|
||||
assert!(cmd.contains(".tox"), "expected .tox in prune clause");
|
||||
assert!(
|
||||
cmd.contains(".pytest_cache"),
|
||||
"expected .pytest_cache in prune clause"
|
||||
);
|
||||
assert!(
|
||||
cmd.contains(".mypy_cache"),
|
||||
"expected .mypy_cache in prune clause"
|
||||
);
|
||||
assert!(cmd.contains("dist"), "expected dist in prune clause");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,13 +6,15 @@ use anyhow::{Context as _, Result, anyhow};
|
|||
use async_trait::async_trait;
|
||||
use fabro_core::error::{Error as CoreError, Result as CoreResult};
|
||||
use fabro_core::graph::NodeSpec;
|
||||
use fabro_core::lifecycle::{AttemptContext, AttemptResultContext, NodeDecision, RunLifecycle};
|
||||
use fabro_core::lifecycle::{AttemptResultContext, RunLifecycle};
|
||||
use fabro_core::outcome::NodeResult;
|
||||
use fabro_core::state::ExecutionState;
|
||||
use fabro_store::{ArtifactKey, ArtifactStore};
|
||||
use fabro_types::{ArtifactUpload, EventBody, RunId, StageId};
|
||||
use fabro_util::error::collect_chain;
|
||||
use fabro_util::workspace_glob::{WorkspaceGlobError, WorkspaceGlobSet};
|
||||
use tokio::fs;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env};
|
||||
|
|
@ -27,7 +29,6 @@ use crate::stage_execution::StageExecutionTracker;
|
|||
|
||||
type WfRunState = ExecutionState<Option<BilledModelUsage>>;
|
||||
type WfNodeResult = NodeResult<Option<BilledModelUsage>>;
|
||||
type WfNodeDecision = NodeDecision<Option<BilledModelUsage>>;
|
||||
type ArtifactIdentity = (String, String);
|
||||
|
||||
const ARTIFACT_UPLOAD_RETRY_DELAYS: [Duration; 3] = [
|
||||
|
|
@ -38,17 +39,16 @@ const ARTIFACT_UPLOAD_RETRY_DELAYS: [Duration; 3] = [
|
|||
|
||||
/// Sub-lifecycle responsible for artifact collection, offloading, and syncing.
|
||||
pub(crate) struct ArtifactLifecycle {
|
||||
pub sandbox: Arc<dyn fabro_sandbox::Sandbox>,
|
||||
pub run_store: RunStoreHandle,
|
||||
pub emitter: Arc<Emitter>,
|
||||
pub run_id: RunId,
|
||||
pub artifact_globs: Vec<String>,
|
||||
pub artifact_sink: Option<ArtifactSink>,
|
||||
/// Per-attempt state: epoch seconds when the attempt started.
|
||||
attempt_start_epoch: std::sync::Mutex<Option<f64>>,
|
||||
captured_artifacts: std::sync::Mutex<HashSet<ArtifactIdentity>>,
|
||||
pub sandbox: Arc<dyn fabro_sandbox::Sandbox>,
|
||||
pub run_store: RunStoreHandle,
|
||||
pub emitter: Arc<Emitter>,
|
||||
pub run_id: RunId,
|
||||
artifact_globs: std::result::Result<WorkspaceGlobSet, WorkspaceGlobError>,
|
||||
pub artifact_sink: Option<ArtifactSink>,
|
||||
captured_artifacts: std::sync::Mutex<HashSet<ArtifactIdentity>>,
|
||||
ledger_initialized: OnceCell<()>,
|
||||
/// Run-scoped stage execution allocator shared with `RunServices`.
|
||||
stage_executions: StageExecutionTracker,
|
||||
stage_executions: StageExecutionTracker,
|
||||
}
|
||||
|
||||
impl ArtifactLifecycle {
|
||||
|
|
@ -57,7 +57,7 @@ impl ArtifactLifecycle {
|
|||
run_store: RunStoreHandle,
|
||||
emitter: Arc<Emitter>,
|
||||
run_id: RunId,
|
||||
artifact_globs: Vec<String>,
|
||||
artifact_globs: &[String],
|
||||
artifact_sink: Option<ArtifactSink>,
|
||||
stage_executions: StageExecutionTracker,
|
||||
) -> Self {
|
||||
|
|
@ -66,64 +66,57 @@ impl ArtifactLifecycle {
|
|||
run_store,
|
||||
emitter,
|
||||
run_id,
|
||||
artifact_globs,
|
||||
artifact_globs: WorkspaceGlobSet::try_new(artifact_globs),
|
||||
artifact_sink,
|
||||
attempt_start_epoch: std::sync::Mutex::new(None),
|
||||
captured_artifacts: std::sync::Mutex::new(HashSet::new()),
|
||||
ledger_initialized: OnceCell::new(),
|
||||
stage_executions,
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_globs(&self) -> CoreResult<&WorkspaceGlobSet> {
|
||||
self.artifact_globs.as_ref().map_err(|error| {
|
||||
CoreError::Other(format!("invalid run.artifacts.include pattern: {error}"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
|
||||
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
|
||||
*self.attempt_start_epoch.lock().expect(
|
||||
"artifact mutex should not be poisoned: no code panics while holding this lock",
|
||||
) = None;
|
||||
let ledger = self
|
||||
.rebuild_captured_artifact_ledger()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let rendered = collect_chain(err.as_ref()).join(": ");
|
||||
CoreError::Other(format!(
|
||||
"failed to rebuild captured artifact ledger: {rendered}"
|
||||
))
|
||||
})?;
|
||||
*self.captured_artifacts.lock().expect(
|
||||
"artifact mutex should not be poisoned: no code panics while holding this lock",
|
||||
) = ledger;
|
||||
let artifact_globs = self.artifact_globs()?;
|
||||
if artifact_globs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.ledger_initialized
|
||||
.get_or_try_init(|| async {
|
||||
let ledger = self
|
||||
.rebuild_captured_artifact_ledger()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let rendered = collect_chain(err.as_ref()).join(": ");
|
||||
CoreError::Other(format!(
|
||||
"failed to rebuild captured artifact ledger: {rendered}"
|
||||
))
|
||||
})?;
|
||||
*self.captured_artifacts.lock().expect(
|
||||
"artifact mutex should not be poisoned: no code panics while holding this lock",
|
||||
) = ledger;
|
||||
Ok::<(), CoreError>(())
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn before_attempt(
|
||||
&self,
|
||||
_ctx: &AttemptContext<'_, WorkflowGraph>,
|
||||
_state: &WfRunState,
|
||||
) -> CoreResult<WfNodeDecision> {
|
||||
// Record epoch seconds (floored to integer for macOS stat mtime parity)
|
||||
let epoch = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0.0, |d| d.as_secs() as f64);
|
||||
*self.attempt_start_epoch.lock().expect(
|
||||
"artifact mutex should not be poisoned: no code panics while holding this lock",
|
||||
) = Some(epoch);
|
||||
Ok(NodeDecision::Continue)
|
||||
}
|
||||
|
||||
async fn after_attempt(
|
||||
&self,
|
||||
ctx: &AttemptResultContext<'_, WorkflowGraph>,
|
||||
state: &WfRunState,
|
||||
) -> CoreResult<()> {
|
||||
if self.artifact_globs.is_empty() {
|
||||
let artifact_globs = self.artifact_globs()?;
|
||||
if artifact_globs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let epoch = self
|
||||
.attempt_start_epoch
|
||||
.lock()
|
||||
.expect("artifact mutex should not be poisoned: no code panics while holding this lock")
|
||||
.unwrap_or(0.0);
|
||||
let node_id = ctx.node.id();
|
||||
// Artifact identity follows the stage execution ordinal so a resumed
|
||||
// reexecution stores its captures under the new `StageId`.
|
||||
|
|
@ -137,14 +130,7 @@ impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
|
|||
let artifact_capture_dir =
|
||||
tempfile::tempdir().map_err(|err| CoreError::Other(err.to_string()))?;
|
||||
|
||||
match collect_artifacts(
|
||||
&*self.sandbox,
|
||||
artifact_capture_dir.path(),
|
||||
&self.artifact_globs,
|
||||
epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
match collect_artifacts(&*self.sandbox, artifact_capture_dir.path(), artifact_globs).await {
|
||||
Ok(summary) => {
|
||||
self.emit_collection_problem_notice(node_id, &summary);
|
||||
let new_assets = self.new_captured_assets(&summary.captured_assets);
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ impl WorkflowLifecycle {
|
|||
run_store.clone(),
|
||||
Arc::clone(emitter),
|
||||
run_options.run_id,
|
||||
run_options.artifact_globs(),
|
||||
run_options.artifact_glob_patterns(),
|
||||
artifact_sink,
|
||||
stage_executions.clone(),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -58,8 +58,8 @@ impl RunOptions {
|
|||
git_author_from_settings(&self.settings)
|
||||
}
|
||||
|
||||
pub fn artifact_globs(&self) -> Vec<String> {
|
||||
self.settings.run.artifacts.include.clone()
|
||||
pub fn artifact_glob_patterns(&self) -> &[String] {
|
||||
&self.settings.run.artifacts.include
|
||||
}
|
||||
|
||||
/// Run branch name from git checkpoint options, if set.
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ use fabro_types::settings::run::{
|
|||
RunIntegrationsSettings, RunInterviewsSettings, RunMetaBranchSettings, RunModelControls,
|
||||
RunModelSettings, RunNamespace, RunPrepareSettings, RunScmSettings, ScmGitHubSettings, TlsMode,
|
||||
};
|
||||
use fabro_util::workspace_glob::WorkspaceGlob;
|
||||
|
||||
use super::{ResolveError, resolve_run_environment};
|
||||
use crate::{
|
||||
|
|
@ -83,7 +84,7 @@ pub fn resolve_run(
|
|||
.collect(),
|
||||
scm: resolve_scm(layer.scm.as_ref()),
|
||||
pull_request,
|
||||
artifacts: resolve_artifacts(layer.artifacts.as_ref()),
|
||||
artifacts: resolve_artifacts(layer.artifacts.as_ref(), errors),
|
||||
integrations: resolve_integrations(layer.integrations.as_ref()),
|
||||
}
|
||||
}
|
||||
|
|
@ -646,7 +647,21 @@ fn resolve_pull_request(pull_request: Option<&RunPullRequestLayer>) -> Option<Pu
|
|||
})
|
||||
}
|
||||
|
||||
fn resolve_artifacts(artifacts: Option<&RunArtifactsLayer>) -> ArtifactsSettings {
|
||||
fn resolve_artifacts(
|
||||
artifacts: Option<&RunArtifactsLayer>,
|
||||
errors: &mut Vec<ResolveError>,
|
||||
) -> ArtifactsSettings {
|
||||
if let Some(artifacts) = artifacts {
|
||||
for (index, pattern) in artifacts.include.iter().enumerate() {
|
||||
if let Err(error) = WorkspaceGlob::try_new(pattern) {
|
||||
errors.push(ResolveError::Invalid {
|
||||
path: format!("run.artifacts.include[{index}]"),
|
||||
reason: error.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ArtifactsSettings {
|
||||
include: artifacts
|
||||
.map(|artifacts| artifacts.include.clone())
|
||||
|
|
|
|||
|
|
@ -60,6 +60,45 @@ fn run_model_controls_default_to_none() {
|
|||
assert!(settings.model.controls.speed.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_artifact_globs_are_validated_during_resolve() {
|
||||
let error = workflow_settings_from_toml(
|
||||
r#"
|
||||
_version = 1
|
||||
|
||||
[run.artifacts]
|
||||
include = ["/tmp/*.md", "../reports/*.md", "src/[abc"]
|
||||
"#,
|
||||
)
|
||||
.expect_err("invalid artifact workspace globs should not resolve");
|
||||
|
||||
let errors = match error {
|
||||
crate::Error::Resolve { errors, .. } => errors,
|
||||
other => panic!("expected structured resolve errors, got {other:#}"),
|
||||
};
|
||||
let invalid = errors
|
||||
.into_iter()
|
||||
.map(|error| match error {
|
||||
crate::ResolveError::Invalid { path, reason } => (path, reason),
|
||||
other => panic!("expected invalid-value error, got {other}"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
invalid
|
||||
.iter()
|
||||
.map(|(path, _)| path.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
"run.artifacts.include[0]",
|
||||
"run.artifacts.include[1]",
|
||||
"run.artifacts.include[2]",
|
||||
]
|
||||
);
|
||||
assert!(invalid[0].1.contains("must be relative"));
|
||||
assert!(invalid[1].1.contains("parent directory"));
|
||||
assert!(invalid[2].1.contains("invalid workspace glob"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_run_defaults_from_empty_settings() {
|
||||
let settings = super::workflow_settings_from_layer(SettingsLayer::default())
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ dirs.workspace = true
|
|||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
anyhow.workspace = true
|
||||
glob.workspace = true
|
||||
thiserror.workspace = true
|
||||
open = "5"
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ pub mod text;
|
|||
pub mod time;
|
||||
pub mod version;
|
||||
pub mod warnings;
|
||||
pub mod workspace_glob;
|
||||
|
||||
#[doc(hidden)]
|
||||
pub use console;
|
||||
|
|
|
|||
326
lib/foundation/fabro-util/src/workspace_glob.rs
Normal file
326
lib/foundation/fabro-util/src/workspace_glob.rs
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
|
||||
const MATCH_OPTIONS: glob::MatchOptions = glob::MatchOptions {
|
||||
case_sensitive: true,
|
||||
require_literal_separator: true,
|
||||
require_literal_leading_dot: false,
|
||||
};
|
||||
|
||||
/// A validated glob matched against normalized paths relative to a workspace
|
||||
/// root.
|
||||
///
|
||||
/// Patterns always use `/` as their separator. `*` and `?` stay within one
|
||||
/// path segment; `**` crosses directory boundaries.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WorkspaceGlob {
|
||||
pattern: glob::Pattern,
|
||||
traversal_root: String,
|
||||
}
|
||||
|
||||
impl WorkspaceGlob {
|
||||
pub fn try_new(source: &str) -> Result<Self, WorkspaceGlobError> {
|
||||
let source = strip_current_dir_prefix(source);
|
||||
if source.is_empty() {
|
||||
return Err(WorkspaceGlobError::Empty);
|
||||
}
|
||||
if source.contains('\\') {
|
||||
return Err(WorkspaceGlobError::BackslashSeparator {
|
||||
pattern: source.to_string(),
|
||||
});
|
||||
}
|
||||
if is_absolute(source) {
|
||||
return Err(WorkspaceGlobError::Absolute {
|
||||
pattern: source.to_string(),
|
||||
});
|
||||
}
|
||||
if source.split('/').any(|segment| segment == "..") {
|
||||
return Err(WorkspaceGlobError::ParentTraversal {
|
||||
pattern: source.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let pattern =
|
||||
glob::Pattern::new(source).map_err(|source_error| WorkspaceGlobError::Syntax {
|
||||
pattern: source.to_string(),
|
||||
source: source_error,
|
||||
})?;
|
||||
|
||||
Ok(Self {
|
||||
pattern,
|
||||
traversal_root: literal_traversal_root(source),
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_match(&self, relative_path: &str) -> bool {
|
||||
let relative_path = normalize_candidate(relative_path);
|
||||
let relative_path = strip_current_dir_prefix(&relative_path);
|
||||
!is_absolute(relative_path)
|
||||
&& !relative_path.split('/').any(|segment| segment == "..")
|
||||
&& self.pattern.matches_with(relative_path, MATCH_OPTIONS)
|
||||
}
|
||||
|
||||
/// A literal directory prefix that can reduce traversal work.
|
||||
///
|
||||
/// This is only an optimization. Callers must still apply
|
||||
/// [`Self::is_match`] to every returned candidate.
|
||||
#[must_use]
|
||||
pub fn traversal_root(&self) -> &str {
|
||||
&self.traversal_root
|
||||
}
|
||||
}
|
||||
|
||||
/// A compiled union of [`WorkspaceGlob`] patterns.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct WorkspaceGlobSet {
|
||||
patterns: Vec<WorkspaceGlob>,
|
||||
}
|
||||
|
||||
impl WorkspaceGlobSet {
|
||||
pub fn try_new<I, S>(patterns: I) -> Result<Self, WorkspaceGlobError>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
let patterns = patterns
|
||||
.into_iter()
|
||||
.map(|pattern| WorkspaceGlob::try_new(pattern.as_ref()))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(Self { patterns })
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.patterns.is_empty()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_match(&self, relative_path: &str) -> bool {
|
||||
self.patterns
|
||||
.iter()
|
||||
.any(|pattern| pattern.is_match(relative_path))
|
||||
}
|
||||
|
||||
/// Return the smallest non-overlapping set of literal traversal roots.
|
||||
#[must_use]
|
||||
pub fn traversal_roots(&self) -> Vec<&str> {
|
||||
let mut roots = self
|
||||
.patterns
|
||||
.iter()
|
||||
.map(WorkspaceGlob::traversal_root)
|
||||
.collect::<Vec<_>>();
|
||||
roots.sort_by(|left, right| {
|
||||
segment_count(left)
|
||||
.cmp(&segment_count(right))
|
||||
.then_with(|| left.cmp(right))
|
||||
});
|
||||
roots.dedup();
|
||||
|
||||
let mut selected: Vec<&str> = Vec::new();
|
||||
for root in roots {
|
||||
if !selected
|
||||
.iter()
|
||||
.any(|ancestor| is_same_or_ancestor(ancestor, root))
|
||||
{
|
||||
selected.push(root);
|
||||
}
|
||||
}
|
||||
selected
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WorkspaceGlobError {
|
||||
#[error("workspace glob cannot be empty")]
|
||||
Empty,
|
||||
|
||||
#[error("workspace glob must use '/' as its path separator: {pattern:?}")]
|
||||
BackslashSeparator { pattern: String },
|
||||
|
||||
#[error("workspace glob must be relative: {pattern:?}")]
|
||||
Absolute { pattern: String },
|
||||
|
||||
#[error("workspace glob cannot traverse to a parent directory: {pattern:?}")]
|
||||
ParentTraversal { pattern: String },
|
||||
|
||||
#[error("invalid workspace glob {pattern:?}: {source}")]
|
||||
Syntax {
|
||||
pattern: String,
|
||||
#[source]
|
||||
source: glob::PatternError,
|
||||
},
|
||||
}
|
||||
|
||||
fn strip_current_dir_prefix(mut path: &str) -> &str {
|
||||
while let Some(stripped) = path.strip_prefix("./") {
|
||||
path = stripped;
|
||||
}
|
||||
path
|
||||
}
|
||||
|
||||
fn is_absolute(path: &str) -> bool {
|
||||
let bytes = path.as_bytes();
|
||||
path.starts_with('/')
|
||||
|| Path::new(path).is_absolute()
|
||||
|| matches!(bytes, [drive, b':', ..] if drive.is_ascii_alphabetic())
|
||||
}
|
||||
|
||||
fn literal_traversal_root(pattern: &str) -> String {
|
||||
let mut literal_segments = Vec::new();
|
||||
let mut saw_meta = false;
|
||||
|
||||
for segment in pattern.split('/').filter(|segment| !segment.is_empty()) {
|
||||
if has_glob_meta(segment) {
|
||||
saw_meta = true;
|
||||
break;
|
||||
}
|
||||
literal_segments.push(segment);
|
||||
}
|
||||
|
||||
if !saw_meta {
|
||||
literal_segments.pop();
|
||||
}
|
||||
literal_segments.join("/")
|
||||
}
|
||||
|
||||
fn has_glob_meta(segment: &str) -> bool {
|
||||
segment
|
||||
.chars()
|
||||
.any(|character| matches!(character, '*' | '?' | '['))
|
||||
}
|
||||
|
||||
fn segment_count(path: &str) -> usize {
|
||||
path.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.count()
|
||||
}
|
||||
|
||||
fn is_same_or_ancestor(ancestor: &str, candidate: &str) -> bool {
|
||||
ancestor.is_empty()
|
||||
|| ancestor == candidate
|
||||
|| candidate
|
||||
.strip_prefix(ancestor)
|
||||
.is_some_and(|suffix| suffix.starts_with('/'))
|
||||
}
|
||||
|
||||
fn normalize_candidate(path: &str) -> Cow<'_, str> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
Cow::Owned(path.replace('\\', "/"))
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
Cow::Borrowed(path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{WorkspaceGlob, WorkspaceGlobError, WorkspaceGlobSet};
|
||||
|
||||
#[test]
|
||||
fn workspace_glob_has_root_relative_segment_semantics() {
|
||||
let cases = [
|
||||
("*.md", "README.md", true),
|
||||
("*.md", "docs/README.md", false),
|
||||
("**/*.md", "README.md", true),
|
||||
("**/*.md", "docs/README.md", true),
|
||||
(".ai/reports/*.md", ".ai/reports/result.md", true),
|
||||
(".ai/reports/*.md", ".ai/reports/nested/result.md", false),
|
||||
(".ai/reports/**/*.md", ".ai/reports/nested/result.md", true),
|
||||
(
|
||||
".ai/plans/????-??-??-*.md",
|
||||
".ai/plans/2026-07-25-globbing.md",
|
||||
true,
|
||||
),
|
||||
(".ai/plans/????-??-??-*.md", ".ai/plans/DRAFTING.md", false),
|
||||
("*/SKILL.md", "rust/SKILL.md", true),
|
||||
("*/SKILL.md", "rust/review/SKILL.md", false),
|
||||
("src/[lm]ib.rs", "src/lib.rs", true),
|
||||
("src/[!m]ib.rs", "src/lib.rs", true),
|
||||
("**/.env", ".env", true),
|
||||
("**/.env", "nested/.env", true),
|
||||
];
|
||||
|
||||
for (pattern, candidate, expected) in cases {
|
||||
let glob = WorkspaceGlob::try_new(pattern).unwrap();
|
||||
assert_eq!(
|
||||
glob.is_match(candidate),
|
||||
expected,
|
||||
"pattern {pattern:?}, candidate {candidate:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_glob_normalizes_a_leading_current_directory() {
|
||||
let glob = WorkspaceGlob::try_new("./src/*.rs").unwrap();
|
||||
|
||||
assert!(glob.is_match("./src/lib.rs"));
|
||||
assert_eq!(glob.traversal_root(), "src");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_glob_rejects_paths_outside_the_root() {
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new(""),
|
||||
Err(WorkspaceGlobError::Empty)
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new("/tmp/*.md"),
|
||||
Err(WorkspaceGlobError::Absolute { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new("C:/tmp/*.md"),
|
||||
Err(WorkspaceGlobError::Absolute { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new(r"dir\*.rs"),
|
||||
Err(WorkspaceGlobError::BackslashSeparator { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new(r"\\server\share\*.md"),
|
||||
Err(WorkspaceGlobError::BackslashSeparator { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new("../*.md"),
|
||||
Err(WorkspaceGlobError::ParentTraversal { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new("src/[abc"),
|
||||
Err(WorkspaceGlobError::Syntax { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
WorkspaceGlob::try_new("src**/*.rs"),
|
||||
Err(WorkspaceGlobError::Syntax { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_glob_set_minimizes_traversal_roots() {
|
||||
let globs = WorkspaceGlobSet::try_new([
|
||||
".ai/reports/*.md",
|
||||
".ai/reports/nested/*.json",
|
||||
".ai/plans/*.md",
|
||||
".other/targeted.txt",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(globs.traversal_roots(), vec![
|
||||
".other",
|
||||
".ai/plans",
|
||||
".ai/reports"
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_pattern_supersedes_narrower_traversal_roots() {
|
||||
let globs = WorkspaceGlobSet::try_new(["**/*.md", ".ai/reports/*.json"]).unwrap();
|
||||
|
||||
assert_eq!(globs.traversal_roots(), vec![""]);
|
||||
assert!(globs.is_match(".ai/reports/result.md"));
|
||||
assert!(globs.is_match(".ai/reports/result.json"));
|
||||
assert!(!globs.is_match(".ai/reports/result.txt"));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue