scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-06 02:48:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/main' into codex/exact-target-checkout

Browse source 
# Conflicts:
#	lib/components/fabro-sandbox/src/clone_retry.rs
#	lib/components/fabro-sandbox/src/daytona/mod.rs
#	lib/components/fabro-sandbox/src/docker.rs
#	lib/components/fabro-sandbox/src/provider/docker.rs
This commit is contained in:
Scott Werner 2026-08-21 12:15:59 -04:00
commit 75fa8eca8b
695 changed files with 8833 additions and 2841 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/nightly.yml vendored
View file

@ -45,6 +45,7 @@ jobs:
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
if: steps.skip.outputs.skip != 'true'
with:
bun-version: 1.3.14
no-cache: true
- name: Install bun deps (for SPA verify)
@ -54,6 +55,8 @@ jobs:
- name: Set up Rust
if: steps.skip.outputs.skip != 'true'
uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
- uses: taiki-e/install-action@773334c0e05d7e699e4d78234494308223f3a2cf # nextest
if: steps.skip.outputs.skip != 'true'

10
.github/workflows/release.yml vendored
View file

@ -49,6 +49,7 @@ jobs:
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
no-cache: true
- name: Install bun deps
@ -67,6 +68,7 @@ jobs:
- name: Set up Rust
uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
targets: ${{ matrix.target }}
- name: Set up zig
@ -136,7 +138,7 @@ jobs:
release:
name: Release
needs: compile
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
permissions:
contents: write
steps:
@ -164,7 +166,7 @@ jobs:
docker:
name: Docker image
needs: compile
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
permissions:
contents: read
packages: write
@ -257,7 +259,7 @@ jobs:
name: Update Homebrew Formula
needs: release
if: ${{ !contains(github.ref_name, '-') }}
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
environment: release
permissions:
contents: read
@ -306,7 +308,7 @@ jobs:
name: Update Homebrew Nightly Formula
needs: release
if: ${{ contains(github.ref_name, '-') }}
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
environment: release
permissions:
contents: read

6
.github/workflows/rust.yml vendored
View file

@ -101,6 +101,8 @@ jobs:
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
cache-on-failure: true
@ -116,6 +118,8 @@ jobs:
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
cache-on-failure: true
@ -140,6 +144,8 @@ jobs:
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
cache-on-failure: true

8
.github/workflows/typescript.yml vendored
View file

@ -44,6 +44,8 @@ jobs:
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
- run: cd apps/fabro-web && bun run typecheck
- run: cd lib/packages/fabro-api-client && bun run typecheck
@ -58,6 +60,8 @@ jobs:
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
- run: cd apps/fabro-web && bun run test
@ -71,7 +75,11 @@ jobs:
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
with:
toolchain: 1.97.1
- run: cargo --locked dev build -- --locked -p fabro-cli --release
- run: wc -c < target/release/fabro

106
Cargo.lock generated
View file

@ -2255,7 +2255,7 @@ dependencies = [
[[package]]
name = "fabro-acp"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"agent-client-protocol",
"agent-client-protocol-tokio",
@ -2274,7 +2274,7 @@ dependencies = [
[[package]]
name = "fabro-agent"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -2320,7 +2320,7 @@ dependencies = [
[[package]]
name = "fabro-api"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"chrono",
"fabro-automation",
@ -2343,7 +2343,7 @@ dependencies = [
[[package]]
name = "fabro-auth"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -2368,7 +2368,7 @@ dependencies = [
[[package]]
name = "fabro-automation"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -2388,11 +2388,11 @@ dependencies = [
[[package]]
name = "fabro-build-support"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
[[package]]
name = "fabro-checkpoint"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"chrono",
"fabro-config",
@ -2408,7 +2408,7 @@ dependencies = [
[[package]]
name = "fabro-cli"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"assert_cmd",
@ -2510,7 +2510,7 @@ dependencies = [
[[package]]
name = "fabro-client"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"bytes",
@ -2539,7 +2539,7 @@ dependencies = [
[[package]]
name = "fabro-config"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -2569,7 +2569,7 @@ dependencies = [
[[package]]
name = "fabro-core"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"async-trait",
"fabro-types",
@ -2584,7 +2584,7 @@ dependencies = [
[[package]]
name = "fabro-db"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -2596,7 +2596,7 @@ dependencies = [
[[package]]
name = "fabro-dev"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"assert_cmd",
@ -2615,7 +2615,7 @@ dependencies = [
[[package]]
name = "fabro-dump"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"bytes",
@ -2629,7 +2629,7 @@ dependencies = [
[[package]]
name = "fabro-environment"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -2651,9 +2651,10 @@ dependencies = [
[[package]]
name = "fabro-github"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
"base64",
"chrono",
"fabro-http",
@ -2665,6 +2666,7 @@ dependencies = [
"jsonwebtoken",
"serde",
"serde_json",
"strum 0.28.0",
"thiserror 2.0.18",
"tokio",
"tracing",
@ -2673,7 +2675,7 @@ dependencies = [
[[package]]
name = "fabro-graphviz"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"fabro-types",
@ -2688,7 +2690,7 @@ dependencies = [
[[package]]
name = "fabro-hooks"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"async-trait",
"fabro-agent",
@ -2711,7 +2713,7 @@ dependencies = [
[[package]]
name = "fabro-http"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"fabro-static",
"http 1.4.0",
@ -2721,7 +2723,7 @@ dependencies = [
[[package]]
name = "fabro-install"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"base64",
@ -2740,7 +2742,7 @@ dependencies = [
[[package]]
name = "fabro-interview"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"async-trait",
"dialoguer",
@ -2755,7 +2757,7 @@ dependencies = [
[[package]]
name = "fabro-llm"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -2797,7 +2799,7 @@ dependencies = [
[[package]]
name = "fabro-macros"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"clap",
"fabro-options-metadata",
@ -2808,7 +2810,7 @@ dependencies = [
[[package]]
name = "fabro-manifest"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"fabro-api",
@ -2829,7 +2831,7 @@ dependencies = [
[[package]]
name = "fabro-mcp"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"axum",
@ -2849,7 +2851,7 @@ dependencies = [
[[package]]
name = "fabro-mcp-server"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -2877,7 +2879,7 @@ dependencies = [
[[package]]
name = "fabro-mcp-store"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"chrono",
"fabro-db",
@ -2895,7 +2897,7 @@ dependencies = [
[[package]]
name = "fabro-model"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"fabro-static",
"http 1.4.0",
@ -2911,7 +2913,7 @@ dependencies = [
[[package]]
name = "fabro-oauth"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"axum",
@ -2933,7 +2935,7 @@ dependencies = [
[[package]]
name = "fabro-options-metadata"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"serde",
"serde_json",
@ -2941,7 +2943,7 @@ dependencies = [
[[package]]
name = "fabro-proc"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"cc",
"libc",
@ -2950,7 +2952,7 @@ dependencies = [
[[package]]
name = "fabro-redact"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"aho-corasick",
"ref-cast",
@ -2966,7 +2968,7 @@ dependencies = [
[[package]]
name = "fabro-sandbox"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -3010,7 +3012,7 @@ dependencies = [
[[package]]
name = "fabro-server"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -3105,7 +3107,7 @@ dependencies = [
[[package]]
name = "fabro-slack"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"fabro-http",
"fabro-interview",
@ -3127,18 +3129,18 @@ dependencies = [
[[package]]
name = "fabro-spa"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"rust-embed",
]
[[package]]
name = "fabro-static"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
[[package]]
name = "fabro-store"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"async-trait",
"bytes",
@ -3168,7 +3170,7 @@ dependencies = [
[[package]]
name = "fabro-telemetry"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"base64",
@ -3194,7 +3196,7 @@ dependencies = [
[[package]]
name = "fabro-template"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"fabro-types",
@ -3208,7 +3210,7 @@ dependencies = [
[[package]]
name = "fabro-test"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"assert_cmd",
@ -3233,7 +3235,7 @@ dependencies = [
[[package]]
name = "fabro-tool"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -3254,7 +3256,7 @@ dependencies = [
[[package]]
name = "fabro-tracker"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-trait",
@ -3268,7 +3270,7 @@ dependencies = [
[[package]]
name = "fabro-types"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"chrono",
"clap",
@ -3291,7 +3293,7 @@ dependencies = [
[[package]]
name = "fabro-util"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"console 0.15.11",
@ -3314,7 +3316,7 @@ dependencies = [
[[package]]
name = "fabro-validate"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"fabro-acp",
"fabro-graphviz",
@ -3327,7 +3329,7 @@ dependencies = [
[[package]]
name = "fabro-variable"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -3344,7 +3346,7 @@ dependencies = [
[[package]]
name = "fabro-vault"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"chrono",
@ -3363,7 +3365,7 @@ dependencies = [
[[package]]
name = "fabro-workflow"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"assert_cmd",
@ -3433,7 +3435,7 @@ dependencies = [
[[package]]
name = "fabro-workflow-version"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"fabro-config",
"fabro-graphviz",
@ -8544,7 +8546,7 @@ dependencies = [
[[package]]
name = "twin-github"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"axum",
"base64",
@ -8563,7 +8565,7 @@ dependencies = [
[[package]]
name = "twin-openai"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
dependencies = [
"anyhow",
"async-stream",

2
Cargo.toml
View file

@ -11,7 +11,7 @@ resolver = "2"
[workspace.package]
edition = "2021"
version = "0.325.0-nightly.0"
version = "0.332.0-nightly.1"
license = "MIT"
[workspace.dependencies]

34
apps/fabro-web/app/routes/runs.test.tsx
View file

@ -3,6 +3,7 @@ import type { BoardColumn, Run } from "@qltysh/fabro-api-client";
import {
buildBoardColumns,
buildFilterOptions,
loadStoredRunsWorkspaceSearchParams,
placeArchivedColumnLast,
persistRunsWorkspacePreferences,
@ -11,6 +12,7 @@ import {
shouldRefreshBoardForEvent,
} from "./runs";
import { summarizeBatchLifecycleAction } from "../components/runs-list/batch-lifecycle";
import { mapRunListItem } from "../data/runs";
import { TEST_PRINCIPAL } from "../lib/test-fixtures";
function boardRun(id: string, column: BoardColumn, questionText?: string): Run {
@ -217,6 +219,38 @@ describe("runs route board mapping", () => {
});
});
describe("runs route filter options", () => {
function runWith(id: string, repoName: string, workflowName: string): Run {
const run = boardRun(id, "running");
return {
...run,
repository: { ...run.repository, name: repoName },
workflow: { ...run.workflow, name: workflowName },
};
}
test("derives sorted unique options from run items", () => {
const items = [
runWith("a", "qlty/beta", "release"),
runWith("b", "qlty/alpha", "hello"),
runWith("c", "qlty/beta", "release"),
].map(mapRunListItem);
expect(buildFilterOptions(items, (item) => item.repo, "all")).toEqual(["alpha", "beta"]);
expect(buildFilterOptions(items, (item) => item.workflow, "all")).toEqual([
"hello",
"release",
]);
});
test("keeps the active selection when no loaded run matches it", () => {
const items = [runWith("a", "qlty/beta", "release")].map(mapRunListItem);
expect(buildFilterOptions(items, (item) => item.repo, "gamma")).toEqual(["beta", "gamma"]);
expect(buildFilterOptions([], (item) => item.workflow, "release")).toEqual(["release"]);
});
});
describe("runs route workspace preferences", () => {
class MemoryStorage {
values = new Map<string, string>();

34
apps/fabro-web/app/routes/runs.tsx
View file

@ -140,6 +140,18 @@ export function buildBoardColumns(
});
}
export function buildFilterOptions(
items: RunItem[],
pick: (item: RunItem) => string,
selected: string,
): string[] {
const values = new Set(items.map(pick));
// Keep the active selection visible even when no loaded run matches it,
// e.g. a stored repo filter while paginating the list view.
if (selected !== "all") values.add(selected);
return Array.from(values).sort();
}
export function placeArchivedColumnLast(columns: Column[], includeArchived: boolean): Column[] {
if (!includeArchived) return columns;
const archived = columns.find((column) => column.id === "archived");
@ -771,18 +783,18 @@ export default function Runs() {
);
const hasGitHubAuth = authConfig.data?.methods.includes("github") === true;
const serverUrl = systemInfo.data?.server_url;
const allRepos = Array.from(
new Set(
initialColumns.flatMap((col: Column) => col.items.map((item: RunItem) => String(item.repo))),
),
// Filter options come from the loaded runs: all runs in columns view, the
// current page in list view (until a facets endpoint provides the full set).
const filterSourceItems: RunItem[] =
view === "list"
? (listRunsPage.data?.data ?? []).map(mapRunListItem)
: initialColumns.flatMap((col: Column) => col.items);
const allRepos = buildFilterOptions(filterSourceItems, (item) => item.repo, repoFilter);
const allWorkflows = buildFilterOptions(
filterSourceItems,
(item) => item.workflow,
workflowFilter,
);
allRepos.sort();
const allWorkflows = Array.from(
new Set(
initialColumns.flatMap((col: Column) => col.items.map((item: RunItem) => String(item.workflow))),
),
);
allWorkflows.sort();
const [columnsState, setColumnsState] = useState(() => ({
base: initialColumns,
columns: initialColumns,

2
docs/internal/events.md
View file

@ -77,7 +77,7 @@ Emitted when the run record is created.
| `source_directory` | string? | Submitter-side source directory |
| `workflow_slug` | string? | Workflow slug |
| `provenance` | object | Actor and request provenance |
| `manifest_blob` | string? | Blob id for the submitted manifest |
| `manifest_blob` | string? | Blob hash for the submitted manifest |
| `git` | object? | Git provenance observed before the run: normalized `origin_url`, `branch`, optional `sha`, and `dirty` status |
| `fork_source_ref` | object? | Source run/checkpoint reference when this run was forked |
| `in_place` | boolean | Whether the run was created with `--in-place` (no git checkpoints) |

16
docs/internal/server-secrets-strategy.md
View file

@ -13,7 +13,7 @@ when does it resolve** — see [Which process resolves what](#which-process-reso
- Resolution is snapshot-based: env and file are read once at construction, then treated as immutable for the life of the process.
- `process env` wins over `server.env` on conflicts.
- Optional integration secrets are vault-only in the **server process**. Do not add optional server integrations to `ServerSecrets`, and do not add bespoke env fallback paths to it.
- Not every credential is a `ServerSecrets` or vault lookup. A third mechanism exists: **settings-declared credentials** in `InterpString` fields, resolved at consumption time from `{{ env.NAME }}` or `{{ secrets.NAME }}`. See [Settings-declared credentials](#settings-declared-credentials).
- Not every credential is a `ServerSecrets` or vault lookup. A third mechanism exists: **settings-declared credentials** in `InterpString` fields, resolved at consumption time from `{{ secrets.NAME }}`. See [Settings-declared credentials](#settings-declared-credentials).
- `fabro server start` never generates secrets. Missing required secrets are a startup error.
- `std::env::set_var` and `std::env::remove_var` are banned workspace-wide. Tests are not exempt. Enforced by clippy via `disallowed_methods` in `clippy.toml`; intentional exceptions must be annotated with a scoped `#[expect(clippy::disallowed_methods, reason = "...")]` at the call site.
@ -70,7 +70,6 @@ than saying "server runtime", which is ambiguous.
| Bootstrap server secret | Server process, via `ServerSecrets` | Once at construction, then immutable |
| Optional integration secret | Server process or worker, via the vault | At use |
| `{{ vars.NAME }}` | Server process | When the run is created, from that run's variable snapshot |
| `{{ env.NAME }}` | The process that owns the value (usually the worker) | At consumption time |
| `{{ secrets.NAME }}` | The process that owns the value, against the server vault | At consumption time |
`docs/public/agents/mcp.mdx` documents the same split for MCP server configuration and is a good
@ -80,18 +79,17 @@ worked example of the shape.
Some credentials are declared in settings rather than looked up by name. Those fields are
`InterpString` (`lib/foundation/fabro-types/src/settings/interp.rs`), which supports narrow
`{{ namespace.NAME }}` tokens with no template logic. Three namespaces resolve: `env` (process
environment, consumption time), `secrets` (vault, consumption time), and `vars` (non-sensitive run
variables, substituted early at run creation). A token whose namespace is unavailable in the
resolution context fails loudly.
`{{ namespace.NAME }}` tokens with no template logic. Two namespaces resolve: `secrets` (vault,
consumption time) and `vars` (non-sensitive run variables, substituted early at run creation).
`{{ env.NAME }}` tokens still parse but never resolve; they fail loudly with a migration message. A
token whose namespace is unavailable in the resolution context also fails loudly.
The reference implementation is LLM provider `extra_headers`, resolved against env plus vault at
The reference implementation is LLM provider `extra_headers`, resolved against the vault at
`lib/foundation/fabro-auth/src/resolve.rs:376-378`:
```toml
[llm.providers.example.extra_headers]
authorization = "Bearer {{ secrets.EXAMPLE_TOKEN }}"
x-tenant = "{{ env.EXAMPLE_TENANT }}"
```
Use this mechanism when the credential belongs to an operator-configured integration declared in
@ -149,7 +147,7 @@ First pick the mechanism. These are the only three:
|---|---|---|
| Bootstrap server secret | Platform env or install-written `server.env` | `state.server_secret(...)` |
| Optional integration secret | Vault (`fabro secret set`, `fabro install`) | `state.vault_secret(...)` |
| Settings-declared credential | `{{ secrets.* }}` or `{{ env.* }}` in an `InterpString` settings field | Resolved at consumption time by the owning process |
| Settings-declared credential | `{{ secrets.* }}` in an `InterpString` settings field | Resolved at consumption time by the owning process |
Then:

6
docs/public/agents/outputs.mdx
View file

@ -219,10 +219,10 @@ When Fabro builds a [preamble](/execution/context#preamble-construction) for a d
- **plan**: success
- Model: claude-sonnet-4-5, 12.4k tokens in / 3.2k out
- Files: src/main.rs, tests/api_test.rs
- Response: See: /path/to/runtime/blobs/<blob_id>.json
- Response: See: /path/to/runtime/blobs/<blob_hash>.json
- **test**: success
- Script: `cargo test 2>&1 || true`
- Stdout: See: /path/to/runtime/blobs/<blob_id>.json
- Stdout: See: /path/to/runtime/blobs/<blob_hash>.json
```
This keeps preambles concise while still giving agents a path to read the full output if needed.
@ -237,7 +237,7 @@ Captured stage artifacts such as screenshots, videos, reports, and traces still
For remote sandboxes (Docker, Daytona), execution-time file access happens inside the sandbox filesystem.
- Blob refs are materialized into `{working_directory}/.fabro/blobs/{blob_id}.json`
- Blob refs are materialized into `{working_directory}/.fabro/blobs/{blob_hash}.json`
- Explicit non-blob `file://` refs keep the existing copy-on-demand behavior and are copied into `{working_directory}/.fabro/artifacts/{filename}` when needed
In both cases, downstream handlers and agents continue to consume ordinary `file://` pointers during execution.

56
docs/public/api-reference/fabro-api.yaml
View file

@ -1,7 +1,7 @@
openapi: "3.1.0"
info:
title: Fabro Run API
version: "0.1.0"
version: "0.2.0"
description: HTTP API for managing Fabro workflow run executions.
tags:
@ -596,6 +596,15 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/DiagnosticsReport"
"504":
description: Diagnostics operation timed out
headers:
x-request-id:
$ref: "#/components/headers/XRequestId"
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorResponse"
/api/v1/openapi.json:
get:
@ -3092,7 +3101,7 @@ paths:
operationId: writeRunBlob
tags: [Run Internals]
summary: Write Run Blob
description: Writes an opaque binary blob and returns its content-addressed blob identifier.
description: Writes an opaque binary blob and returns its content-addressed blob hash.
parameters:
- $ref: "#/components/parameters/RunId"
requestBody:
@ -3137,15 +3146,15 @@ paths:
schema:
$ref: "#/components/schemas/ErrorResponse"
/api/v1/runs/{id}/blobs/{blobId}:
/api/v1/runs/{id}/blobs/{blobHash}:
get:
operationId: readRunBlob
tags: [Run Internals]
summary: Read Run Blob
description: Reads a previously stored blob by identifier.
description: Reads a previously stored blob by hash.
parameters:
- $ref: "#/components/parameters/RunId"
- $ref: "#/components/parameters/BlobId"
- $ref: "#/components/parameters/BlobHash"
responses:
"200":
description: Blob contents
@ -5974,14 +5983,13 @@ components:
default: 65536
example: 65536
BlobId:
name: blobId
BlobHash:
name: blobHash
in: path
required: true
description: Content-addressed blob identifier.
description: Content-addressed blob hash.
schema:
type: string
pattern: '^[0-9a-f]{64}$'
$ref: "#/components/schemas/BlobHash"
example: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
ArtifactFilename:
@ -9152,9 +9160,11 @@ components:
example: graphs/main.fabro
WorkflowVersionId:
description: SHA-256 identity of validated canonical workflow-version bytes.
description: >-
SHA-256 identity of validated canonical workflow-version bytes. Hex input is
case-insensitive; Fabro emits the canonical lowercase form.
type: string
pattern: "^[0-9a-f]{64}$"
pattern: "^[0-9A-Fa-f]{64}$"
example: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
WorkflowVersion:
@ -10283,16 +10293,22 @@ components:
description: Assigned event sequence number.
example: 42
BlobHash:
description: >-
Content-addressed SHA-256 hash of a stored blob. Hex input is case-insensitive;
Fabro emits the canonical lowercase form.
type: string
pattern: "^[0-9A-Fa-f]{64}$"
example: 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
WriteBlobResponse:
description: Content-addressed identifier for a stored blob.
description: Content-addressed hash of a stored blob.
type: object
required:
- id
- hash
properties:
id:
type: string
description: Blob identifier.
example: 550e8400-e29b-41d4-a716-446655440000
hash:
$ref: "#/components/schemas/BlobHash"
CommandTermination:
description: Terminal state for a command execution.
@ -10401,7 +10417,7 @@ components:
example: src/lib.rs
sha256:
type: ["string", "null"]
description: Optional lowercase hex SHA-256 checksum for the file contents.
description: Optional SHA-256 checksum for the file contents; hex input is case-insensitive.
example: 3f785df4c5b7d3f1f4c1f0ecb0f55f1d9f6f6a3d9f0a8a98f7a74f29d1f81a2c
expected_bytes:
type: ["integer", "null"]
@ -11434,6 +11450,8 @@ components:
type: ["string", "null"]
definition_blob:
type: ["string", "null"]
spec_blob:
type: ["string", "null"]
git:
oneOf:
- $ref: "#/components/schemas/GitContext"

10
docs/public/core-concepts/models.mdx
View file

@ -59,13 +59,18 @@ Fabro performs this selection once when creating a run and persists the chosen p
| `gemini-3.1-flash-lite` | gemini | `gemini-flash-lite`, `gemini-3.1-flash-lite-preview` | 1M | $0.25 / $1.50 | 200 tok/s |
| `kimi-k2.5` | moonshot | | 262K | $0.60 / $3.00 | 50 tok/s |
| `kimi-k3` | moonshot | `kimi` | 1M | $3.00 / $15.00 | n/a |
| `kimi-k3-fast` | venice | `kimi-fast` | 1M | $4.50 / $22.50 | n/a |
| `deepseek-v4-flash` | deepseek | `deepseek`, `deepseek-v4`, `deepseek-flash` | 1,048,576 | $0.14 / $0.28 | n/a |
| `deepseek-v4-pro` | deepseek | | 1,048,576 | $0.435 / $0.87 | n/a |
| `grok-4.6` | venice | `grok`, `grok46`, `grok-46` | 500K | $2.27 / $6.80 | n/a |
| `laguna-s-2.1` | poolside | `laguna`, `laguna-s` | 1M | $0.10 / $0.20 | n/a |
| `laguna-xs-2.1` | poolside | `laguna-xs` | 262K | $0.10 / $0.20 | n/a |
| `glm-5.2` | zai | `glm`, `glm5`, `glm52`, `glm5.2` | 1M | $1.40 / $4.40 | n/a |
| `glm-5.3` | venice | `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` | 1M | $1.75 / $5.50 | n/a |
| `minimax-m2.5` | minimax | `minimax` | 197K | $0.30 / $1.20 | 45 tok/s |
| `mercury-2` | inception | `mercury` | 131K | $0.25 / $0.75 | 1000 tok/s |
| `qwen3.8-max` | venice | `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` | 1M | $2.50 / $7.50 | n/a |
| `qwen3.8-27b` | venice | `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` | 262K | $0.45 / $3.20 | n/a |
Each provider requires its own API key. Server-backed workflows read provider credentials from the server vault (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPSEEK_API_KEY`, or `POOLSIDE_API_KEY` set with `fabro secret set` or `fabro provider login`). Standalone SDK/CLI flows can opt into env-backed credential sources explicitly. See the [Quick Start](/getting-started/quick-start) for setup.
@ -169,6 +174,10 @@ Provider `billing_policy` defaults from `adapter` and controls usage-cost estima
Provider fields in configuration, APIs, and model routing are provider ID strings. Built-in names like `anthropic`, `openai`, and `gemini` still work, but custom IDs like `proxy` work anywhere a provider ID is accepted.
</Note>
### Venice
Fabro ships a built-in [Venice](/integrations/venice) provider with a curated catalog of Venice-hosted Kimi, Grok, GLM, DeepSeek, and Qwen models. Store its API key with `fabro provider login --provider venice`. Pin `provider = "venice"` when a shared model slug must use Venice instead of a higher-priority direct provider.
### Poolside
Fabro ships a built-in [Poolside](/integrations/poolside) provider for Laguna S 2.1 and Laguna XS 2.1 over Poolside's OpenAI-compatible API. Store a direct API key with `fabro provider login --provider poolside`. The same model slugs are also available through the opt-in OpenRouter provider; its vendor-namespaced strings remain provider-only `api_id` values.
@ -232,6 +241,7 @@ When no model or provider is specified, Fabro chooses the default offering on th
| `moonshot` | `kimi-k3` |
| `poolside` | `laguna-s-2.1` |
| `zai` | `glm-5.2` |
| `venice` | `deepseek-v4-flash` |
| `minimax` | `minimax-m2.5` |
| `inception` | `mercury-2` |

1
docs/public/docs.json
View file

@ -97,6 +97,7 @@
"integrations/litellm",
"integrations/bedrock",
"integrations/deepseek",
"integrations/venice",
"integrations/poolside",
"integrations/openrouter",
"integrations/modal",

2
docs/public/examples/nlspec-conformance.mdx
View file

@ -123,7 +123,7 @@ Do not rewrite working code. Make targeted fixes to the specific failures.
### Max visits as a safety valve
`max_visits=5` on the `fix` node prevents infinite loops. If the agent can't pass in 5 iterations, the workflow moves on with the best result so far. Tune this based on spec complexity: a 30-line spec might need 2 iterations, a 2,000-line spec might need 10.
`max_visits=5` on the `fix` node prevents infinite loops. The node can execute up to 5 times; a sixth visit fails the run rather than looping forever. Tune this based on spec complexity: a 30-line spec might need 2 iterations, a 2,000-line spec might need 10.
### Goal gate on full conformance

4
docs/public/execution/context.mdx
View file

@ -243,8 +243,8 @@ Checkpoints and checkpoint-completed events persist these `blob://` refs, not ho
Before Fabro builds a preamble or starts the next stage, it resolves any blob refs into execution-local files so handlers and agents still see normal `file://` references:
- Local execution materializes blobs under `{run_dir}/runtime/blobs/{blob_id}.json`
- Remote sandboxes materialize blobs under `{working_directory}/.fabro/blobs/{blob_id}.json`
- Local execution materializes blobs under `{run_dir}/runtime/blobs/{blob_hash}.json`
- Remote sandboxes materialize blobs under `{working_directory}/.fabro/blobs/{blob_hash}.json`
These materialized `file://` paths are runtime-only. They are not written back into durable context snapshots.

2
docs/public/execution/run-configuration.mdx
View file

@ -321,7 +321,7 @@ memory = "8GB"
| `network.allow` | CIDRs for `cidr_allow_list`; entries are validated as CIDRs. |
| `lifecycle.preserve` | Keep the created sandbox after the run finishes. |
| `lifecycle.stop_on_terminal` | Stop the sandbox when the run reaches a terminal state. |
| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. |
| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. Defaults to `"120m"`; `"0s"` disables auto-stop. |
| `labels` | Provider labels. Merge by key across layers. |
| `env` | Environment variables passed to command and agent execution. Merge by key across layers. |

4
docs/public/integrations/daytona.mdx
View file

@ -198,6 +198,10 @@ The `lifecycle.auto_stop` setting tells Daytona to stop the sandbox after a peri
auto_stop = "30m"
```
When `auto_stop` is unset, Fabro applies a default of 120 minutes so a sandbox leaked by an interrupted run is still reclaimed. Set `auto_stop = "0s"` to disable auto-stop and let the sandbox run indefinitely.
Daytona counts inactivity from the last sandbox interaction (a command, file operation, or other API call). Time an agent spends on LLM inference does not touch the sandbox, so intervals shorter than your longest inference call risk stopping the sandbox mid-run.
## Server defaults
When running via `fabro server start`, the server config at `~/.fabro/settings.toml` can set default Daytona settings for all runs. Run config TOML values override server defaults. Labels are **merged** — run config labels win on key collisions. The `network` setting uses simple override (run config replaces the server default entirely).

125
docs/public/integrations/venice.mdx Normal file
View file

@ -0,0 +1,125 @@
---
title: "Venice"
description: "Run Kimi, Grok, GLM, DeepSeek, and Qwen models through Venice"
---
[Venice](https://venice.ai/) provides an OpenAI-compatible API for hosted text models. Fabro enables the `venice` provider in its built-in catalog and maps stable Fabro model slugs to Venice's API model IDs.
## Prerequisites
- A Venice account
- An inference API key from [venice.ai/settings/api](https://venice.ai/settings/api)
- A running Fabro server
## Configure credentials
Store the API key in the target Fabro server vault:
```bash
fabro provider login --provider venice
# For a non-default remote server:
fabro provider login --server https://your-fabro.example --provider venice
# Or set the vault token directly:
fabro secret set VENICE_API_KEY
fabro secret --server https://your-fabro.example set VENICE_API_KEY
```
Standalone SDK usage outside a Fabro server can use an env-backed credential source explicitly:
```bash
export VENICE_API_KEY=<api-key>
```
Fabro sends bearer-authenticated Chat Completions requests to `https://api.venice.ai/api/v1`.
## Included models
| Fabro model slug | Venice API ID | Context | Max output | Role and aliases |
|---|---|---:|---:|---|
| `kimi-k3` | `kimi-k3` | 1,000,000 | 131,072 | Alias `kimi` |
| `kimi-k3-fast` | `kimi-k3-fast-api` | 1,000,000 | 131,072 | Alias `kimi-fast` |
| `grok-4.6` | `grok-4-6` | 500,000 | 32,000 | Aliases `grok`, `grok46`, `grok-46` |
| `glm-5.3` | `z-ai-glm-5-3` | 1,000,000 | 131,072 | Aliases `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` |
| `deepseek-v4-flash` | `deepseek-v4-flash-0731` | 1,000,000 | 32,768 | Provider default; aliases `deepseek`, `deepseek-v4`, `deepseek-flash` |
| `deepseek-v4-pro` | `deepseek-v4-pro-0813` | 1,000,000 | 32,768 | Alias `deepseek-pro` |
| `qwen3.8-max` | `qwen-3-8-max` | 1,000,000 | 131,072 | Aliases `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` |
| `qwen3.8-27b` | `qwen-3-8-27b` | 262,144 | 131,072 | Aliases `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` |
Venice API IDs are also valid provider-scoped selectors. Fabro persists the stable Fabro slug and the selected provider when it creates a run.
## Select Venice explicitly
Some Venice models use the same stable slugs as direct providers. An unqualified selector chooses the highest-priority ready provider. For example, `deepseek` can select the direct DeepSeek provider when both API keys are configured.
Pin Venice when the run must use Venice:
```bash
fabro model list --provider venice
fabro model test --provider venice --model deepseek-v4-flash --deep
fabro run workflow.fabro --provider venice --model deepseek-v4-flash
```
In a workflow stylesheet:
```dot title="workflow.fabro"
digraph Example {
graph [
model_stylesheet="
* { provider: venice; model: deepseek-v4-flash; }
.complex { provider: venice; model: qwen; }
.fast { provider: venice; model: kimi-fast; }
"
]
start [shape=Mdiamond, label="Start"]
work [label="Implement", class="complex"]
check [label="Check", class="fast"]
exit [shape=Msquare, label="Exit"]
start -> work -> check -> exit
}
```
The generic Qwen aliases `qwen` and `qwen3.8` select Qwen 3.8 Max. Use a size-specific alias such as `qwen-27b` to select Qwen 3.8 27B.
## Capabilities and reasoning
All included models support tool calling and reasoning. Kimi K3, Kimi K3 Fast, Grok 4.6, Qwen 3.8 Max, and Qwen 3.8 27B also accept image input.
Fabro exposes native reasoning-effort controls only when Venice supports them:
| Model | Reasoning effort values |
|---|---|
| `grok-4.6` | `low`, `medium`, `high`, `xhigh` |
| `glm-5.3` | `low`, `high`, `max` |
| `deepseek-v4-flash` | `low`, `high`, `max` |
| `qwen3.8-27b` | `low`, `medium`, `xhigh` |
The other models reason by default but do not expose a Venice reasoning-effort control. Fabro omits sampling parameters for Kimi and DeepSeek because those routes do not use them with their configured reasoning behavior.
## Pricing and prompt caching
The built-in catalog uses Venice's published prices per million tokens:
| Model | Uncached input | Cache hit | Output |
|---|---:|---:|---:|
| `kimi-k3` | $3.75 | $0.375 | $18.75 |
| `kimi-k3-fast` | $4.50 | $0.45 | $22.50 |
| `grok-4.6` | $2.27 | $0.57 | $6.80 |
| `glm-5.3` | $1.75 | $0.325 | $5.50 |
| `deepseek-v4-flash` | $0.175 | $0.035 | $0.35 |
| `deepseek-v4-pro` | $1.65 | $0.165 | $4.95 |
| `qwen3.8-max` | $2.50 | $0.3125 | $7.50 |
| `qwen3.8-27b` | $0.45 | n/a | $3.20 |
Fabro reports cached input separately when Venice returns cache usage for the selected model. Prices and model availability can change upstream; use `fabro model list --provider venice` to inspect the catalog shipped with your Fabro version and the [Venice model catalog](https://docs.venice.ai/models/overview) for the current upstream service.
## Troubleshooting
**"No credential was found for provider 'venice'"** — Store `VENICE_API_KEY` in the server vault with `fabro provider login --provider venice`. Pass `--server` when configuring a remote Fabro server.
**A shared model used another provider** — Pin Venice with `--provider venice` or `provider: venice` in the workflow stylesheet. Unqualified selectors use provider priority.
**A Venice API model ID is rejected without a provider** — Use the stable Fabro slug for portable selection, or qualify the API ID with the provider, such as `venice:qwen-3-8-max`.

4
lib/apps/fabro-cli/src/commands/dump.rs
View file

@ -86,8 +86,8 @@ async fn write_run_dump(
dump.add_file_bytes("run.log", log);
}
dump.hydrate_referenced_blobs_with_reader(|blob_id| {
Box::pin(async move { client.read_run_blob(run_id, &blob_id).await })
dump.hydrate_referenced_blobs_with_reader(|blob_hash| {
Box::pin(async move { client.read_run_blob(run_id, &blob_hash).await })
})
.await?;

1
lib/apps/fabro-cli/src/commands/run/attach.rs
View file

@ -849,6 +849,7 @@ mod tests {
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
};

10
lib/apps/fabro-cli/src/commands/run/output.rs
View file

@ -5,7 +5,7 @@ use anyhow::{Context as _, Result};
use cli_table::format::{Border, Justify, Separator};
use cli_table::{Cell, CellStruct, Style, Table};
use fabro_api::types;
use fabro_types::{BlobHash, PullRequestLink, RunId, StageId, parse_blob_ref};
use fabro_types::{PullRequestLink, RunId, StageId, parse_blob_ref};
use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus};
use fabro_util::error::render_with_causes;
use fabro_util::printer::Printer;
@ -325,11 +325,11 @@ async fn resolve_response_string(
run_id: &RunId,
response: &str,
) -> Result<Option<String>> {
let Some(blob_id) = blob_id_from_response(response) else {
let Some(blob_hash) = parse_blob_ref(response) else {
return Ok(Some(response.to_string()));
};
let Some(bytes) = client.read_run_blob(run_id, &blob_id).await? else {
let Some(bytes) = client.read_run_blob(run_id, &blob_hash).await? else {
return Ok(None);
};
let value: serde_json::Value =
@ -341,10 +341,6 @@ async fn resolve_response_string(
}))
}
fn blob_id_from_response(response: &str) -> Option<BlobHash> {
parse_blob_ref(response)
}
async fn list_artifact_display_entries_with_client(
client: &server_client::Client,
run_id: &RunId,

6
lib/apps/fabro-cli/src/commands/run/runner.rs
View file

@ -1018,12 +1018,12 @@ impl RunStoreBackend for HttpRunStore {
.await
}
async fn read_blob(&self, id: &BlobHash) -> Result<Option<bytes::Bytes>> {
async fn read_blob(&self, blob_hash: &BlobHash) -> Result<Option<bytes::Bytes>> {
self.with_retries("read run blob", || {
let client = self.client.clone_for_reuse();
let run_id = self.run_id;
let blob_id = *id;
async move { client.read_run_blob(&run_id, &blob_id).await }
let blob_hash = *blob_hash;
async move { client.read_run_blob(&run_id, &blob_hash).await }
})
.await
}

21
lib/apps/fabro-cli/tests/it/cmd/attach.rs
View file

@ -66,20 +66,8 @@ fn format_output_snapshot(output: &Output, filters: &[(String, String)]) -> Stri
}
fn normalize_attach_json_progress_event(mut event: Value) -> Value {
if let Some(properties) = event.get_mut("properties").and_then(Value::as_object_mut) {
if properties.contains_key("manifest_blob") {
properties.insert(
"manifest_blob".to_string(),
Value::String("[BLOB_ID]".to_string()),
);
}
if properties.contains_key("definition_blob") {
properties.insert(
"definition_blob".to_string(),
Value::String("[BLOB_ID]".to_string()),
);
}
}
// manifest_blob/definition_blob hashes are already rewritten to
// [BLOB_HASH] by the shared json_snapshot_filters regexes.
// Strip v2-shape server/version fields that the bridge emits,
// since the test fixture's socket path is randomised per run.
if let Some(settings) = event
@ -896,7 +884,7 @@ fn attach_json_errors_without_prompting_for_human_input() {
}
}
},
"manifest_blob": "[BLOB_ID]",
"manifest_blob": "[BLOB_HASH]",
"provenance": {
"client": {
"name": "fabro-cli",
@ -1024,6 +1012,7 @@ fn attach_json_errors_without_prompting_for_human_input() {
}
},
"source_directory": "[TEMP_DIR]",
"spec_blob": "[BLOB_HASH]",
"title": "Wait for approval",
"web_url": "http://localhost:3000/runs/[ULID]",
"workflow_slug": "human-gate",
@ -1036,7 +1025,7 @@ fn attach_json_errors_without_prompting_for_human_input() {
"event": "run.submitted",
"id": "[EVENT_ID]",
"properties": {
"definition_blob": "[BLOB_ID]"
"definition_blob": "[BLOB_HASH]"
},
"run_id": "[ULID]",
"ts": "[TIMESTAMP]"

1
lib/apps/fabro-cli/tests/it/support/mod.rs
View file

@ -53,6 +53,7 @@ pub(crate) fn run_projection_json(run_id: &str, status: &serde_json::Value) -> s
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
};

84
lib/apps/fabro-server/src/diagnostics.rs
View file

@ -6,7 +6,7 @@ use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use fabro_auth::auth_issue_message;
use fabro_llm::client::Client as LlmClient;
use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe};
use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe_with_timeout};
use fabro_model::{Catalog, ProviderId};
use fabro_redact::redact_string;
use fabro_sandbox::{DockerSandboxProvider, daytona};
@ -23,6 +23,9 @@ use tokio::time::timeout;
use crate::server::AppState;
const EXTERNAL_SERVICE_PROBE_TIMEOUT: Duration = Duration::from_secs(15);
const DOCKER_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
fn http_client_or_check(
name: &str,
status: CheckStatus,
@ -252,13 +255,20 @@ async fn probe_single_provider(
None,
);
};
let model_id = model.id.clone();
let model_id = model.id.to_string();
let outcome = run_basic_model_probe_with_timeout(
&model_id,
&provider,
client,
EXTERNAL_SERVICE_PROBE_TIMEOUT,
)
.await;
let outcome = run_basic_model_probe(model_id.as_str(), &provider, client).await;
match outcome.status {
ModelTestStatus::Ok => ProviderProbeResult {
provider,
model_id: Some(model_id.to_string()),
model_id: Some(model_id),
status: ProviderProbeStatus::Ok,
error_message: None,
diagnostic_detail: None,
@ -267,12 +277,7 @@ async fn probe_single_provider(
let raw = outcome
.error_message
.unwrap_or_else(|| "provider probe failed".to_string());
provider_probe_error(
provider,
Some(model_id.to_string()),
redact_string(&raw),
None,
)
provider_probe_error(provider, Some(model_id), redact_string(&raw), None)
}
}
}
@ -388,7 +393,7 @@ async fn check_github_app(state: &AppState) -> CheckResult {
Err(result) => return result,
};
let probe = timeout(
Duration::from_secs(15),
EXTERNAL_SERVICE_PROBE_TIMEOUT,
http.get(format!("{}/user", fabro_github::github_api_base_url()))
.header("Authorization", format!("Bearer {token}"))
.header("Accept", "application/vnd.github+json")
@ -538,7 +543,7 @@ async fn check_github_app(state: &AppState) -> CheckResult {
Err(result) => return result,
};
let auth_result = timeout(
Duration::from_secs(15),
EXTERNAL_SERVICE_PROBE_TIMEOUT,
fabro_github::get_authenticated_app(&http, &jwt, &fabro_github::github_api_base_url()),
)
.await;
@ -581,7 +586,7 @@ async fn check_docker_sandbox(state: &AppState) -> CheckResult {
.await
.map_err(|err| err.display_with_causes())
},
Duration::from_secs(5),
DOCKER_PROBE_TIMEOUT,
)
.await
}
@ -656,7 +661,14 @@ async fn check_cloud_sandbox(state: &AppState) -> CheckResult {
};
};
match state.check_daytona_api_key(api_key).await {
let probe = state
.check_daytona_api_key_with_timeout(api_key, EXTERNAL_SERVICE_PROBE_TIMEOUT)
.await;
cloud_sandbox_probe_check(probe)
}
fn cloud_sandbox_probe_check(probe: anyhow::Result<daytona::DaytonaKeyCheck>) -> CheckResult {
match probe {
Ok(check) if check.ok() => CheckResult {
name: "Cloud Sandbox".to_string(),
status: CheckStatus::Pass,
@ -678,13 +690,29 @@ async fn check_cloud_sandbox(state: &AppState) -> CheckResult {
daytona::required_perms_display()
)),
},
Err(err) => CheckResult {
name: "Cloud Sandbox".to_string(),
status: CheckStatus::Error,
summary: "Daytona credential rejected".to_string(),
details: vec![CheckDetail::new(format!("{err:#}"))],
remediation: Some("Verify DAYTONA_API_KEY value and Daytona reachability".to_string()),
},
Err(err) => {
if let Some(timeout) = err.downcast_ref::<daytona::DaytonaCredentialProbeTimeout>() {
return CheckResult {
name: "Cloud Sandbox".to_string(),
status: CheckStatus::Error,
summary: format!("timeout ({:?})", timeout.timeout()),
details: vec![CheckDetail::new("Daytona probe timed out".to_string())],
remediation: Some(
"Verify DAYTONA_API_KEY value and Daytona reachability".to_string(),
),
};
}
CheckResult {
name: "Cloud Sandbox".to_string(),
status: CheckStatus::Error,
summary: "Daytona credential rejected".to_string(),
details: vec![CheckDetail::new(format!("{err:#}"))],
remediation: Some(
"Verify DAYTONA_API_KEY value and Daytona reachability".to_string(),
),
}
}
}
}
@ -750,7 +778,7 @@ async fn check_brave_search(state: &AppState) -> CheckResult {
Err(result) => return result,
};
let probe = timeout(Duration::from_secs(15), async move {
let probe = timeout(EXTERNAL_SERVICE_PROBE_TIMEOUT, async move {
http.get("https://api.search.brave.com/res/v1/web/search?q=test&count=1")
.header("X-Subscription-Token", api_key)
.send()
@ -1154,6 +1182,18 @@ enabled = false
);
}
#[test]
fn check_cloud_sandbox_reports_timeout() {
let result = cloud_sandbox_probe_check(Err(anyhow::Error::new(
daytona::DaytonaCredentialProbeTimeout::new(Duration::from_millis(1)),
)));
assert_eq!(result.name, "Cloud Sandbox");
assert_eq!(result.status, CheckStatus::Error);
assert_eq!(result.summary, "timeout (1ms)");
assert_eq!(result.details[0].text, "Daytona probe timed out");
}
#[tokio::test]
async fn check_brave_search_ignores_env_backed_api_key() {
let state = TestAppStateBuilder::new()

8
lib/apps/fabro-server/src/principal_middleware.rs
View file

@ -14,7 +14,7 @@ use strum::IntoStaticStr;
use crate::auth::{AuthErrorCode, JwtError, REFRESH_TOKEN_PREFIX};
use crate::error::ApiError;
use crate::jwt_auth::{self, AuthMode, ConfiguredAuth};
use crate::server::{AppState, parse_blob_id_path, parse_run_id_path, parse_stage_id_path};
use crate::server::{AppState, parse_blob_hash_path, parse_run_id_path, parse_stage_id_path};
use crate::worker_token::{self, WORKER_TOKEN_KID, WorkerScopeSet};
#[derive(Clone, Debug)]
@ -295,14 +295,14 @@ impl FromRequestParts<Arc<AppState>> for RequireRunBlob {
parts: &mut Parts,
state: &Arc<AppState>,
) -> Result<Self, Self::Rejection> {
let Path((id, blob_id)): Path<(String, String)> = Path::from_request_parts(parts, state)
let Path((id, blob_hash)): Path<(String, String)> = Path::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let run_id = parse_run_id_path(&id)?;
let blob_id = parse_blob_id_path(&blob_id)?;
let blob_hash = parse_blob_hash_path(&blob_hash)?;
require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id)
.map_err(IntoResponse::into_response)?;
Ok(Self(run_id, blob_id))
Ok(Self(run_id, blob_hash))
}
}

1
lib/apps/fabro-server/src/run_files.rs
View file

@ -2387,6 +2387,7 @@ index 1111111..2222222 160000
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
},

27
lib/apps/fabro-server/src/server.rs
View file

@ -1455,6 +1455,15 @@ impl AppState {
pub(crate) async fn check_daytona_api_key(
&self,
api_key: String,
) -> anyhow::Result<daytona::DaytonaKeyCheck> {
self.check_daytona_api_key_with_timeout(api_key, daytona::DAYTONA_CREDENTIAL_PROBE_TIMEOUT)
.await
}
pub(crate) async fn check_daytona_api_key_with_timeout(
&self,
api_key: String,
probe_timeout: Duration,
) -> anyhow::Result<daytona::DaytonaKeyCheck> {
let base_url = self
.config_env_lookup(EnvVars::DAYTONA_API_URL)
@ -1463,8 +1472,14 @@ impl AppState {
let org_id = self.config_env_lookup(EnvVars::DAYTONA_ORGANIZATION_ID);
let http_client = fabro_http::http_client().context("failed to build HTTP client")?;
daytona::check_daytona_api_key_with(&base_url, org_id.as_deref(), api_key, http_client)
.await
daytona::check_daytona_api_key_with_timeout(
&base_url,
org_id.as_deref(),
api_key,
http_client,
probe_timeout,
)
.await
}
/// Borrow the persistent store so sibling modules can open run readers
@ -2889,11 +2904,11 @@ pub(crate) fn parse_stage_id_path(stage_id: &str) -> Result<StageId, Response> {
#[allow(
clippy::result_large_err,
reason = "Blob ID parsing returns HTTP 400 responses directly."
reason = "Blob hash parsing returns HTTP 400 responses directly."
)]
pub(crate) fn parse_blob_id_path(blob_id: &str) -> Result<BlobHash, Response> {
BlobHash::from_str(blob_id)
.map_err(|_| ApiError::bad_request("Invalid blob ID.").into_response())
pub(crate) fn parse_blob_hash_path(blob_hash: &str) -> Result<BlobHash, Response> {
BlobHash::from_str(blob_hash)
.map_err(|_| ApiError::bad_request("Invalid blob hash.").into_response())
}
#[allow(

11
lib/apps/fabro-server/src/server/handler/artifacts.rs
View file

@ -32,7 +32,7 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new()
.route("/runs/{id}/checkpoint", get(get_checkpoint))
.route("/runs/{id}/blobs", post(write_run_blob))
.route("/runs/{id}/blobs/{blobId}", get(read_run_blob))
.route("/runs/{id}/blobs/{blobHash}", get(read_run_blob))
.route("/runs/{id}/artifacts", get(list_run_artifacts))
.route("/runs/{id}/artifacts/download", get(download_run_artifacts))
.route(
@ -105,10 +105,7 @@ async fn write_run_blob(
}
match state.stores.runs.open_run(&id).await {
Ok(run_store) => match run_store.write_blob(&body).await {
Ok(blob_id) => Json(WriteBlobResponse {
id: blob_id.to_string(),
})
.into_response(),
Ok(blob_hash) => Json(WriteBlobResponse { hash: blob_hash }).into_response(),
Err(err) => {
ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response()
}
@ -118,11 +115,11 @@ async fn write_run_blob(
}
async fn read_run_blob(
RequireRunBlob(id, blob_id): RequireRunBlob,
RequireRunBlob(id, blob_hash): RequireRunBlob,
State(state): State<Arc<AppState>>,
) -> Response {
match state.stores.runs.open_run_reader(&id).await {
Ok(run_store) => match run_store.read_blob(&blob_id).await {
Ok(run_store) => match run_store.read_blob(&blob_hash).await {
Ok(Some(bytes)) => octet_stream_response(bytes),
Ok(None) => ApiError::not_found("Blob not found.").into_response(),
Err(err) => {

1
lib/apps/fabro-server/src/server/handler/events.rs
View file

@ -627,6 +627,7 @@ mod stage_events_tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

2
lib/apps/fabro-server/src/server/handler/mod.rs
View file

@ -101,7 +101,7 @@ pub(super) fn demo_routes() -> Router<Arc<AppState>> {
)
.route("/runs/{id}/attach", get(demo::run_events_stub))
.route("/runs/{id}/blobs", post(not_implemented))
.route("/runs/{id}/blobs/{blobId}", get(not_implemented))
.route("/runs/{id}/blobs/{blobHash}", get(not_implemented))
.route(
"/runs/{id}/stages/{stageId}/logs/output",
get(not_implemented),

1
lib/apps/fabro-server/src/server/handler/pair.rs
View file

@ -1027,6 +1027,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

1
lib/apps/fabro-server/src/server/handler/sessions.rs
View file

@ -1923,6 +1923,7 @@ reasoning = false
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
};

59
lib/apps/fabro-server/src/server/handler/system.rs
View file

@ -1,5 +1,7 @@
use std::collections::BTreeMap;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
use fabro_slack::config::{
@ -9,6 +11,7 @@ use fabro_slack::config::{
use fabro_static::EnvVars;
use fabro_types::settings::server::GithubIntegrationSettings;
use fabro_vault::Vault;
use tokio::time::timeout;
use super::super::{
AggregateBilling, AggregateBillingTotals, ApiError, AppState, BilledTokenCounts,
@ -21,6 +24,8 @@ use super::super::{
resource_sampler, spawn_blocking, system_sandbox_provider, to_i64,
};
const SERVER_DIAGNOSTICS_TIMEOUT: Duration = Duration::from_secs(25);
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new()
.route("/repos/github/{owner}/{name}", get(get_github_repo))
@ -683,11 +688,34 @@ async fn get_github_repo(
}
async fn run_diagnostics(_auth: RequiredUser, State(state): State<Arc<AppState>>) -> Response {
(
StatusCode::OK,
Json(diagnostics::run_all(state.as_ref()).await),
diagnostics_response_with_timeout(
Box::pin(diagnostics::run_all(state.as_ref())),
SERVER_DIAGNOSTICS_TIMEOUT,
)
.into_response()
.await
}
async fn diagnostics_response_with_timeout<F>(
diagnostics: F,
operation_timeout: Duration,
) -> Response
where
F: Future<Output = diagnostics::DiagnosticsReport>,
{
let Ok(report) = timeout(operation_timeout, diagnostics).await else {
tracing::warn!(
timeout_secs = operation_timeout.as_secs(),
"server diagnostics timed out"
);
return ApiError::with_code(
StatusCode::GATEWAY_TIMEOUT,
"Server diagnostics timed out.",
"diagnostics_timeout",
)
.into_response();
};
(StatusCode::OK, Json(report)).into_response()
}
pub(in crate::server) async fn openapi_spec() -> Response {
@ -737,3 +765,26 @@ async fn get_aggregate_billing(
};
(StatusCode::OK, Json(response)).into_response()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn diagnostics_response_returns_gateway_timeout_before_client_deadline() {
let response = diagnostics_response_with_timeout(
std::future::pending::<diagnostics::DiagnosticsReport>(),
Duration::from_millis(1),
)
.await;
let body = fabro_test::expect_axum_json(
response,
StatusCode::GATEWAY_TIMEOUT,
"GET /api/v1/system/diagnostics timeout",
)
.await;
assert_eq!(body["errors"][0]["code"], "diagnostics_timeout");
assert_eq!(body["errors"][0]["detail"], "Server diagnostics timed out.");
}
}

41
lib/apps/fabro-server/src/server/handler/workflow_versions.rs
View file

@ -116,7 +116,7 @@ mod tests {
use axum::body::{Body, to_bytes};
use axum::http::{Method, Request, StatusCode, header};
use axum::response::IntoResponse;
use fabro_types::WorkflowVersionId;
use fabro_types::{BlobHash, WorkflowVersion, WorkflowVersionId};
use serde_json::{Value, json};
use tower::ServiceExt;
@ -233,6 +233,45 @@ mod tests {
);
}
#[tokio::test]
async fn create_rejects_workflow_config_with_missing_goal_file_before_storage() {
let state = TestAppStateBuilder::new().build();
let app = test_support::build_test_router(Arc::clone(&state));
let payload = json!({
"entrypoint": "workflow.fabro",
"files": {
"workflow.fabro": GRAPH,
"workflow.toml": "_version = 1\n[run.goal]\nfile = \"prompts/goal.md\"\n"
},
"workflow_dependencies": {}
});
let version = serde_json::from_value::<WorkflowVersion>(payload.clone()).unwrap();
let id = WorkflowVersionId::from(BlobHash::new(&version.canonical_bytes().unwrap()));
let response = app
.oneshot(request(serde_json::to_vec(&payload).unwrap()))
.await
.unwrap();
let body = fabro_test::expect_axum_json(
response,
StatusCode::UNPROCESSABLE_ENTITY,
"POST /api/v1/workflow-versions with missing run goal file",
)
.await;
assert_eq!(error_code(&body), INVALID_VERSION_CODE);
assert!(
!state
.store_ref()
.blobs()
.await
.unwrap()
.exists(&id.into())
.await
.unwrap()
);
}
#[tokio::test]
async fn unavailable_dependency_has_specific_code() {
let state = TestAppStateBuilder::new().build();

31
lib/apps/fabro-server/src/server/tests.rs
View file

@ -10,7 +10,7 @@ use std::sync::{Arc as StdArc, Mutex as StdMutex};
use async_zip::base::read::mem::ZipFileReader;
use axum::body::Body;
use axum::http::{Method, Request, header};
use chrono::{Duration as ChronoDuration, Utc};
use chrono::{Duration as ChronoDuration, SubsecRound as _, Utc};
use fabro_automation::{AutomationId, AutomationTarget};
use fabro_config::bind::Bind;
use fabro_config::{
@ -4069,7 +4069,9 @@ async fn create_run_from_manifest_resolves_generated_id_after_variable_snapshot(
let body = response_json!(response, StatusCode::CREATED).await;
let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();
assert!(run_id.created_at() >= variable.updated_at);
// RunId is a ULID whose timestamp only has millisecond precision, so
// truncate the variable timestamp to milliseconds before comparing.
assert!(run_id.created_at() >= variable.updated_at.trunc_subsecs(3));
}
#[tokio::test]
@ -4650,6 +4652,7 @@ async fn append_default_run_created(run_store: &fabro_store::RunDatabase, run_id
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -4701,6 +4704,7 @@ async fn create_slack_notification_run(
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -5774,6 +5778,7 @@ async fn list_run_stages_distinguishes_visits() {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -5910,6 +5915,7 @@ async fn list_run_stages_exposes_execution_identity_for_resumed_stage() {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -7097,6 +7103,7 @@ async fn create_completed_run_ready_for_pull_request(
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
fork_source_ref: None,
};
@ -7113,6 +7120,7 @@ async fn create_completed_run_ready_for_pull_request(
automation: None,
provenance: run_spec.provenance.clone(),
manifest_blob: None,
spec_blob: None,
git,
fork_source_ref: None,
retried_from: None,
@ -11034,7 +11042,7 @@ async fn get_checkpoint_returns_null_initially() {
}
#[tokio::test]
async fn write_and_read_run_blob_round_trip() {
async fn write_and_read_run_blob_accepts_uppercase_hash() {
let state = test_app_state();
let app = crate::test_support::build_test_router(Arc::clone(&state));
@ -11057,11 +11065,14 @@ async fn write_and_read_run_blob_round_trip() {
.unwrap();
let response = app.clone().oneshot(req).await.unwrap();
let body = response_json!(response, StatusCode::OK).await;
let blob_id = body["id"].as_str().unwrap();
let blob_hash = body["hash"].as_str().unwrap();
let req = Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/blobs/{blob_id}")))
.uri(api(&format!(
"/runs/{run_id}/blobs/{}",
blob_hash.to_uppercase()
)))
.body(Body::empty())
.unwrap();
let response = app.oneshot(req).await.unwrap();
@ -11459,7 +11470,7 @@ async fn worker_token_accepts_run_scoped_routes_and_falls_back_to_user_jwt() {
let worker_token = issue_test_worker_token(&run_id);
let other_run_id = create_run_with_bearer(&app, &user_jwt).await;
let other_worker_token = issue_test_worker_token(&other_run_id);
let blob_id = state
let blob_hash = state
.stores
.runs
.open_run(&run_id)
@ -11553,7 +11564,7 @@ async fn worker_token_accepts_run_scoped_routes_and_falls_back_to_user_jwt() {
.clone()
.oneshot(bearer_request(
Method::GET,
&format!("/runs/{run_id}/blobs/{blob_id}"),
&format!("/runs/{run_id}/blobs/{blob_hash}"),
&worker_token,
Body::empty(),
))
@ -12058,7 +12069,7 @@ async fn worker_token_is_rejected_on_user_only_routes() {
let user_jwt = issue_test_user_jwt();
let run_id = create_run_with_bearer(&app, &user_jwt).await;
let worker_token = issue_test_worker_token(&run_id);
let blob_id = BlobHash::new(b"blob");
let blob_hash = BlobHash::new(b"blob");
let user_only_routes = vec![
(Method::GET, "/runs".to_string()),
(Method::POST, "/runs".to_string()),
@ -12121,7 +12132,7 @@ async fn worker_token_is_rejected_on_user_only_routes() {
.clone()
.oneshot(bearer_request(
Method::GET,
&format!("/runs/{run_id}/blobs/{blob_id}"),
&format!("/runs/{run_id}/blobs/{blob_hash}"),
&worker_token,
Body::empty(),
))
@ -14082,6 +14093,7 @@ async fn create_preserved_local_sandbox_run(state: &Arc<AppState>, run_id: RunId
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -14831,6 +14843,7 @@ async fn delete_run_retry_after_missing_provider_resource_removes_metadata() {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

1
lib/apps/fabro-server/tests/it/api/run_files.rs
View file

@ -68,6 +68,7 @@ async fn append_completed_run_with_final_patch(
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

5
lib/components/fabro-agent/src/lib.rs
View file

@ -59,8 +59,9 @@ pub use question_tools::{
};
pub use sandbox::{
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingRequest, ExecStreamingResult,
GrepOptions, RefreshOutcome, Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector,
StdioProcess, StdioProcessHandle, format_lines_numbered, shell_quote,
GrepOptions, RefreshOutcome, RemoteCredentialAction, Sandbox, SandboxEvent,
SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle, TokenProvenance,
TokenSnapshot, format_lines_numbered, shell_quote,
};
pub use session::{
CompletionCoordinator, Session, SessionControlHandle, SessionInputTiming,

7
lib/components/fabro-agent/src/sandbox.rs
View file

@ -3,7 +3,8 @@
// `crate::delegate_sandbox!` invocations continue to work.
pub use fabro_sandbox::{
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingRequest, ExecStreamingResult,
GrepOptions, RefreshOutcome, Sandbox, SandboxEvent, SandboxEventCallback, SandboxFile,
StderrCollector, StdioProcess, StdioProcessHandle, StdioProcessTermination, WalkOptions,
delegate_sandbox, format_lines_numbered, shell_quote,
GrepOptions, RefreshOutcome, RemoteCredentialAction, Sandbox, SandboxEvent,
SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess, StdioProcessHandle,
StdioProcessTermination, TokenProvenance, TokenSnapshot, WalkOptions, delegate_sandbox,
format_lines_numbered, shell_quote,
};

4
lib/components/fabro-agent/src/tools.rs
View file

@ -226,7 +226,7 @@ pub fn make_edit_file_tool() -> RegisteredTool {
};
ctx.env
.write_file(file_path, &new_content)
.write_existing_file(file_path, &new_content)
.await
.map_err(|e| e.display_with_causes())?;
Ok(format!("Successfully edited {file_path}"))
@ -1002,6 +1002,7 @@ mod tests {
)
.await;
assert_eq!(result.unwrap(), "Successfully wrote to /out.txt");
assert_eq!(env.existing_file_write_count(), 0);
let written = env.written_files.lock().unwrap();
assert_eq!(written.len(), 1);
assert_eq!(written[0].0, "/out.txt");
@ -1036,6 +1037,7 @@ mod tests {
)
.await;
assert_eq!(result.unwrap(), "Successfully edited /f.txt");
assert_eq!(env.existing_file_write_count(), 1);
let written = env.written_files.lock().unwrap();
assert_eq!(written.len(), 1);
assert_eq!(written[0].1, "goodbye world");

129
lib/components/fabro-dump/src/lib.rs
View file

@ -4,6 +4,7 @@
)]
use std::collections::HashMap;
use std::collections::hash_map::Entry;
#[expect(
clippy::disallowed_types,
reason = "in-memory Vec<u8>::write_all for jsonl serialization; no filesystem or network I/O"
@ -214,31 +215,42 @@ impl RunDump {
for entry in &mut self.entries {
match &mut entry.contents {
RunDumpContents::Json(value) => {
let mut blob_ids = Vec::new();
collect_blob_refs_in_value(value, &mut blob_ids);
for blob_id in blob_ids {
if cache.contains_key(&blob_id) {
let mut blob_hashes = Vec::new();
collect_blob_refs_in_value(value, &mut blob_hashes);
for blob_hash in blob_hashes {
if cache.contains_key(&blob_hash) {
continue;
}
let blob = read_blob(blob_id).await?.with_context(|| {
format!("blob {blob_id:?} is missing from the store")
let blob = read_blob(blob_hash).await?.with_context(|| {
format!("blob {blob_hash:?} is missing from the store")
})?;
let hydrated: serde_json::Value = serde_json::from_slice(&blob)
.with_context(|| format!("blob {blob_id:?} is not valid JSON"))?;
cache.insert(blob_id, hydrated);
.with_context(|| format!("blob {blob_hash:?} is not valid JSON"))?;
cache.insert(blob_hash, hydrated);
}
replace_blob_refs_in_value(value, &cache)?;
}
RunDumpContents::Text(text) => {
let Some(blob_id) = parse_blob_ref(text) else {
let Some(blob_hash) = parse_blob_ref(text) else {
continue;
};
let blob = read_blob(blob_id)
.await?
.with_context(|| format!("blob {blob_id:?} is missing from the store"))?;
*text = serde_json::from_slice::<String>(&blob).with_context(|| {
format!("blob {blob_id:?} is not a JSON string text log")
})?;
let hydrated = match cache.entry(blob_hash) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let blob = read_blob(blob_hash).await?.with_context(|| {
format!("blob {blob_hash:?} is missing from the store")
})?;
let hydrated: serde_json::Value = serde_json::from_slice(&blob)
.with_context(|| format!("blob {blob_hash:?} is not valid JSON"))?;
entry.insert(hydrated)
}
};
*text = hydrated
.as_str()
.with_context(|| {
format!("blob {blob_hash:?} is not a JSON string text log")
})?
.to_string();
}
RunDumpContents::Bytes(_) => {}
}
@ -386,21 +398,21 @@ fn validate_relative_path(kind: &str, value: &str) -> Result<PathBuf> {
Ok(normalized)
}
fn collect_blob_refs_in_value(value: &serde_json::Value, blob_ids: &mut Vec<BlobHash>) {
fn collect_blob_refs_in_value(value: &serde_json::Value, blob_hashes: &mut Vec<BlobHash>) {
match value {
serde_json::Value::String(current) => {
if let Some(blob_id) = parse_blob_ref(current) {
blob_ids.push(blob_id);
if let Some(blob_hash) = parse_blob_ref(current) {
blob_hashes.push(blob_hash);
}
}
serde_json::Value::Array(items) => {
for item in items {
collect_blob_refs_in_value(item, blob_ids);
collect_blob_refs_in_value(item, blob_hashes);
}
}
serde_json::Value::Object(map) => {
for item in map.values() {
collect_blob_refs_in_value(item, blob_ids);
collect_blob_refs_in_value(item, blob_hashes);
}
}
serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {}
@ -413,13 +425,12 @@ fn replace_blob_refs_in_value(
) -> Result<()> {
match value {
serde_json::Value::String(current) => {
let Some(blob_id) = parse_blob_ref(current) else {
let Some(blob_hash) = parse_blob_ref(current) else {
return Ok(());
};
let hydrated = cache
.get(&blob_id)
.cloned()
.with_context(|| format!("blob {blob_id:?} is missing from the hydration cache"))?;
let hydrated = cache.get(&blob_hash).cloned().with_context(|| {
format!("blob {blob_hash:?} is missing from the hydration cache")
})?;
*value = hydrated;
}
serde_json::Value::Array(items) => {
@ -475,8 +486,7 @@ mod tests {
use fabro_types::{
Checkpoint, CheckpointRecord, Conclusion, RunDiff, RunSandbox, RunSandboxInstance,
RunSandboxPlan, RunStatus, SandboxProviderKind, StageCompletion, StageModelUsage,
StageOutcome, StartRecord, SuccessReason, WorkflowSettings, first_event_seq, fixtures,
test_support,
StageOutcome, StartRecord, SuccessReason, first_event_seq, fixtures, test_support,
};
use futures::executor;
@ -484,24 +494,18 @@ mod tests {
fn sample_run_spec() -> RunSpec {
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: Graph::new("ship"),
graph_source: Some("digraph Ship {}".to_string()),
workflow_slug: Some("demo".to_string()),
automation: None,
graph: Graph::new("ship"),
graph_source: Some("digraph Ship {}".to_string()),
workflow_slug: Some("demo".to_string()),
source_directory: Some("/tmp/project".to_string()),
git: Some(fabro_types::GitContext {
git: Some(fabro_types::GitContext {
origin_url: "https://github.com/fabro-sh/fabro.git".to_string(),
branch: "main".to_string(),
sha: None,
dirty: fabro_types::DirtyStatus::Clean,
}),
labels: HashMap::from([("team".to_string(), "platform".to_string())]),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
labels: HashMap::from([("team".to_string(), "platform".to_string())]),
..test_support::test_run_spec()
}
}
@ -724,8 +728,8 @@ mod tests {
#[test]
fn hydrate_referenced_blobs_ignores_legacy_artifact_file_refs() {
let blob = serde_json::to_vec("hydrated legacy text").unwrap();
let blob_id = fabro_types::BlobHash::new(&blob);
let legacy_ref = format!("file:///sandbox/.fabro/artifacts/{blob_id}.json");
let blob_hash = fabro_types::BlobHash::new(&blob);
let legacy_ref = format!("file:///sandbox/.fabro/artifacts/{blob_hash}.json");
let mut dump = RunDump {
entries: vec![RunDumpEntry::json(
"run.json",
@ -736,10 +740,10 @@ mod tests {
};
executor::block_on(async {
dump.hydrate_referenced_blobs_with_reader(|read_blob_id| {
dump.hydrate_referenced_blobs_with_reader(|read_blob_hash| {
let blob = blob.clone();
Box::pin(async move {
assert_eq!(read_blob_id, blob_id);
assert_eq!(read_blob_hash, blob_hash);
Ok(Some(bytes::Bytes::from(blob)))
})
})
@ -752,4 +756,43 @@ mod tests {
};
assert_eq!(value["stdout"], legacy_ref);
}
#[test]
fn hydrate_referenced_blobs_fetches_shared_blobs_once() {
let blob = serde_json::to_vec("offloaded response text").unwrap();
let blob_hash = fabro_types::BlobHash::new(&blob);
let blob_ref = fabro_types::format_blob_ref(&blob_hash);
let mut dump = RunDump {
entries: vec![
RunDumpEntry::json("run.json", serde_json::json!({ "response": blob_ref })),
RunDumpEntry::text("stages/001-demo@1/response.md", blob_ref.clone()),
],
stage_ranks: HashMap::new(),
dump_log_index: None,
};
let reads = std::cell::Cell::new(0);
executor::block_on(async {
dump.hydrate_referenced_blobs_with_reader(|read_blob_hash| {
reads.set(reads.get() + 1);
let blob = blob.clone();
Box::pin(async move {
assert_eq!(read_blob_hash, blob_hash);
Ok(Some(bytes::Bytes::from(blob)))
})
})
.await
})
.unwrap();
assert_eq!(reads.get(), 1, "shared blob should be fetched once");
let RunDumpContents::Json(value) = &dump.entries[0].contents else {
panic!("entry should be JSON");
};
assert_eq!(value["response"], "offloaded response text");
let RunDumpContents::Text(text) = &dump.entries[1].contents else {
panic!("entry should be text");
};
assert_eq!(text, "offloaded response text");
}
}

5
lib/components/fabro-github/Cargo.toml
View file

@ -9,12 +9,17 @@ description = "GitHub App authentication and API helpers for Fabro"
[lib]
doctest = false
[features]
test-support = []
[lints]
workspace = true
[dependencies]
anyhow.workspace = true
async-trait.workspace = true
serde.workspace = true
strum.workspace = true
serde_json.workspace = true
fabro-http.workspace = true
fabro-redact.workspace = true

5
lib/components/fabro-github/src/lib.rs
View file

@ -9,6 +9,11 @@ use fabro_types::settings::run::MergeStrategy;
use serde::Deserialize;
use tokio::process::Command;
pub mod token_source;
#[cfg(any(test, feature = "test-support"))]
pub mod test_support;
pub const GITHUB_API_BASE_URL: &str = "https://api.github.com";
/// Returns the GitHub API base URL, allowing override via `GITHUB_BASE_URL` env

26
lib/components/fabro-github/src/test_support.rs Normal file
View file

@ -0,0 +1,26 @@
use std::sync::Arc;
use crate::InstallationToken;
use crate::token_source::{InstallationTokenMinter as InnerMinter, InstallationTokenSource};
#[async_trait::async_trait]
pub trait InstallationTokenMinter: Send + Sync {
async fn mint(&self) -> anyhow::Result<InstallationToken>;
}
struct TestMinterAdapter(Arc<dyn InstallationTokenMinter>);
#[async_trait::async_trait]
impl InnerMinter for TestMinterAdapter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.0.mint().await
}
}
#[must_use]
pub fn installation_token_source(
repo: impl Into<String>,
minter: Arc<dyn InstallationTokenMinter>,
) -> Arc<InstallationTokenSource> {
InstallationTokenSource::with_minter(repo.into(), Box::new(TestMinterAdapter(minter)))
}

685
lib/components/fabro-github/src/token_source.rs Normal file
View file

@ -0,0 +1,685 @@
//! Cached GitHub installation-token source.
//!
//! One [`InstallationTokenSource`] can serve GitHub-token consumers that share
//! a repository and permission scope. Reusing mature tokens keeps consumers
//! out of GitHub's token-replication lag window, where a token minted
//! milliseconds earlier is rejected with 404 "Repository not found" or an
//! authentication failure.
//!
//! The source also reports *provenance*: when it minted the token it returned,
//! and which mint generation it belongs to. Retry classification, logging, and
//! failure reports all read that one fact instead of threading booleans
//! through call stacks.
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context as _;
use chrono::{DateTime, Utc};
use tokio::sync::Mutex;
use crate::{GitHubAppCredentials, GitHubCredentials, InstallationToken};
/// How long before expiry a cached installation token stops being reused.
///
/// Must comfortably exceed the longest git operation that pins a resolved
/// token, so a token handed out just above the margin still outlives the
/// operation. GitHub App installation tokens live 60 minutes.
pub const REFRESH_MARGIN: Duration = Duration::from_mins(10);
/// Where the token a resolve returned came from.
///
/// Time metadata exists only for tokens this source minted. Static
/// credentials (a PAT, or a pre-minted installation token) carry no
/// `minted_at`, so token age is undefined for them and they are never
/// treated as freshly minted.
#[derive(
Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, strum::Display,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum TokenProvenance {
/// This resolve minted the token.
Minted {
minted_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
},
/// This resolve returned a token minted by an earlier resolve.
Reused {
minted_at: DateTime<Utc>,
expires_at: DateTime<Utc>,
},
/// A fixed credential the source cannot re-mint.
Static,
}
/// Non-secret description of the token a resolve returned. Shared by the
/// source, refresh outcomes, logs, and events.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TokenSnapshot {
/// Increments per mint; 0 for `Static`.
pub generation: u64,
pub provenance: TokenProvenance,
}
impl TokenSnapshot {
#[must_use]
pub fn minted_at(&self) -> Option<DateTime<Utc>> {
match self.provenance {
TokenProvenance::Minted { minted_at, .. }
| TokenProvenance::Reused { minted_at, .. } => Some(minted_at),
TokenProvenance::Static => None,
}
}
#[must_use]
pub fn expires_at(&self) -> Option<DateTime<Utc>> {
match self.provenance {
TokenProvenance::Minted { expires_at, .. }
| TokenProvenance::Reused { expires_at, .. } => Some(expires_at),
TokenProvenance::Static => None,
}
}
/// Age of the token at `now`. `None` for static credentials, whose age is
/// undefined.
#[must_use]
pub fn age_at(&self, now: DateTime<Utc>) -> Option<Duration> {
let minted_at = self.minted_at()?;
Some((now - minted_at).to_std().unwrap_or(Duration::ZERO))
}
/// Age of the token in milliseconds, measured now.
#[must_use]
pub fn age_ms(&self) -> Option<u64> {
self.age_at(Utc::now())
.map(|age| u64::try_from(age.as_millis()).unwrap_or(u64::MAX))
}
#[must_use]
pub fn is_static(&self) -> bool {
matches!(self.provenance, TokenProvenance::Static)
}
}
/// A token secret that never appears in `Debug` output. Call
/// [`SecretString::expose`] at the point of use (URL embedding, git
/// credentials) — never in a log line.
#[derive(Clone)]
pub struct SecretString(String);
impl SecretString {
#[must_use]
pub fn new(secret: String) -> Self {
Self(secret)
}
#[must_use]
pub fn expose(&self) -> &str {
&self.0
}
}
impl fmt::Debug for SecretString {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("SecretString(<redacted>)")
}
}
/// A token handed out by [`InstallationTokenSource::resolve`]: the secret plus
/// its non-secret snapshot. Only the snapshot may cross logging or event
/// boundaries.
#[derive(Debug, Clone)]
pub struct ResolvedToken {
pub token: SecretString,
pub snapshot: TokenSnapshot,
/// The source tried to refresh an expiring token, but returned the still-
/// valid cached token after the mint failed.
pub refresh_failed: bool,
}
/// Mints installation tokens for [`InstallationTokenSource`]. Abstracted so
/// tests can script mint results without HTTP.
#[async_trait::async_trait]
pub(crate) trait InstallationTokenMinter: Send + Sync {
async fn mint(&self) -> anyhow::Result<InstallationToken>;
}
/// Real minter backed by GitHub App credentials.
struct AppTokenMinter {
creds: GitHubAppCredentials,
http: fabro_http::HttpClient,
owner: String,
repo: String,
base_url: String,
permissions: serde_json::Value,
}
#[async_trait::async_trait]
impl InstallationTokenMinter for AppTokenMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.creds
.mint_installation_token(
&self.http,
&self.owner,
&self.repo,
&self.base_url,
self.permissions.clone(),
None,
)
.await
}
}
/// A minted token plus the metadata the cache tracks for it.
struct CachedToken {
token: InstallationToken,
minted_at: DateTime<Utc>,
generation: u64,
}
impl CachedToken {
fn resolved(&self, provenance: TokenProvenance) -> ResolvedToken {
ResolvedToken {
token: SecretString::new(self.token.token.clone()),
snapshot: TokenSnapshot {
generation: self.generation,
provenance,
},
refresh_failed: false,
}
}
}
enum SourceState {
/// A fixed personal access token — no expiry metadata.
Pat(SecretString),
/// A pre-minted installation token — fixed, rejected client-side once
/// expired.
Installation(InstallationToken),
/// GitHub App credentials that mint installation tokens on demand.
///
/// The async lock is held across the mint, making `resolve()`
/// single-flight: concurrent near-expiry callers wait and receive the
/// same generation instead of racing to mint.
App {
minter: Box<dyn InstallationTokenMinter>,
cache: Mutex<Option<CachedToken>>,
},
}
/// Cached installation-token source for one origin repository.
///
/// Static credentials pass through unchanged. App credentials mint through
/// the shared cache: a resolve reuses the cached token until it is within
/// [`REFRESH_MARGIN`] of expiry, then mints a new generation.
pub struct InstallationTokenSource {
/// `owner/repo`, for logs only.
repo: String,
state: SourceState,
}
impl InstallationTokenSource {
/// Build a source for `creds` against the repository in `origin_url`.
///
/// `permissions` scopes minted installation tokens; static credentials
/// pass through and ignore it.
pub fn for_origin(
creds: &GitHubCredentials,
origin_url: &str,
permissions: serde_json::Value,
) -> anyhow::Result<Arc<Self>> {
let normalized = crate::normalize_repo_origin_url(origin_url);
let (owner, repo) = crate::parse_github_owner_repo(&normalized)
.context("parsing GitHub origin for token source")?;
Self::for_repository(creds, owner, repo, permissions)
}
/// Build a source for an already parsed GitHub repository.
pub fn for_repository(
creds: &GitHubCredentials,
owner: String,
repo: String,
permissions: serde_json::Value,
) -> anyhow::Result<Arc<Self>> {
let repo_display = format!("{owner}/{repo}");
let state = match creds {
GitHubCredentials::Pat(token) => SourceState::Pat(SecretString::new(token.clone())),
GitHubCredentials::Installation(token) => SourceState::Installation(token.clone()),
GitHubCredentials::App(app) => {
let http = fabro_http::http_client()
.map_err(anyhow::Error::new)
.context("building HTTP client for token source")?;
SourceState::App {
minter: Box::new(AppTokenMinter {
creds: app.clone(),
http,
owner,
repo,
base_url: crate::github_api_base_url(),
permissions,
}),
cache: Mutex::new(None),
}
}
};
Ok(Arc::new(Self {
repo: repo_display,
state,
}))
}
/// Build a source for a personal access token.
#[must_use]
pub fn pat(token: String) -> Arc<Self> {
Arc::new(Self {
repo: String::new(),
state: SourceState::Pat(SecretString::new(token)),
})
}
/// Build a source for a pre-minted installation token.
#[must_use]
pub fn installation(token: InstallationToken) -> Arc<Self> {
Arc::new(Self {
repo: String::new(),
state: SourceState::Installation(token),
})
}
/// Build a minting source over a custom minter.
#[cfg(any(test, feature = "test-support"))]
#[must_use]
pub(crate) fn with_minter(repo: String, minter: Box<dyn InstallationTokenMinter>) -> Arc<Self> {
Arc::new(Self {
repo,
state: SourceState::App {
minter,
cache: Mutex::new(None),
},
})
}
/// Whether this source can mint new tokens (GitHub App credentials).
#[must_use]
pub fn mints_installation_tokens(&self) -> bool {
matches!(self.state, SourceState::App { .. })
}
/// Resolve a token, reusing the cached one until it nears expiry.
pub async fn resolve(&self) -> anyhow::Result<ResolvedToken> {
match &self.state {
SourceState::Pat(_) | SourceState::Installation(_) => self.resolve_static(),
SourceState::App { minter, cache } => {
let mut cache = cache.lock().await;
// Re-check under the lock: a waiter queued behind a minter
// finds the fresh token here instead of minting again.
if let Some(cached) = cache.as_ref() {
if !cached.token.near_expiry(REFRESH_MARGIN) {
let resolved = cached.resolved(TokenProvenance::Reused {
minted_at: cached.minted_at,
expires_at: cached.token.expires_at,
});
tracing::debug!(
repo = %self.repo,
generation = cached.generation,
expires_at = %cached.token.expires_at,
"Reusing cached GitHub installation token"
);
return Ok(resolved);
}
}
match self.mint_locked(minter.as_ref(), &mut cache).await {
Ok(resolved) => Ok(resolved),
Err(err) => {
if let Some(cached) = cache.as_ref() {
if cached.token.valid_token().is_ok() {
tracing::warn!(
error = %format!("{err:#}"),
repo = %self.repo,
generation = cached.generation,
expires_at = %cached.token.expires_at,
"GitHub installation token refresh failed; using cached token"
);
let mut resolved = cached.resolved(TokenProvenance::Reused {
minted_at: cached.minted_at,
expires_at: cached.token.expires_at,
});
resolved.refresh_failed = true;
return Ok(resolved);
}
}
Err(err)
}
}
}
}
}
/// Mint a fresh token for the first repository clone and seed the cache
/// with it.
///
/// The clone deliberately never reuses a warm cache: retrying a clone with
/// the token minted for it is the established replication-lag recovery,
/// and reuse of older tokens for clones is a separate follow-up. Seeding
/// makes the clone token generation 1, so later refreshes reuse it until
/// it nears expiry.
pub async fn mint_for_clone(&self) -> anyhow::Result<ResolvedToken> {
match &self.state {
SourceState::Pat(_) | SourceState::Installation(_) => self.resolve_static(),
SourceState::App { minter, cache } => {
let mut cache = cache.lock().await;
self.mint_locked(minter.as_ref(), &mut cache).await
}
}
}
fn resolve_static(&self) -> anyhow::Result<ResolvedToken> {
let secret = match &self.state {
SourceState::Pat(token) => token.clone(),
SourceState::Installation(token) => SecretString::new(token.valid_token()?.to_owned()),
SourceState::App { .. } => unreachable!("resolve_static called for App credentials"),
};
Ok(ResolvedToken {
token: secret,
snapshot: TokenSnapshot {
generation: 0,
provenance: TokenProvenance::Static,
},
refresh_failed: false,
})
}
async fn mint_locked(
&self,
minter: &dyn InstallationTokenMinter,
cache: &mut Option<CachedToken>,
) -> anyhow::Result<ResolvedToken> {
let token = minter
.mint()
.await
.context("minting GitHub installation access token")?;
let generation = cache.as_ref().map_or(0, |cached| cached.generation) + 1;
let minted_at = Utc::now();
tracing::info!(
repo = %self.repo,
generation,
expires_at = %token.expires_at,
"Minted GitHub installation token"
);
let cached = CachedToken {
token,
minted_at,
generation,
};
let resolved = cached.resolved(TokenProvenance::Minted {
minted_at,
expires_at: cached.token.expires_at,
});
*cache = Some(cached);
Ok(resolved)
}
}
#[cfg(test)]
mod tests {
use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::anyhow;
use super::*;
enum MintAction {
Token(&'static str, DateTime<Utc>),
Error(&'static str),
}
struct MockMinter {
calls: AtomicUsize,
script: Mutex<VecDeque<MintAction>>,
}
impl MockMinter {
fn new(script: Vec<MintAction>) -> Self {
Self {
calls: AtomicUsize::new(0),
script: Mutex::new(script.into()),
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
#[async_trait::async_trait]
impl InstallationTokenMinter for MockMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.calls.fetch_add(1, Ordering::SeqCst);
match self.script.lock().await.pop_front().expect("mint script") {
MintAction::Token(token, expires_at) => Ok(InstallationToken {
token: token.to_string(),
expires_at,
}),
MintAction::Error(message) => Err(anyhow!(message)),
}
}
}
struct SharedMinter(Arc<MockMinter>);
#[async_trait::async_trait]
impl InstallationTokenMinter for SharedMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.0.mint().await
}
}
fn mintable(script: Vec<MintAction>) -> (Arc<InstallationTokenSource>, Arc<MockMinter>) {
let minter = Arc::new(MockMinter::new(script));
let source = InstallationTokenSource::with_minter(
"owner/repo".to_string(),
Box::new(SharedMinter(Arc::clone(&minter))),
);
(source, minter)
}
#[tokio::test]
async fn pat_resolves_as_static_generation_zero() {
let source = InstallationTokenSource::for_origin(
&GitHubCredentials::Pat("ghp_pat".to_string()),
"https://github.com/owner/repo.git",
serde_json::json!({ "contents": "write" }),
)
.unwrap();
let resolved = source.resolve().await.unwrap();
assert_eq!(resolved.token.expose(), "ghp_pat");
assert_eq!(resolved.snapshot.generation, 0);
assert!(resolved.snapshot.is_static());
assert!(!source.mints_installation_tokens());
}
#[tokio::test]
async fn static_installation_token_resolves_until_expiry() {
let valid = InstallationTokenSource::for_origin(
&GitHubCredentials::Installation(InstallationToken {
token: "ghs_static".to_string(),
expires_at: Utc::now() + chrono::Duration::minutes(30),
}),
"https://github.com/owner/repo.git",
serde_json::json!({}),
)
.unwrap();
let resolved = valid.resolve().await.unwrap();
assert_eq!(resolved.token.expose(), "ghs_static");
assert!(resolved.snapshot.is_static());
let expired = InstallationTokenSource::for_origin(
&GitHubCredentials::Installation(InstallationToken {
token: "ghs_expired".to_string(),
expires_at: Utc::now() - chrono::Duration::seconds(1),
}),
"https://github.com/owner/repo.git",
serde_json::json!({}),
)
.unwrap();
assert!(expired.resolve().await.is_err());
}
#[tokio::test]
async fn resolve_reuses_cached_token_before_the_margin() {
let (source, minter) = mintable(vec![MintAction::Token(
"ghs_gen1",
Utc::now() + chrono::Duration::minutes(30),
)]);
let first = source.resolve().await.unwrap();
let second = source.resolve().await.unwrap();
assert_eq!(minter.calls(), 1);
assert_eq!(first.snapshot.generation, 1);
assert_eq!(second.snapshot.generation, 1);
assert!(matches!(
first.snapshot.provenance,
TokenProvenance::Minted { .. }
));
assert!(matches!(
second.snapshot.provenance,
TokenProvenance::Reused { .. }
));
assert_eq!(second.token.expose(), "ghs_gen1");
}
#[tokio::test]
async fn resolve_mints_a_new_generation_inside_the_margin() {
let (source, minter) = mintable(vec![
// Expires inside REFRESH_MARGIN, so the second resolve re-mints.
MintAction::Token("ghs_gen1", Utc::now() + chrono::Duration::minutes(5)),
MintAction::Token("ghs_gen2", Utc::now() + chrono::Duration::minutes(60)),
]);
let first = source.resolve().await.unwrap();
let second = source.resolve().await.unwrap();
assert_eq!(minter.calls(), 2);
assert_eq!(first.snapshot.generation, 1);
assert_eq!(second.snapshot.generation, 2);
assert!(matches!(
second.snapshot.provenance,
TokenProvenance::Minted { .. }
));
assert_eq!(second.token.expose(), "ghs_gen2");
}
#[tokio::test]
async fn resolve_uses_a_valid_cached_token_when_refresh_fails() {
let (source, minter) = mintable(vec![
MintAction::Token("ghs_gen1", Utc::now() + chrono::Duration::minutes(5)),
MintAction::Error("mint failed"),
]);
let first = source.resolve().await.unwrap();
let second = source.resolve().await.unwrap();
assert_eq!(minter.calls(), 2);
assert_eq!(first.snapshot.generation, 1);
assert_eq!(second.snapshot.generation, 1);
assert!(second.refresh_failed);
assert!(matches!(
second.snapshot.provenance,
TokenProvenance::Reused { .. }
));
assert_eq!(second.token.expose(), "ghs_gen1");
}
#[tokio::test]
async fn concurrent_resolves_share_one_generation() {
// Single mint in the script: a second mint would panic on an empty
// script, so success proves single-flight.
let (source, minter) = mintable(vec![MintAction::Token(
"ghs_gen1",
Utc::now() + chrono::Duration::minutes(60),
)]);
let handles: Vec<_> = (0..8)
.map(|_| {
let source = Arc::clone(&source);
tokio::spawn(async move { source.resolve().await })
})
.collect();
for handle in handles {
let resolved = handle.await.unwrap().unwrap();
assert_eq!(resolved.snapshot.generation, 1);
assert_eq!(resolved.token.expose(), "ghs_gen1");
}
assert_eq!(minter.calls(), 1);
}
#[tokio::test]
async fn mint_for_clone_always_mints_and_seeds_the_cache() {
let (source, minter) = mintable(vec![MintAction::Token(
"ghs_clone",
Utc::now() + chrono::Duration::minutes(60),
)]);
let clone_token = source.mint_for_clone().await.unwrap();
assert_eq!(clone_token.snapshot.generation, 1);
assert!(matches!(
clone_token.snapshot.provenance,
TokenProvenance::Minted { .. }
));
// A later resolve reuses the clone token instead of minting again.
let refreshed = source.resolve().await.unwrap();
assert_eq!(refreshed.snapshot.generation, 1);
assert_eq!(refreshed.token.expose(), "ghs_clone");
assert!(matches!(
refreshed.snapshot.provenance,
TokenProvenance::Reused { .. }
));
assert_eq!(minter.calls(), 1);
}
#[tokio::test]
async fn mint_failure_surfaces_with_context() {
let (source, _minter) = mintable(vec![MintAction::Error("mint failed")]);
let err = format!("{:#}", source.resolve().await.unwrap_err());
assert!(err.contains("mint failed"), "got: {err}");
assert!(
err.contains("minting GitHub installation access token"),
"got: {err}"
);
}
#[test]
fn secret_string_debug_never_prints_the_secret() {
let secret = SecretString::new("ghs_super_secret".to_string());
let rendered = format!("{secret:?}");
assert!(!rendered.contains("ghs_super_secret"), "{rendered}");
}
#[test]
fn snapshot_age_is_defined_only_for_minted_tokens() {
let now = Utc::now();
let minted = TokenSnapshot {
generation: 3,
provenance: TokenProvenance::Minted {
minted_at: now - chrono::Duration::seconds(42),
expires_at: now + chrono::Duration::minutes(60),
},
};
assert_eq!(minted.age_at(now), Some(Duration::from_secs(42)));
let fixed = TokenSnapshot {
generation: 0,
provenance: TokenProvenance::Static,
};
assert_eq!(fixed.age_at(now), None);
assert_eq!(fixed.expires_at(), None);
}
}

92
lib/components/fabro-llm/src/adapter_registry.rs
View file

@ -269,91 +269,19 @@ mod tests {
.unwrap_or_else(|error| panic!("built-in model '{selector}' should resolve: {error}"))
}
/// One row of the route-equivalence table: model id plus the
/// `(deployment_id, transport, codec, billing_policy, agent_profile)`
/// tuple it must resolve to.
type RouteRow = (
&'static str,
&'static str,
AdapterKind,
CodecKind,
BillingPolicy,
AgentProfileKind,
);
/// The compat mapping as an executable table: every built-in catalog
/// model resolves to exactly this tuple. Adding or rerouting a built-in
/// model means updating this table deliberately.
#[test]
fn builtin_catalog_route_equivalence_table() {
use AdapterKind as T;
use AgentProfileKind as P;
use BillingPolicy as B;
use CodecKind as C;
#[rustfmt::skip]
let expected: &[RouteRow] = &[
// model id deployment_id transport codec billing profile
("claude-fable-5", "claude-fable-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5),
("claude-haiku-4-5", "claude-haiku-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-opus-4-6", "claude-opus-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-opus-4-7", "claude-opus-4-7", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-opus-4-8", "claude-opus-4-8", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-opus-5", "claude-opus-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5),
("claude-sonnet-4-5", "claude-sonnet-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-sonnet-4-6", "claude-sonnet-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic),
("claude-sonnet-5", "claude-sonnet-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5),
("deepseek-v4-flash", "deepseek-v4-flash", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("deepseek-v4-pro", "deepseek-v4-pro", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("gemini-3-flash-preview", "gemini-3-flash-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini),
("gemini-3.1-flash-lite", "gemini-3.1-flash-lite", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini),
("gemini-3.1-pro-preview", "gemini-3.1-pro-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini),
("gemini-3.1-pro-preview-customtools", "gemini-3.1-pro-preview-customtools", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini),
("gemini-3.5-flash", "gemini-3.5-flash", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini),
("glm-4.7", "glm-4.7", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("glm-5.2", "glm-5.2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("gpt-5.4", "gpt-5.4", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi),
("gpt-5.4-mini", "gpt-5.4-mini", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi),
("gpt-5.4-pro", "gpt-5.4-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi),
("gpt-5.5", "gpt-5.5", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi),
("gpt-5.5-pro", "gpt-5.5-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi),
("gpt-5.6-luna", "gpt-5.6-luna", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56),
("gpt-5.6-sol", "gpt-5.6-sol", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56),
("gpt-5.6-terra", "gpt-5.6-terra", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56),
("kimi-k2.5", "kimi-k2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi),
("kimi-k3", "kimi-k3", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi),
("laguna-s-2.1", "poolside/laguna-s-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("laguna-xs-2.1", "poolside/laguna-xs-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("mercury-2", "mercury-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("minimax-m2.5", "minimax-m2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("venice-uncensored-1-2", "venice-uncensored-1-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
("venice-uncensored-role-play", "venice-uncensored-role-play", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi),
];
fn every_builtin_catalog_offering_resolves() {
let catalog = Catalog::builtin();
let mut model_ids: Vec<&str> = catalog
.list(None)
.iter()
.map(|model| model.id.as_str())
.collect();
model_ids.sort_unstable();
let mut expected_ids: Vec<&str> = expected.iter().map(|row| row.0).collect();
expected_ids.sort_unstable();
assert_eq!(
model_ids, expected_ids,
"route-equivalence table must cover every built-in model row"
);
for (model_id, deployment_id, transport, codec, billing_policy, agent_profile) in expected {
let model = select_from_all(catalog, model_id);
let route = resolve_route(catalog, model)
.unwrap_or_else(|| panic!("built-in model '{model_id}' should resolve"));
assert_eq!(route.deployment_id, *deployment_id, "{model_id}");
assert_eq!(route.transport, *transport, "{model_id}");
assert_eq!(route.codec, *codec, "{model_id}");
assert_eq!(route.billing_policy, *billing_policy, "{model_id}");
assert_eq!(route.agent_profile, *agent_profile, "{model_id}");
for model in catalog.list(None) {
let route = resolve_route(catalog, model).unwrap_or_else(|| {
panic!(
"built-in offering '{}/{}' should resolve",
model.provider, model.id
)
});
assert_eq!(route.provider, model.provider);
assert!(!route.deployment_id.is_empty());
}
}

13
lib/components/fabro-llm/src/client.rs
View file

@ -2037,17 +2037,20 @@ reasoning = false
client
}
/// Live-dispatch counterpart of the adapter_registry route-equivalence
/// table: for every built-in model, `resolve_provider` lands on the same
/// provider the resolved route names.
/// For every built-in model selector, live dispatch and catalog selection
/// choose the same provider from the same ready-provider set.
#[tokio::test]
async fn dispatch_agrees_with_resolve_route_for_every_builtin_model() {
let catalog = catalog_with("");
let client = client_with_all_catalog_providers(&catalog).await;
let ready_providers = catalog.all_provider_ids();
for model in catalog.list(None) {
let route = adapter_registry::resolve_route(&catalog, model)
.expect("built-in model should resolve to a route");
let selected = catalog
.select(model.id.as_str(), None, &ready_providers)
.expect("built-in model should be selectable");
let route = adapter_registry::resolve_route(&catalog, selected)
.expect("selected built-in model should resolve to a route");
let mut request = test_request();
request.model = model.id.to_string();

56
lib/components/fabro-llm/src/error.rs
View file

@ -355,7 +355,14 @@ pub fn error_from_status_code(
// error types
let kind = match status_code {
401 => ProviderErrorKind::Authentication,
403 => ProviderErrorKind::AccessDenied,
// A 412 is never about the request: no LLM request carries
// conditional-request preconditions. Fireworks documents it as
// "Account is suspended or there's an issue with account status",
// also emitted for a LoRA model that failed to load
// (https://docs.fireworks.ai/guides/inference-error-codes). The same
// family as `account_deactivated`: deterministic here, but another
// provider has independent billing and model inventory.
403 | 412 => ProviderErrorKind::AccessDenied,
404 => ProviderErrorKind::NotFound,
408 => {
return Error::RequestTimeout {
@ -728,6 +735,53 @@ mod tests {
assert_eq!(err.provider_kind(), Some(ProviderErrorKind::QuotaExceeded));
}
/// Fireworks reports an account suspension (spending cap reached or
/// unpaid invoices) as HTTP 412 with `code: "PRECONDITION_FAILED"` in
/// the body. A chat completion carries no conditional-request
/// preconditions, so a 412 is always an account-level lockout, never a
/// defect in the request: it must not classify as `InvalidRequest`, and
/// a fallback provider with independent billing must stay eligible.
#[test]
fn account_suspension_412_is_failover_eligible() {
let err = error_from_status_code(
412,
"Account lithoscomputer is suspended, possibly due to reaching \
the monthly spending limit or failure to pay past invoices."
.into(),
"fireworks".into(),
// The openai_compatible dialect reads `error.type` as the code,
// so the discriminating `PRECONDITION_FAILED` only reaches this
// mapping through the status code.
Some("error".into()),
Some(serde_json::json!({
"error": {
"message": "Account lithoscomputer is suspended, possibly due to reaching the monthly spending limit or failure to pay past invoices. Please go to https://fireworks.ai/account/billing for more information.",
"param": null,
"code": "PRECONDITION_FAILED",
"type": "error"
},
"request_id": "chatcmpl-d9652b89a6604931ac27dddd5ef5bdc0"
})),
None,
);
assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied));
assert!(!err.retryable());
assert!(err.failover_eligible());
// A bare 412 with no parseable body classifies the same way.
let err = error_from_status_code(
412,
"Precondition Failed".into(),
"fireworks".into(),
None,
None,
None,
);
assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied));
assert!(err.failover_eligible());
}
#[test]
fn kind_from_error_code_covers_every_dialect() {
for (code, expected) in [

43
lib/components/fabro-llm/src/model_test.rs
View file

@ -1,3 +1,4 @@
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
@ -63,22 +64,38 @@ pub async fn run_basic_model_probe(
model_id: &str,
provider: impl ToString,
client: Arc<Client>,
) -> ModelTestOutcome {
run_basic_model_probe_with_timeout(
model_id,
provider,
client,
Duration::from_secs(ModelTestMode::Basic.timeout_secs()),
)
.await
}
pub async fn run_basic_model_probe_with_timeout(
model_id: &str,
provider: impl ToString,
client: Arc<Client>,
probe_timeout: Duration,
) -> ModelTestOutcome {
let params = GenerateParams::new(model_id, client)
.provider(provider.to_string())
.prompt("Say OK")
.max_tokens(16);
let result = time::timeout(
Duration::from_secs(ModelTestMode::Basic.timeout_secs()),
generate::generate(params),
)
.await;
basic_model_probe_outcome(generate::generate(params), probe_timeout).await
}
match result {
async fn basic_model_probe_outcome<F>(probe: F, probe_timeout: Duration) -> ModelTestOutcome
where
F: Future<Output = Result<GenerateResult, crate::Error>>,
{
match time::timeout(probe_timeout, probe).await {
Ok(Ok(_)) => ModelTestOutcome::ok(),
Ok(Err(err)) => ModelTestOutcome::error(err.to_string()),
Err(_) => ModelTestOutcome::error("timeout (30s)"),
Err(_) => ModelTestOutcome::error(format!("timeout ({probe_timeout:?})")),
}
}
@ -244,6 +261,18 @@ mod tests {
);
}
#[tokio::test]
async fn basic_model_probe_reports_configured_timeout() {
let outcome = basic_model_probe_outcome(
std::future::pending::<Result<GenerateResult, crate::Error>>(),
Duration::from_millis(1),
)
.await;
assert_eq!(outcome.status, ModelTestStatus::Error);
assert_eq!(outcome.error_message.as_deref(), Some("timeout (1ms)"));
}
#[test]
fn deep_test_omits_effort_for_reasoning_without_effort_controls() {
let info = test_model_with(ModelFeatures {

7
lib/components/fabro-sandbox/Cargo.toml
View file

@ -9,8 +9,8 @@ description = "Sandbox trait and implementations for Fabro agent execution envir
[features]
default = ["local"]
local = []
docker = ["dep:bollard", "dep:tar", "dep:fabro-github"]
daytona = ["dep:daytona-sdk", "dep:daytona-api-client", "dep:git2", "dep:fabro-github", "dep:fabro-config", "dep:fabro-http", "dep:reqwest-middleware", "dep:rand", "dep:tokio-tungstenite", "dep:futures-util", "dep:rustls"]
docker = ["dep:bollard", "dep:tar"]
daytona = ["dep:daytona-sdk", "dep:daytona-api-client", "dep:git2", "dep:fabro-config", "dep:fabro-http", "dep:reqwest-middleware", "dep:rand", "dep:tokio-tungstenite", "dep:futures-util", "dep:rustls"]
test-support = []
[lib]
@ -47,7 +47,7 @@ tar = { workspace = true, optional = true }
# daytona
fabro-config = { path = "../../foundation/fabro-config", optional = true }
fabro-github = { path = "../fabro-github", optional = true }
fabro-github = { path = "../fabro-github" }
fabro-types = { path = "../../foundation/fabro-types" }
chrono = { workspace = true }
@ -64,6 +64,7 @@ futures-util = { workspace = true, optional = true }
rustls = { version = "0.23", default-features = false, features = ["std", "ring"], optional = true }
[dev-dependencies]
fabro-github = { path = "../fabro-github", features = ["test-support"] }
tokio = { workspace = true, features = ["test-util", "macros"] }
tempfile = "3"
serde_json.workspace = true

407
lib/components/fabro-sandbox/src/clone_retry.rs
View file

@ -1,407 +0,0 @@
//! Retry for the first repository clone in a clone-based sandbox.
//!
//! Clone-based providers can mint a GitHub App installation token and clone
//! with it immediately. GitHub can reject that first clone before the token is
//! available to the git endpoint. On a private repository, the rejection can
//! arrive as `Repository not found.` or an authentication failure.
//!
//! Only a token minted during the current clone operation makes those messages
//! safe to retry. Static PATs and pre-minted installation tokens fail fast.
//!
//! Retries reuse the same token on purpose. Replication of a given token only
//! makes progress, so each attempt strictly improves the odds, while re-minting
//! would restart the replication clock.
use std::future::Future;
use std::time::Duration;
use fabro_types::SandboxProviderKind;
use fabro_util::backoff::BackoffPolicy;
use tokio::time;
/// Total clone attempts, including the first.
const MAX_ATTEMPTS: u32 = 3;
/// Why a failed clone attempt is worth repeating.
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)]
#[strum(serialize_all = "snake_case")]
pub(crate) enum CloneRetryReason {
/// A freshly minted installation token has not reached the GitHub edge
/// cache site serving this clone yet.
TokenReplication,
/// The clone failed on infrastructure, unrelated to credentials.
TransientInfra,
}
/// What a clone failure message tells us about retry safety.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum CloneMessageClass {
Retry(CloneRetryReason),
Permanent,
Unknown,
}
impl CloneMessageClass {
pub(crate) fn retry_reason(self) -> Option<CloneRetryReason> {
match self {
Self::Retry(reason) => Some(reason),
Self::Permanent | Self::Unknown => None,
}
}
}
/// A failed clone or fetch attempt: the terminal error plus whether the
/// failure is worth retrying.
pub(crate) struct CloneAttemptFailure {
pub(crate) error: crate::Error,
pub(crate) retry_reason: Option<CloneRetryReason>,
}
/// Message fragments that mean the clone failed on infrastructure.
///
/// These are safe to retry whether or not the clone was authenticated.
const TRANSIENT_HINTS: &[&str] = &[
"could not resolve host",
"temporary failure in name resolution",
"connection refused",
"connection reset",
"connection timed out",
"timed out",
"network is unreachable",
"no route to host",
"tls handshake",
"early eof",
"rpc failed",
"unexpected disconnect",
"the remote end hung up unexpectedly",
"index-pack failed",
"service unavailable",
"gateway timeout",
"too many requests",
"rate limit",
];
/// Message fragments GitHub uses when a token is not yet visible.
///
/// Only meaningful when the clone carried credentials. The same lag surfaces as
/// 404 or as an auth failure depending on which endpoint answers first.
const TOKEN_REPLICATION_HINTS: &[&str] = &[
"repository not found",
"authentication failed",
"invalid username or password",
"bad credentials",
];
/// Classify a failed clone by its rendered message.
///
/// `token_was_freshly_minted` gates the token-replication reading. A static
/// credential cannot become valid during backoff, so auth failures for it are
/// permanent.
pub(crate) fn classify_message(message: &str, token_was_freshly_minted: bool) -> CloneMessageClass {
let lower = message.to_ascii_lowercase();
if TRANSIENT_HINTS.iter().any(|hint| lower.contains(hint)) {
return CloneMessageClass::Retry(CloneRetryReason::TransientInfra);
}
if TOKEN_REPLICATION_HINTS
.iter()
.any(|hint| lower.contains(hint))
{
return if token_was_freshly_minted {
CloneMessageClass::Retry(CloneRetryReason::TokenReplication)
} else {
CloneMessageClass::Permanent
};
}
let permanent = lower.contains("could not read username")
|| lower.contains("terminal prompts disabled")
|| lower.contains("permission denied")
|| (lower.contains("permission to") && lower.contains("denied"))
|| (lower.contains("destination path") && lower.contains("already exists"))
|| (lower.contains("remote branch") && lower.contains("not found"));
if permanent {
return CloneMessageClass::Permanent;
}
CloneMessageClass::Unknown
}
/// Backoff between clone attempts: 3s, then 9s.
///
/// GitHub's guidance for token replication is to wait a few seconds and retry
/// with the same token. Sub-second delays land inside the same replication
/// window and spend an attempt for nothing.
fn backoff() -> BackoffPolicy {
BackoffPolicy {
initial_delay: Duration::from_secs(3),
factor: 3.0,
max_delay: Duration::from_secs(10),
jitter: false,
}
}
/// Run a clone, repeating it while the failure looks transient.
///
/// `attempt` receives the 1-based attempt number. `classify` decides whether an
/// error is worth repeating; `None` returns it to the caller untouched. When a
/// deadline is present, a retry starts only when its backoff fits before that
/// deadline. The final error is returned as-is.
pub(crate) async fn retry_clone<T, E, Attempt, Fut, Classify>(
provider: SandboxProviderKind,
deadline: Option<time::Instant>,
mut attempt: Attempt,
classify: Classify,
) -> Result<T, E>
where
Attempt: FnMut(u32) -> Fut,
Fut: Future<Output = Result<T, E>>,
Classify: Fn(&E) -> Option<CloneRetryReason>,
{
let backoff = backoff();
for attempt_number in 1..MAX_ATTEMPTS {
match attempt(attempt_number).await {
Ok(value) => return Ok(value),
Err(err) => {
let Some(reason) = classify(&err) else {
return Err(err);
};
let delay = backoff.delay_for_attempt(attempt_number);
if deadline.is_some_and(|deadline| {
delay >= deadline.saturating_duration_since(time::Instant::now())
}) {
return Err(err);
}
// The failure text can carry git stderr, so log the category
// rather than the message. The caller still reports the full
// error if the attempts run out.
tracing::warn!(
provider = %provider,
attempt = attempt_number,
max_attempts = MAX_ATTEMPTS,
reason = %reason,
delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
"Git clone failed, retrying"
);
time::sleep(delay).await;
}
}
}
attempt(MAX_ATTEMPTS).await
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
/// Records the attempt numbers a closure was called with.
#[derive(Default)]
struct Attempts(Mutex<Vec<u32>>);
impl Attempts {
fn record(&self, attempt: u32) {
self.0.lock().expect("attempt log mutex").push(attempt);
}
fn recorded(&self) -> Vec<u32> {
self.0.lock().expect("attempt log mutex").clone()
}
}
/// A classifier that treats every failure as worth repeating.
const ALWAYS_RETRY: fn(&String) -> Option<CloneRetryReason> =
|_| Some(CloneRetryReason::TokenReplication);
#[test]
fn private_repo_not_found_after_a_successful_mint_is_a_replication_lag() {
assert_eq!(
classify_message("repository not found: Repository not found.", true),
CloneMessageClass::Retry(CloneRetryReason::TokenReplication)
);
}
#[test]
fn not_found_without_a_fresh_token_is_permanent() {
assert_eq!(
classify_message("repository not found: Repository not found.", false),
CloneMessageClass::Permanent
);
}
#[test]
fn auth_failure_with_a_fresh_token_is_a_replication_lag() {
assert_eq!(
classify_message(
"fatal: Authentication failed for 'https://github.com/owner/repo'",
true
),
CloneMessageClass::Retry(CloneRetryReason::TokenReplication)
);
assert_eq!(
classify_message(
"fatal: Authentication failed for 'https://github.com/owner/repo'",
false
),
CloneMessageClass::Permanent
);
}
#[test]
fn infra_failures_retry_without_credentials() {
for message in [
"fatal: unable to access: Could not resolve host: github.com",
"error: RPC failed; curl 56 recv failure",
"fatal: early EOF",
"Operation timed out",
] {
assert_eq!(
classify_message(message, false),
CloneMessageClass::Retry(CloneRetryReason::TransientInfra),
"expected {message:?} to be transient"
);
}
}
#[test]
fn genuine_failures_are_not_retried() {
for message in [
"fatal: could not read Username for 'https://github.com'",
"remote: Permission to owner/repo.git denied",
"fatal: destination path 'repo' already exists",
] {
assert_eq!(
classify_message(message, true),
CloneMessageClass::Permanent,
"expected {message:?} to fail fast"
);
}
}
#[test]
fn unrecognized_failures_remain_unknown() {
assert_eq!(
classify_message("git clone stopped for an unexpected reason", true),
CloneMessageClass::Unknown
);
}
#[test]
fn backoff_waits_seconds_not_milliseconds() {
let backoff = backoff();
assert_eq!(backoff.delay_for_attempt(1), Duration::from_secs(3));
assert_eq!(backoff.delay_for_attempt(2), Duration::from_secs(9));
}
#[tokio::test(start_paused = true)]
async fn first_success_runs_one_attempt() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
None,
|attempt| {
attempts.record(attempt);
async move { Ok::<_, String>(attempt) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Ok(1));
assert_eq!(attempts.recorded(), vec![1]);
}
#[tokio::test(start_paused = true)]
async fn retries_until_a_later_attempt_succeeds() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
None,
|attempt| {
attempts.record(attempt);
async move {
if attempt < 3 {
Err("Repository not found.".to_string())
} else {
Ok(attempt)
}
}
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Ok(3));
assert_eq!(attempts.recorded(), vec![1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn exhausted_attempts_return_the_final_error() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
None,
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>(format!("Repository not found. (attempt {attempt})")) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(
result,
Err("Repository not found. (attempt 3)".to_string()),
"the caller should see the last failure, not the first"
);
assert_eq!(attempts.recorded(), vec![1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn unretryable_failure_stops_immediately() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
None,
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("permission denied".to_string()) }
},
|_: &String| None,
)
.await;
assert_eq!(result, Err("permission denied".to_string()));
assert_eq!(
attempts.recorded(),
vec![1],
"a deterministic failure should not wait out the backoff"
);
}
#[tokio::test(start_paused = true)]
async fn deadline_stops_retry_when_backoff_does_not_fit() {
let attempts = Attempts::default();
let deadline = time::Instant::now() + Duration::from_secs(2);
let result = retry_clone(
SandboxProviderKind::Docker,
Some(deadline),
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("temporary failure".to_string()) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Err("temporary failure".to_string()));
assert_eq!(attempts.recorded(), vec![1]);
assert_eq!(time::Instant::now() + Duration::from_secs(2), deadline);
}
}

872
lib/components/fabro-sandbox/src/daytona/mod.rs
View file

File diff suppressed because it is too large Load diff

187
lib/components/fabro-sandbox/src/docker.rs
View file

@ -17,6 +17,7 @@ use bollard::exec::{CreateExecOptions, StartExecOptions, StartExecResults};
use bollard::image::CreateImageOptions;
use bollard::models::{ContainerInspectResponse, HostConfig};
use fabro_github::GitHubCredentials;
use fabro_github::token_source::InstallationTokenSource;
use fabro_types::{CommandOutputStream, CommandTermination, RunId, SandboxProviderKind};
use fabro_util::time::elapsed_ms;
use futures::StreamExt;
@ -26,7 +27,9 @@ use tokio::{fs, time};
use tokio_util::sync::CancellationToken;
use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason};
use crate::git_retry::{self, CredentialContext};
use crate::managed_labels::{self, MANAGED_LABEL, RUN_ID_LABEL};
use crate::push_credentials::{self, PushCredentialState};
use crate::redact::redact_auth_url;
use crate::sandbox::{
self, BASH_ENV_VAR, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, REMOTE_BASH,
@ -37,7 +40,7 @@ use crate::{
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
ExecStreamingRequest, ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent,
SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess, StdioProcessHandle,
StdioProcessTermination, WalkOptions, clone_retry, format_lines_numbered, shell_quote,
StdioProcessTermination, WalkOptions, format_lines_numbered, shell_quote,
};
const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for every command, with no `sh` fallback; use an \
@ -64,6 +67,11 @@ enum CloneStep {
Local,
}
struct DockerCloneFailure {
error: crate::Error,
retry_reason: Option<git_retry::GitRetryReason>,
}
fn env_entry_name(entry: &str) -> &str {
entry.split_once('=').map_or(entry, |(name, _)| name)
}
@ -134,7 +142,7 @@ impl Default for DockerSandboxOptions {
pub struct DockerSandbox {
docker: Docker,
config: DockerSandboxOptions,
github_app: Option<GitHubCredentials>,
push_credentials: PushCredentialState,
run_id: Option<RunId>,
clone_origin_url: Option<String>,
clone_branch: Option<String>,
@ -165,7 +173,7 @@ enum ContainerStartAction {
impl DockerSandbox {
pub fn new(
config: DockerSandboxOptions,
github_app: Option<GitHubCredentials>,
github_app: Option<&GitHubCredentials>,
run_id: Option<RunId>,
clone_origin_url: Option<String>,
clone_branch: Option<String>,
@ -180,7 +188,7 @@ impl DockerSandbox {
)?;
}
let docker = Docker::connect_with_local_defaults().map_err(crate::Error::docker_connect)?;
Ok(Self::with_docker_client(
Self::with_docker_client(
docker,
config,
github_app,
@ -188,22 +196,26 @@ impl DockerSandbox {
clone_origin_url,
clone_branch,
clone_commit_sha,
))
)
}
fn with_docker_client(
docker: Docker,
config: DockerSandboxOptions,
github_app: Option<GitHubCredentials>,
github_app: Option<&GitHubCredentials>,
run_id: Option<RunId>,
clone_origin_url: Option<String>,
clone_branch: Option<String>,
clone_commit_sha: Option<String>,
) -> Self {
Self {
) -> crate::Result<Self> {
let push_credentials = PushCredentialState::new(push_credentials::build_token_source(
github_app,
clone_origin_url.as_deref(),
)?);
Ok(Self {
docker,
config,
github_app,
push_credentials,
run_id,
clone_origin_url,
clone_branch,
@ -216,7 +228,7 @@ impl DockerSandbox {
cached_os_version: std::sync::OnceLock::new(),
rg_available: OnceCell::const_new(),
event_callback: None,
}
})
}
pub async fn reconnect(
@ -736,7 +748,7 @@ impl DockerSandbox {
let error =
result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url));
let message = match step {
CloneStep::Network if self.github_app.is_none() => {
CloneStep::Network if self.push_credentials.source().is_none() => {
"Git clone failed. If this is a private repository, configure a GitHub App with \
`fabro install` and install it for your organization."
}
@ -791,20 +803,23 @@ impl DockerSandbox {
async fn retry_git_transfer(
&self,
command: &str,
op: &'static str,
label: &'static str,
exec_label: &'static str,
clone_deadline: time::Instant,
token_was_freshly_minted: bool,
credential_context: CredentialContext,
auth_url: Option<&fabro_redact::DisplaySafeUrl>,
) -> Result<(), clone_retry::CloneAttemptFailure> {
clone_retry::retry_clone(
) -> Result<(), DockerCloneFailure> {
let plan = git_retry::RetryPlan::clone_default(Some(clone_deadline));
git_retry::retry_clone(
SandboxProviderKind::Docker,
Some(clone_deadline),
op,
&plan,
|_attempt| async move {
let remaining = clone_deadline.saturating_duration_since(time::Instant::now());
let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX);
if timeout_ms == 0 {
return Err(clone_retry::CloneAttemptFailure {
return Err(DockerCloneFailure {
error: crate::Error::message(format!(
"{label} deadline expired before retry"
)),
@ -818,7 +833,7 @@ impl DockerSandbox {
..ExecStreamingRequest::new(command)
})
.await
.map_err(|error| clone_retry::CloneAttemptFailure {
.map_err(|error| DockerCloneFailure {
error: crate::Error::context(
format!("{label} transport failed"),
error,
@ -829,8 +844,8 @@ impl DockerSandbox {
if result.is_success() {
return Ok(());
}
let retry_reason = classify_docker_clone_result(&result, token_was_freshly_minted);
Err(clone_retry::CloneAttemptFailure {
let retry_reason = classify_docker_clone_result(&result, credential_context);
Err(DockerCloneFailure {
error: self.clone_failure_error(
result,
exec_label,
@ -840,7 +855,7 @@ impl DockerSandbox {
retry_reason,
})
},
|failure: &clone_retry::CloneAttemptFailure| failure.retry_reason,
|failure: &DockerCloneFailure| failure.retry_reason,
)
.await
}
@ -853,22 +868,31 @@ impl DockerSandbox {
) -> crate::Result<()> {
self.verify_git_available().await?;
let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT)?;
let token_was_freshly_minted = self
.github_app
.as_ref()
.is_some_and(GitHubCredentials::mints_installation_token);
// The clone mints its own token (never a warm-cache reuse) and seeds
// the shared source, so the first refresh compares against the clone
// token instead of believing nothing was ever embedded.
let resolved_token = match self.push_credentials.source() {
Some(source) => Some(source.mint_for_clone().await.map_err(|err| {
crate::Error::context_anyhow("Failed to get GitHub App credentials for clone", err)
})?),
None => None,
};
// The clone call site maps its mint knowledge onto the credential
// context: a token minted for this clone is FreshApp; a static
// credential cannot become valid by waiting.
let clone_credential_context =
CredentialContext::from_snapshot(resolved_token.as_ref().map(|token| &token.snapshot));
let auth_url = match &self.github_app {
Some(creds) => Some(
fabro_github::resolve_authenticated_url(
&fabro_github::GitHubContext::new(creds, &fabro_github::github_api_base_url()),
&origin_url,
)
.await
.map_err(|error| crate::Error::Context {
message: "Failed to get GitHub App credentials for clone".to_string(),
source: error.into_boxed_dyn_error(),
})?,
let auth_url = match &resolved_token {
Some(token) => Some(
fabro_github::embed_token_in_url(&origin_url, token.token.expose()).map_err(
|err| {
crate::Error::context_anyhow(
"Failed to build authenticated GitHub clone URL",
err,
)
},
)?,
),
None => None,
};
@ -935,10 +959,11 @@ impl DockerSandbox {
if let Err(failure) = self
.retry_git_transfer(
&fetch_command,
"fetch",
"Docker exact fetch",
"git fetch exact commit",
clone_deadline,
token_was_freshly_minted,
clone_credential_context,
auth_url.as_ref(),
)
.await
@ -972,10 +997,11 @@ impl DockerSandbox {
if let Err(failure) = self
.retry_git_transfer(
&command,
"clone",
"Docker git clone",
"git clone",
clone_deadline,
token_was_freshly_minted,
clone_credential_context,
auth_url.as_ref(),
)
.await
@ -1002,6 +1028,11 @@ impl DockerSandbox {
let _ = self.repo_cloned.set(true);
let _ = self.origin_url.set(origin_url.clone());
self.set_working_directory(layout.execution_directory.clone())?;
if let Some(token) = resolved_token {
// The clone URL embedded this token in `origin`; record it so
// refreshes compare against the clone generation.
self.push_credentials.record_embedded(token).await;
}
if let Some(auth_url) = auth_url.as_ref() {
let command = format!(
@ -1517,15 +1548,9 @@ fn git_clone_command(clone_url: &str, branch: Option<&str>, checkout_path: &str)
fn classify_docker_clone_result(
result: &ExecResult,
token_was_freshly_minted: bool,
) -> Option<clone_retry::CloneRetryReason> {
let stderr = clone_retry::classify_message(&result.stderr, token_was_freshly_minted);
match stderr {
clone_retry::CloneMessageClass::Unknown => {
clone_retry::classify_message(&result.stdout, token_was_freshly_minted).retry_reason()
}
class => class.retry_reason(),
}
cred: CredentialContext,
) -> Option<git_retry::GitRetryReason> {
git_retry::classify_output(&result.stderr, &result.stdout, cred).retry_reason()
}
fn host_config(config: &DockerSandboxOptions) -> HostConfig {
@ -2311,11 +2336,19 @@ impl Sandbox for DockerSandbox {
)]
}
async fn git_push_ref(&self, refspec: &str) -> crate::Result<()> {
async fn git_push_ref(
&self,
refspec: &str,
plan: &crate::RetryPlan,
) -> Result<crate::PushReport, crate::PushError> {
if !self.repo_cloned() {
return Ok(());
return Ok(crate::PushReport::default());
}
crate::git_push_via_exec(self, refspec).await
let credentials = self
.origin_url
.get()
.map(|origin_url| (&self.push_credentials, origin_url.as_str()));
sandbox::git_push_via_exec(self, credentials, refspec, plan).await
}
fn origin_url(&self) -> Option<&str> {
@ -2325,49 +2358,23 @@ impl Sandbox for DockerSandbox {
self.origin_url.get().map(String::as_str)
}
#[tracing::instrument(name = "git_op", skip_all, fields(op = "refresh-credentials"))]
async fn refresh_push_credentials(&self) -> crate::Result<RefreshOutcome> {
if !self.repo_cloned() {
return Ok(RefreshOutcome::Skipped);
return Ok(RefreshOutcome::none());
}
let Some(origin_url) = self.origin_url.get() else {
return Ok(RefreshOutcome::Skipped);
return Ok(RefreshOutcome::none());
};
let Some(creds) = &self.github_app else {
return Ok(RefreshOutcome::Skipped);
};
// Only a GitHub App installation token can be re-minted; a static PAT or
// a pre-minted Installation token is fixed, so re-embedding it changes
// nothing. Short-circuit to Skipped before the resolve + set-url exec.
if !creds.mints_installation_token() {
return Ok(RefreshOutcome::Skipped);
}
self.push_credentials
.refresh(origin_url, |auth_url| {
push_credentials::set_auth_url_via_exec(self, auth_url)
})
.await
}
let auth_url = fabro_github::resolve_authenticated_url(
&fabro_github::GitHubContext::new(creds, &fabro_github::github_api_base_url()),
origin_url,
)
.await
.map_err(|_| {
crate::Error::message("Failed to refresh push credentials: token_mint_failed")
})?;
let command = format!(
"git -c maintenance.auto=0 remote set-url origin {}",
shell_quote(auth_url.as_raw_url().as_str())
);
let result = self
.docker_exec_shell(&command, 10_000, Some(self.working_directory()), None, None)
.await?;
if !result.is_success() {
return Err(result.into_exec_error_with_redactor(
"git remote set-url origin (refresh push credentials)",
|s| redact_auth_url(s, Some(&auth_url)),
));
}
// Static creds were short-circuited to Skipped above; reaching here means
// a GitHub App installation token was freshly minted.
Ok(RefreshOutcome::Refreshed)
fn push_token_source(&self) -> Option<Arc<InstallationTokenSource>> {
self.push_credentials.source().cloned()
}
}
@ -2690,7 +2697,10 @@ mod tests {
duration_ms: 1,
};
assert_eq!(classify_docker_clone_result(&result, true), None);
assert_eq!(
classify_docker_clone_result(&result, CredentialContext::FreshApp),
None
);
}
#[test]
@ -2977,7 +2987,8 @@ mod tests {
None,
None,
None,
);
)
.expect("test sandbox should build");
sandbox
.container_id
.set(container_id.to_string())

14
lib/components/fabro-sandbox/src/error.rs
View file

@ -18,6 +18,13 @@ pub enum Error {
source: Box<dyn std::error::Error + Send + Sync + 'static>,
},
#[error("{message}")]
AnyhowContext {
message: String,
#[source]
source: anyhow::Error,
},
#[cfg(feature = "docker")]
#[error("Failed to connect to Docker daemon")]
DockerConnect {
@ -68,6 +75,13 @@ impl Error {
}
}
pub fn context_anyhow(message: impl Into<String>, source: anyhow::Error) -> Self {
Self::AnyhowContext {
message: message.into(),
source,
}
}
pub fn exec(label: impl Into<String>, result: ExecResult) -> Self {
Self::Exec {
label: label.into(),

762
lib/components/fabro-sandbox/src/git_retry.rs Normal file
View file

@ -0,0 +1,762 @@
//! Retry for git operations against GitHub from clone-based sandboxes.
//!
//! Clone-based providers can mint a GitHub App installation token and use it
//! immediately. GitHub can reject that first operation before the token is
//! available to the git endpoint. On a private repository, the rejection can
//! arrive as `Repository not found.` or an authentication failure.
//!
//! Only a token minted recently makes those messages safe to retry. Static
//! PATs and pre-minted installation tokens fail fast; a mature App token can
//! still hit a service-side blip that presents the same surface, so it
//! retries as transient infrastructure.
//!
//! Retries reuse the same token on purpose. Replication of a given token only
//! makes progress, so each attempt strictly improves the odds, while
//! re-minting would restart the replication clock.
use std::future::Future;
use std::time::Duration;
use chrono::Utc;
use fabro_github::token_source::TokenSnapshot;
#[cfg(test)]
use fabro_github::token_source::{REFRESH_MARGIN, TokenProvenance};
use fabro_types::SandboxProviderKind;
pub use fabro_types::run_event::GitPushRetryReason as GitRetryReason;
use fabro_util::backoff::BackoffPolicy;
use tokio::time;
/// How long after its mint a token is presumed to still be replicating to
/// GitHub's git endpoints. Matches the observed scale of the lag (seconds,
/// occasionally tens of seconds).
pub(crate) const REPLICATION_HORIZON: Duration = Duration::from_mins(1);
/// What a git failure message tells us about retry safety.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum GitMessageClass {
Retry(GitRetryReason),
Permanent,
Unknown,
}
impl GitMessageClass {
pub(crate) fn retry_reason(self) -> Option<GitRetryReason> {
match self {
Self::Retry(reason) => Some(reason),
Self::Permanent | Self::Unknown => None,
}
}
}
/// What the operation's credentials say about retrying auth-shaped failures.
///
/// Derived from the [`TokenSnapshot`] of the token embedded for the attempt,
/// so classification reads provenance as data instead of threading booleans
/// through call stacks.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CredentialContext {
/// An installation token younger than [`REPLICATION_HORIZON`] — a 404 or
/// auth failure is likely replication lag; retry with the same token.
FreshApp,
/// An installation token older than the horizon. A 404 with it is
/// indistinguishable from a service-side blip at this layer, so it stays
/// transient rather than proving access loss.
MatureApp,
/// A PAT or pre-minted token — it cannot become valid by waiting.
Static,
/// No credentials at all.
None,
}
impl CredentialContext {
#[must_use]
pub fn from_snapshot(snapshot: Option<&TokenSnapshot>) -> Self {
match snapshot {
None => Self::None,
Some(snapshot) => match snapshot.age_at(Utc::now()) {
None => Self::Static,
Some(age) if age < REPLICATION_HORIZON => Self::FreshApp,
Some(_) => Self::MatureApp,
},
}
}
}
/// Message fragments that mean the operation failed on infrastructure.
///
/// These are safe to retry whether or not the operation was authenticated.
const TRANSIENT_HINTS: &[&str] = &[
"could not resolve host",
"temporary failure in name resolution",
"connection refused",
"connection reset",
"connection timed out",
"timed out",
"network is unreachable",
"no route to host",
"tls handshake",
"early eof",
"rpc failed",
"unexpected disconnect",
"the remote end hung up unexpectedly",
"index-pack failed",
"service unavailable",
"gateway timeout",
"too many requests",
"rate limit",
];
/// Message fragments GitHub uses when a token is not yet visible.
///
/// Only meaningful when the operation carried credentials. The same lag
/// surfaces as 404 or as an auth failure depending on which endpoint answers
/// first.
const TOKEN_REPLICATION_HINTS: &[&str] = &[
"repository not found",
"authentication failed",
"invalid username or password",
"bad credentials",
// git CLI over HTTP.
"the requested url returned error: 401",
"the requested url returned error: 403",
"the requested url returned error: 404",
// libgit2 (the run-metadata writer pushes through git2).
"unexpected http status code: 401",
"unexpected http status code: 403",
"unexpected http status code: 404",
];
/// Whether a failure message has the 404/auth-failure shape GitHub produces
/// for both token-replication lag and a drifted or missing embedded token.
pub(crate) fn matches_auth_failure_hints(message: &str) -> bool {
let lower = message.to_ascii_lowercase();
TOKEN_REPLICATION_HINTS
.iter()
.any(|hint| lower.contains(hint))
}
pub(crate) fn output_matches_auth_failure_hints(stderr: &str, stdout: &str) -> bool {
matches_auth_failure_hints(stderr) || matches_auth_failure_hints(stdout)
}
/// Classify a failed git operation by its rendered message.
///
/// `cred` gates the reading of 404/auth-failure messages: a fresh App token
/// retries as replication lag, a mature one as transient infrastructure, and
/// a static credential (or none) fails fast because waiting cannot make it
/// valid.
pub(crate) fn classify_message(message: &str, cred: CredentialContext) -> GitMessageClass {
let lower = message.to_ascii_lowercase();
if TRANSIENT_HINTS.iter().any(|hint| lower.contains(hint)) {
return GitMessageClass::Retry(GitRetryReason::TransientInfra);
}
if TOKEN_REPLICATION_HINTS
.iter()
.any(|hint| lower.contains(hint))
{
return match cred {
CredentialContext::FreshApp => GitMessageClass::Retry(GitRetryReason::TokenReplication),
CredentialContext::MatureApp => GitMessageClass::Retry(GitRetryReason::TransientInfra),
CredentialContext::Static | CredentialContext::None => GitMessageClass::Permanent,
};
}
let permanent = lower.contains("could not read username")
|| lower.contains("terminal prompts disabled")
|| lower.contains("permission denied")
|| (lower.contains("permission to") && lower.contains("denied"))
|| (lower.contains("destination path") && lower.contains("already exists"))
|| (lower.contains("remote branch") && lower.contains("not found"));
if permanent {
return GitMessageClass::Permanent;
}
GitMessageClass::Unknown
}
pub(crate) fn classify_output(
stderr: &str,
stdout: &str,
cred: CredentialContext,
) -> GitMessageClass {
let by_stderr = classify_message(stderr, cred);
if by_stderr == GitMessageClass::Unknown {
classify_message(stdout, cred)
} else {
by_stderr
}
}
/// Classify a rendered git failure message, returning the retry reason when
/// the failure is transient for these credentials. `None` means the failure
/// is permanent or unrecognized.
#[must_use]
pub fn classify_failure(message: &str, cred: CredentialContext) -> Option<GitRetryReason> {
classify_message(message, cred).retry_reason()
}
/// Backoff between attempts: 3s, then 9s.
///
/// GitHub's guidance for token replication is to wait a few seconds and retry
/// with the same token. Sub-second delays land inside the same replication
/// window and spend an attempt for nothing.
fn clone_backoff() -> BackoffPolicy {
BackoffPolicy {
initial_delay: Duration::from_secs(3),
factor: 3.0,
max_delay: Duration::from_secs(10),
jitter: false,
}
}
/// Attempt and time bounds for one retried git operation.
///
/// All bounds are optional so existing behaviors are expressible unchanged.
/// The effective deadline is the minimum of the bounds that are present
/// (`start + max_elapsed`, `outer_deadline`); each attempt runs with
/// `min(per_attempt_timeout, remaining)` over the caps that are present, and
/// no attempt or backoff starts past the effective deadline.
#[derive(Debug, Clone)]
pub struct RetryPlan {
/// Total attempts, including the first.
pub max_attempts: u32,
pub backoff: BackoffPolicy,
/// Wall clock for this whole operation.
pub max_elapsed: Option<Duration>,
/// Cap for any single attempt.
pub per_attempt_timeout: Option<Duration>,
/// Caller-supplied absolute bound.
pub outer_deadline: Option<time::Instant>,
}
impl RetryPlan {
/// The clone policy both providers already trust: 3 attempts, 3s/9s
/// backoff, no plan-level bounds. Docker supplies its existing absolute
/// five-minute deadline through `outer_deadline`; Daytona supplies none.
#[must_use]
pub fn clone_default(outer_deadline: Option<time::Instant>) -> Self {
Self {
max_attempts: 3,
backoff: clone_backoff(),
max_elapsed: None,
per_attempt_timeout: None,
outer_deadline,
}
}
/// Checkpoint pushes stay cheap: the next checkpoint re-pushes the same
/// branch anyway. Worst case ~90 seconds of wall clock.
#[must_use]
pub fn checkpoint_push() -> Self {
Self {
max_attempts: 3,
backoff: clone_backoff(),
max_elapsed: Some(Duration::from_secs(90)),
per_attempt_timeout: Some(Duration::from_mins(1)),
outer_deadline: None,
}
}
/// The terminal publish push guards the whole run's value, so it gets a
/// real budget: 5 attempts with growing backoff (~3s/10s/33s/60s),
/// bounded at 4 minutes of wall clock. The 4-minute bound must stay
/// under the token source's `REFRESH_MARGIN` (see the margin-invariant
/// test) so a pinned token always outlives the operation.
#[must_use]
pub fn publish_push() -> Self {
Self {
max_attempts: 5,
backoff: BackoffPolicy {
initial_delay: Duration::from_secs(3),
factor: 10.0 / 3.0,
max_delay: Duration::from_mins(1),
jitter: false,
},
max_elapsed: Some(Duration::from_mins(4)),
per_attempt_timeout: Some(Duration::from_mins(1)),
outer_deadline: None,
}
}
/// The absolute deadline this operation must finish by, if any bound is
/// present.
pub(crate) fn effective_deadline(&self, start: time::Instant) -> Option<time::Instant> {
let elapsed_deadline = self.max_elapsed.map(|max| start + max);
match (elapsed_deadline, self.outer_deadline) {
(Some(a), Some(b)) => Some(a.min(b)),
(Some(a), None) => Some(a),
(None, Some(b)) => Some(b),
(None, None) => None,
}
}
/// Time cap for an attempt starting now: the per-attempt cap bounded by
/// the time remaining before the effective deadline.
pub(crate) fn attempt_timeout(&self, deadline: Option<time::Instant>) -> Option<Duration> {
let remaining = deadline.map(|d| d.saturating_duration_since(time::Instant::now()));
match (self.per_attempt_timeout, remaining) {
(Some(cap), Some(remaining)) => Some(cap.min(remaining)),
(Some(cap), None) => Some(cap),
(None, remaining) => remaining,
}
}
pub(crate) fn retry_delay(
&self,
attempt_number: u32,
deadline: Option<time::Instant>,
) -> Option<Duration> {
let delay = self.backoff.delay_for_attempt(attempt_number);
if deadline.is_some_and(|deadline| {
delay >= deadline.saturating_duration_since(time::Instant::now())
}) {
None
} else {
Some(delay)
}
}
}
/// Run a clone operation, repeating it while the failure looks transient.
///
/// `attempt` receives the 1-based attempt number. `classify` decides whether
/// an error is worth repeating; `None` returns it to the caller untouched.
/// A retry starts only when its backoff fits before the plan's effective
/// deadline. The final error is returned as-is.
pub(crate) async fn retry_clone<T, E, Attempt, Fut, Classify>(
provider: SandboxProviderKind,
op: &str,
plan: &RetryPlan,
mut attempt: Attempt,
classify: Classify,
) -> Result<T, E>
where
Attempt: FnMut(u32) -> Fut,
Fut: Future<Output = Result<T, E>>,
Classify: Fn(&E) -> Option<GitRetryReason>,
{
let deadline = plan.effective_deadline(time::Instant::now());
for attempt_number in 1..plan.max_attempts.max(1) {
match attempt(attempt_number).await {
Ok(value) => return Ok(value),
Err(err) => {
let Some(reason) = classify(&err) else {
return Err(err);
};
let Some(delay) = plan.retry_delay(attempt_number, deadline) else {
return Err(err);
};
// The failure text can carry git stderr, so log the category
// rather than the message. The caller still reports the full
// error if the attempts run out.
tracing::warn!(
provider = %provider,
op,
attempt = attempt_number,
max_attempts = plan.max_attempts,
reason = %reason,
delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
"Git operation failed, retrying"
);
time::sleep(delay).await;
}
}
}
attempt(plan.max_attempts.max(1)).await
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
/// Records the attempt numbers a closure was called with.
#[derive(Default)]
struct Attempts(Mutex<Vec<u32>>);
impl Attempts {
fn record(&self, attempt: u32) {
self.0.lock().expect("attempt log mutex").push(attempt);
}
fn recorded(&self) -> Vec<u32> {
self.0.lock().expect("attempt log mutex").clone()
}
}
/// A classifier that treats every failure as worth repeating.
const ALWAYS_RETRY: fn(&String) -> Option<GitRetryReason> =
|_| Some(GitRetryReason::TokenReplication);
fn fresh_snapshot(age: Duration, ttl: Duration) -> TokenSnapshot {
let now = Utc::now();
TokenSnapshot {
generation: 1,
provenance: TokenProvenance::Minted {
minted_at: now - chrono::Duration::from_std(age).unwrap(),
expires_at: now + chrono::Duration::from_std(ttl).unwrap(),
},
}
}
#[test]
fn credential_context_reads_token_age_from_provenance() {
assert_eq!(
CredentialContext::from_snapshot(None),
CredentialContext::None
);
assert_eq!(
CredentialContext::from_snapshot(Some(&TokenSnapshot {
generation: 0,
provenance: TokenProvenance::Static,
})),
CredentialContext::Static
);
assert_eq!(
CredentialContext::from_snapshot(Some(&fresh_snapshot(
Duration::from_secs(5),
Duration::from_hours(1)
))),
CredentialContext::FreshApp
);
assert_eq!(
CredentialContext::from_snapshot(Some(&fresh_snapshot(
Duration::from_mins(2),
Duration::from_hours(1)
))),
CredentialContext::MatureApp
);
}
#[test]
fn private_repo_not_found_with_a_fresh_token_is_a_replication_lag() {
assert_eq!(
classify_message(
"repository not found: Repository not found.",
CredentialContext::FreshApp
),
GitMessageClass::Retry(GitRetryReason::TokenReplication)
);
}
#[test]
fn not_found_with_a_mature_token_is_transient_not_permanent() {
// A service-side blip is indistinguishable from access loss at this
// layer, so a mature-App 404 stays retryable.
assert_eq!(
classify_message(
"repository not found: Repository not found.",
CredentialContext::MatureApp
),
GitMessageClass::Retry(GitRetryReason::TransientInfra)
);
}
#[test]
fn not_found_with_static_or_no_credentials_is_permanent() {
for cred in [CredentialContext::Static, CredentialContext::None] {
assert_eq!(
classify_message("repository not found: Repository not found.", cred),
GitMessageClass::Permanent,
"{cred:?} cannot become valid by waiting"
);
}
}
#[test]
fn auth_failure_classification_follows_the_credential_context() {
let message = "fatal: Authentication failed for 'https://github.com/owner/repo'";
assert_eq!(
classify_message(message, CredentialContext::FreshApp),
GitMessageClass::Retry(GitRetryReason::TokenReplication)
);
assert_eq!(
classify_message(message, CredentialContext::MatureApp),
GitMessageClass::Retry(GitRetryReason::TransientInfra)
);
assert_eq!(
classify_message(message, CredentialContext::Static),
GitMessageClass::Permanent
);
}
#[test]
fn infra_failures_retry_without_credentials() {
for message in [
"fatal: unable to access: Could not resolve host: github.com",
"error: RPC failed; curl 56 recv failure",
"fatal: early EOF",
"Operation timed out",
] {
assert_eq!(
classify_message(message, CredentialContext::None),
GitMessageClass::Retry(GitRetryReason::TransientInfra),
"expected {message:?} to be transient"
);
}
}
#[test]
fn genuine_failures_are_not_retried() {
for message in [
"fatal: could not read Username for 'https://github.com'",
"remote: Permission to owner/repo.git denied",
"fatal: destination path 'repo' already exists",
] {
assert_eq!(
classify_message(message, CredentialContext::FreshApp),
GitMessageClass::Permanent,
"expected {message:?} to fail fast"
);
}
}
#[test]
fn unrecognized_failures_remain_unknown() {
assert_eq!(
classify_message(
"git operation stopped for an unexpected reason",
CredentialContext::FreshApp
),
GitMessageClass::Unknown
);
}
#[test]
fn backoff_waits_seconds_not_milliseconds() {
let plan = RetryPlan::clone_default(None);
assert_eq!(plan.backoff.delay_for_attempt(1), Duration::from_secs(3));
assert_eq!(plan.backoff.delay_for_attempt(2), Duration::from_secs(9));
}
#[test]
fn publish_backoff_grows_toward_a_one_minute_cap() {
let plan = RetryPlan::publish_push();
assert_eq!(plan.backoff.delay_for_attempt(1), Duration::from_secs(3));
assert_eq!(plan.backoff.delay_for_attempt(2), Duration::from_secs(10));
assert!(plan.backoff.delay_for_attempt(3) < Duration::from_secs(35));
assert_eq!(plan.backoff.delay_for_attempt(4), Duration::from_mins(1));
}
/// `REFRESH_MARGIN` must exceed every push plan's `max_elapsed`: a push
/// pins the token of its single successful resolve, and any token the
/// source returns has at least the margin of validity left, so the pinned
/// token must outlive the whole operation.
#[test]
fn refresh_margin_exceeds_every_push_plan_elapsed_bound() {
for plan in [RetryPlan::checkpoint_push(), RetryPlan::publish_push()] {
let max_elapsed = plan.max_elapsed.expect("push plans bound elapsed time");
assert!(
REFRESH_MARGIN > max_elapsed,
"margin invariant violated: {max_elapsed:?}"
);
}
}
#[test]
fn effective_deadline_takes_the_minimum_of_present_bounds() {
let start = time::Instant::now();
let outer = start + Duration::from_secs(30);
let unbounded = RetryPlan::clone_default(None);
assert_eq!(unbounded.effective_deadline(start), None);
let outer_only = RetryPlan::clone_default(Some(outer));
assert_eq!(outer_only.effective_deadline(start), Some(outer));
let mut both = RetryPlan::checkpoint_push();
both.outer_deadline = Some(outer);
assert_eq!(both.effective_deadline(start), Some(outer));
both.outer_deadline = Some(start + Duration::from_mins(10));
assert_eq!(
both.effective_deadline(start),
Some(start + Duration::from_secs(90))
);
}
#[tokio::test(start_paused = true)]
async fn attempt_timeout_is_capped_by_the_remaining_deadline() {
let plan = RetryPlan::checkpoint_push();
let deadline = Some(time::Instant::now() + Duration::from_secs(20));
assert_eq!(
plan.attempt_timeout(deadline),
Some(Duration::from_secs(20))
);
assert_eq!(plan.attempt_timeout(None), Some(Duration::from_mins(1)));
let unbounded = RetryPlan::clone_default(None);
assert_eq!(unbounded.attempt_timeout(None), None);
}
#[tokio::test(start_paused = true)]
async fn first_success_runs_one_attempt() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
"clone",
&RetryPlan::clone_default(None),
|attempt| {
attempts.record(attempt);
async move { Ok::<_, String>(attempt) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Ok(1));
assert_eq!(attempts.recorded(), vec![1]);
}
#[tokio::test(start_paused = true)]
async fn retries_until_a_later_attempt_succeeds() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
"clone",
&RetryPlan::clone_default(None),
|attempt| {
attempts.record(attempt);
async move {
if attempt < 3 {
Err("Repository not found.".to_string())
} else {
Ok(attempt)
}
}
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Ok(3));
assert_eq!(attempts.recorded(), vec![1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn exhausted_attempts_return_the_final_error() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
"clone",
&RetryPlan::clone_default(None),
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>(format!("Repository not found. (attempt {attempt})")) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(
result,
Err("Repository not found. (attempt 3)".to_string()),
"the caller should see the last failure, not the first"
);
assert_eq!(attempts.recorded(), vec![1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn unretryable_failure_stops_immediately() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Docker,
"clone",
&RetryPlan::clone_default(None),
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("permission denied".to_string()) }
},
|_: &String| None,
)
.await;
assert_eq!(result, Err("permission denied".to_string()));
assert_eq!(
attempts.recorded(),
vec![1],
"a deterministic failure should not wait out the backoff"
);
}
/// Docker clone parity: the caller's absolute deadline stops retries when
/// the backoff no longer fits before it.
#[tokio::test(start_paused = true)]
async fn outer_deadline_stops_retry_when_backoff_does_not_fit() {
let attempts = Attempts::default();
let deadline = time::Instant::now() + Duration::from_secs(2);
let result = retry_clone(
SandboxProviderKind::Docker,
"clone",
&RetryPlan::clone_default(Some(deadline)),
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("temporary failure".to_string()) }
},
ALWAYS_RETRY,
)
.await;
assert_eq!(result, Err("temporary failure".to_string()));
assert_eq!(attempts.recorded(), vec![1]);
assert_eq!(time::Instant::now() + Duration::from_secs(2), deadline);
}
/// Daytona clone parity: with no bounds at all, attempts are limited only
/// by `max_attempts` and backoff.
#[tokio::test(start_paused = true)]
async fn unbounded_plan_runs_all_attempts() {
let attempts = Attempts::default();
let result = retry_clone(
SandboxProviderKind::Daytona,
"clone",
&RetryPlan::clone_default(None),
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("temporary failure".to_string()) }
},
|_: &String| Some(GitRetryReason::TransientInfra),
)
.await;
assert!(result.is_err());
assert_eq!(attempts.recorded(), vec![1, 2, 3]);
}
#[tokio::test(start_paused = true)]
async fn max_elapsed_stops_retry_when_backoff_does_not_fit() {
let attempts = Attempts::default();
let plan = RetryPlan {
max_attempts: 5,
backoff: clone_backoff(),
max_elapsed: Some(Duration::from_secs(4)),
per_attempt_timeout: None,
outer_deadline: None,
};
let result = retry_clone(
SandboxProviderKind::Docker,
"push",
&plan,
|attempt| {
attempts.record(attempt);
async move { Err::<(), _>("temporary failure".to_string()) }
},
ALWAYS_RETRY,
)
.await;
assert!(result.is_err());
// Attempt 1 fails instantly, 3s backoff fits inside 4s, attempt 2
// fails, and the 9s backoff no longer fits.
assert_eq!(attempts.recorded(), vec![1, 2]);
}
}

18
lib/components/fabro-sandbox/src/lib.rs
View file

@ -9,13 +9,13 @@ pub mod sandbox_spec;
#[cfg(any(feature = "docker", feature = "daytona"))]
mod clone_source;
#[cfg(any(feature = "docker", feature = "daytona", test))]
mod clone_retry;
mod git_retry;
#[cfg(any(feature = "docker", feature = "daytona", test))]
mod managed_labels;
#[cfg(any(feature = "docker", feature = "daytona", test))]
mod push_credentials;
pub mod redact;
pub mod details;
@ -39,7 +39,11 @@ pub use details::sandbox_details;
#[cfg(feature = "docker")]
pub use docker::{DockerSandbox, DockerSandboxOptions};
pub use error::{Error, Result, default_redacted_output_tail, display_for_log};
pub use fabro_github::token_source::{
InstallationTokenSource, ResolvedToken, TokenProvenance, TokenSnapshot,
};
pub use fabro_types::{RunSandboxInstance, SandboxProviderKind};
pub use git_retry::{CredentialContext, GitRetryReason, RetryPlan, classify_failure};
pub use local::LocalSandbox;
#[cfg(feature = "daytona")]
pub use provider::daytona::DaytonaSandboxProvider;
@ -49,13 +53,15 @@ pub use provider::{
LocalSandboxProvider, SandboxCreateSpec, SandboxLookupError, SandboxProvider,
SandboxProviderRegistry,
};
pub use push_credentials::RefreshErrorKind;
pub use reconnect::{reconnect, reconnect_for_run, reconnect_for_run_with_callback};
pub use sandbox::{
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
ExecStreamingRequest, ExecStreamingResult, GitRunInfo, GitSetupIntent, GrepOptions,
RefreshOutcome, Sandbox, SandboxEvent, SandboxEventCallback, SandboxFile, StderrCollector,
StdioProcess, StdioProcessHandle, StdioProcessTermination, WalkOptions, format_lines_numbered,
git_push_via_exec, redacted_output_tail, setup_git_via_exec, shell_quote,
PushAttempt, PushError, PushReport, RefreshOutcome, RemoteCredentialAction, Sandbox,
SandboxEvent, SandboxEventCallback, SandboxFile, StderrCollector, StdioProcess,
StdioProcessHandle, StdioProcessTermination, WalkOptions, format_lines_numbered,
redacted_output_tail, setup_git_via_exec, shell_quote,
};
pub use sandbox_spec::SandboxSpec;
pub use terminal::{TerminalSession, TerminalSize, open_terminal_for_run};

23
lib/components/fabro-sandbox/src/local.rs
View file

@ -13,8 +13,8 @@ use tokio::{fs, time};
use tokio_util::sync::CancellationToken;
use crate::sandbox::{
BASH_ENV_VAR, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, StdioProcessControl, optional_timeout,
validate_bash_probe, write_process_stdin,
self, BASH_ENV_VAR, BASH_PROBE_SCRIPT, BASH_PROBE_TIMEOUT_MS, StdioProcessControl,
optional_timeout, validate_bash_probe, write_process_stdin,
};
use crate::{
CommandOutputCallback, DEFAULT_EXEC_OUTPUT_TAIL_BYTES, DirEntry, ExecResult,
@ -878,20 +878,31 @@ impl Sandbox for LocalSandbox {
Ok(())
}
async fn git_push_ref(&self, refspec: &str) -> crate::Result<()> {
async fn git_push_ref(
&self,
refspec: &str,
plan: &crate::RetryPlan,
) -> Result<crate::PushReport, crate::PushError> {
let has_origin = match self
.exec_command("git remote get-url origin", 10_000, None, None, None)
.await
{
Ok(result) if result.is_success() => true,
Ok(_) => false,
Err(err) => return Err(crate::Error::context("git remote get-url origin", err)),
Err(err) => {
return Err(crate::PushError {
report: crate::PushReport::default(),
error: crate::Error::context("git remote get-url origin", err),
});
}
};
if !has_origin {
return Ok(());
return Ok(crate::PushReport::default());
}
crate::git_push_via_exec(self, refspec).await
// Local pushes use whatever credentials the host repository already
// carries; there is no managed credential state to lease.
sandbox::git_push_via_exec(self, None, refspec, plan).await
}
async fn cleanup(&self) -> crate::Result<()> {

2
lib/components/fabro-sandbox/src/provider/docker.rs
View file

@ -100,7 +100,7 @@ impl SandboxProvider for DockerSandboxProvider {
let sandbox = DockerSandbox::new(
config,
github_app,
github_app.as_ref(),
run_id,
clone_origin_url,
clone_branch,

629
lib/components/fabro-sandbox/src/push_credentials.rs Normal file
View file

@ -0,0 +1,629 @@
//! Shared push-credential state for clone-based sandbox providers.
//!
//! Docker and Daytona embed GitHub credentials into the cloned repository's
//! `origin` remote and refresh them before pushes. Both providers hold this
//! state so the compare → `set-url` → record sequence, the generation
//! tracking, and the refresh-error logging behave identically across
//! providers. The token cache itself sits below the providers, in
//! [`fabro_github::token_source::InstallationTokenSource`].
use std::future::Future;
use std::sync::Arc;
use fabro_github::GitHubCredentials;
use fabro_github::token_source::{InstallationTokenSource, ResolvedToken, TokenSnapshot};
use fabro_redact::DisplaySafeUrl;
pub use fabro_types::run_event::GitCredentialRefreshError as RefreshErrorKind;
use tokio::sync::{Mutex, MutexGuard};
use crate::redact;
use crate::sandbox::{RefreshOutcome, RemoteCredentialAction};
/// Build the shared installation-token source for a clone-based sandbox.
///
/// Returns `None` when there are no managed credentials or no GitHub origin
/// to scope them to. Minted tokens carry the same `contents: write`
/// permission the clone token uses.
pub(crate) fn build_token_source(
github_app: Option<&GitHubCredentials>,
clone_origin_url: Option<&str>,
) -> crate::Result<Option<Arc<InstallationTokenSource>>> {
let Some(creds) = github_app else {
return Ok(None);
};
let Some(origin_url) = clone_origin_url.filter(|url| !url.trim().is_empty()) else {
return Ok(None);
};
let normalized = fabro_github::normalize_repo_origin_url(origin_url);
let Ok((owner, repo)) = fabro_github::parse_github_owner_repo(&normalized) else {
// Non-GitHub origins never clone in these providers, so there is no
// remote to keep credentials fresh for.
return Ok(None);
};
InstallationTokenSource::for_repository(
creds,
owner,
repo,
serde_json::json!({ "contents": "write" }),
)
.map(Some)
.map_err(|err| crate::Error::context_anyhow("Failed to build GitHub token source", err))
}
/// Push-credential state one provider instance tracks for its `origin`
/// remote.
pub(crate) struct PushCredentialState {
source: Option<Arc<InstallationTokenSource>>,
/// Serializes compare → `set-url` → record. The token source's
/// single-flight ends before the sandbox exec, so without this lock a
/// refresh-ahead tick and a push could both see the old embedded
/// generation and race on `.git/config.lock`. Holds the last
/// successfully embedded token: its secret is already in the remote URL
/// inside the sandbox, so retaining it adds no exposure, and it is what
/// a push falls back to when a refresh fails. The tracked value is local
/// belief, not ground truth — agent code inside the sandbox can rewrite
/// `origin`.
embedded: Mutex<Option<ResolvedToken>>,
}
impl PushCredentialState {
pub(crate) fn new(source: Option<Arc<InstallationTokenSource>>) -> Self {
Self {
source,
embedded: Mutex::new(None),
}
}
pub(crate) fn source(&self) -> Option<&Arc<InstallationTokenSource>> {
self.source.as_ref()
}
/// Record the token embedded in `origin` outside the refresh path — the
/// clone is the first operation to embed a token, and it seeds this
/// state so the first refresh compares against the clone token instead
/// of believing nothing was ever embedded.
pub(crate) async fn record_embedded(&self, token: ResolvedToken) {
*self.embedded.lock().await = Some(token);
}
/// Refresh the credentials embedded in `origin`.
///
/// Resolves through the shared source, skips the `set-url` exec when the
/// resolved generation is already embedded, and records the new
/// generation only after `set_url` succeeds. `set_url` receives the
/// authenticated URL to embed and runs under the embed lock.
pub(crate) async fn refresh<F, Fut>(
&self,
origin_url: &str,
set_url: F,
) -> crate::Result<RefreshOutcome>
where
F: FnOnce(DisplaySafeUrl) -> Fut,
Fut: Future<Output = crate::Result<()>>,
{
let Some(source) = &self.source else {
return Ok(RefreshOutcome::none());
};
let mut embedded = self.embedded.lock().await;
let resolved = match source.resolve().await {
Ok(resolved) => resolved,
Err(err) => {
// The refresh-error path is defined, not incidental: the push
// proceeds with the last embedded token, so log which one
// that is instead of losing the credential state.
if let Some(prev) = embedded.as_ref() {
tracing::warn!(
error = %format!("{err:#}"),
generation = prev.snapshot.generation,
provenance = %prev.snapshot.provenance,
token_age_ms = prev.snapshot.age_ms(),
"GitHub token refresh failed; origin keeps the last embedded credentials"
);
} else {
tracing::warn!(
error = %format!("{err:#}"),
"GitHub token refresh failed and no credentials were ever embedded"
);
}
return Err(crate::Error::context_anyhow(
"Failed to refresh push credentials",
err,
));
}
};
if embedded
.as_ref()
.is_some_and(|prev| prev.snapshot.generation == resolved.snapshot.generation)
{
return Ok(RefreshOutcome::unchanged(resolved.snapshot));
}
let auth_url = fabro_github::embed_token_in_url(origin_url, resolved.token.expose())
.map_err(|err| {
crate::Error::context_anyhow("Failed to build authenticated origin URL", err)
})?;
set_url(auth_url).await?;
let snapshot = resolved.snapshot;
*embedded = Some(resolved);
Ok(RefreshOutcome::embedded(snapshot))
}
}
/// What [`CredentialLease::ensure_embedded`] did for one push attempt.
#[derive(Debug, Clone, Copy)]
pub(crate) struct EnsureOutcome {
pub action: RemoteCredentialAction,
/// The token embedded in the remote right now — never an unembedded mint.
pub token: Option<TokenSnapshot>,
pub refresh_error: Option<RefreshErrorKind>,
}
/// Scoped pin of push credentials for one push operation.
///
/// Holds the provider's embed mutex until dropped, so no other refresh can
/// re-embed mid-operation — a refresh-ahead tick crossing the cache margin
/// during a retrying push waits here instead of swapping the remote out from
/// under the pin. Internally retains up to two secrets: the last successfully
/// embedded token (the fallback) and the operation's resolved target, so both
/// drift re-embedding and the refresh-error fallback work. Only non-secret
/// snapshots leave the lease.
///
/// A successful resolve happens at most once per operation and is never
/// replaced; the pin transitions to the target only through a successful
/// embed. The token source's refresh margin exceeds every push plan's elapsed
/// bound, so the pinned token always outlives the operation.
pub(crate) struct CredentialLease<'a> {
source: Option<&'a InstallationTokenSource>,
/// Embed-mutex guard: the last successfully embedded token.
embedded: MutexGuard<'a, Option<ResolvedToken>>,
/// The operation's resolved target, including a cached fallback when a
/// refresh mint failed.
target: Option<ResolvedToken>,
/// Skip an immediate duplicate resolve after lease acquisition already
/// failed. A later push attempt can retry after backoff.
defer_resolve_once: bool,
}
impl PushCredentialState {
/// Acquire the push-credential lease for one push operation.
///
/// Resolves the operation's target token up front. A failed refresh can
/// return a valid cached token; the first attempt uses it, and
/// [`CredentialLease::ensure_embedded`] retries the refresh after push
/// backoff. A resolve with no cached or embedded token fails acquisition.
pub(crate) async fn lease(&self) -> crate::Result<CredentialLease<'_>> {
let embedded = self.embedded.lock().await;
let Some(source) = self.source.as_deref() else {
return Ok(CredentialLease {
source: None,
embedded,
target: None,
defer_resolve_once: false,
});
};
match source.resolve().await {
Ok(resolved) => {
let defer_resolve_once = resolved.refresh_failed;
Ok(CredentialLease {
source: Some(source),
embedded,
target: Some(resolved),
defer_resolve_once,
})
}
Err(err) => {
if let Some(prev) = embedded.as_ref() {
tracing::warn!(
error = %format!("{err:#}"),
generation = prev.snapshot.generation,
provenance = %prev.snapshot.provenance,
token_age_ms = prev.snapshot.age_ms(),
"token resolve failed; push pins the last embedded credentials"
);
Ok(CredentialLease {
source: Some(source),
embedded,
target: None,
defer_resolve_once: true,
})
} else {
tracing::warn!(
error = %format!("{err:#}"),
"token resolve failed and no credentials were ever embedded"
);
Err(crate::Error::message(
"Failed to refresh push credentials: token_mint_failed",
))
}
}
}
}
}
impl CredentialLease<'_> {
/// Non-secret description of the token embedded in the remote right now.
pub(crate) fn snapshot(&self) -> Option<TokenSnapshot> {
self.embedded.as_ref().map(|token| token.snapshot)
}
/// Embed the pinned generation if the remote does not carry it.
///
/// One call covers the initial embed, a deferred embed after an earlier
/// failure, and drift repair (`force` re-embeds even when the tracked
/// generation matches, for remotes rewritten inside the sandbox). While
/// the lease has no target, this retries the failed `resolve()` first —
/// retrying a failed resolve discards no fresh token, so it cannot
/// restart any replication clock. Refresh failures are recorded, never
/// propagated: the push proceeds with the last embedded token.
pub(crate) async fn ensure_embedded(
&mut self,
sandbox: &dyn crate::Sandbox,
origin_url: &str,
force: bool,
) -> crate::Result<EnsureOutcome> {
let Some(source) = self.source else {
return Ok(EnsureOutcome {
action: RemoteCredentialAction::None,
token: None,
refresh_error: None,
});
};
let mut refresh_error = self.defer_resolve_once.then_some(RefreshErrorKind::Mint);
if self.defer_resolve_once {
self.defer_resolve_once = false;
} else if self
.target
.as_ref()
.is_none_or(|resolved| resolved.refresh_failed)
{
match source.resolve().await {
Ok(resolved) => {
refresh_error = resolved.refresh_failed.then_some(RefreshErrorKind::Mint);
self.target = Some(resolved);
}
Err(err) => {
tracing::warn!(
error = %format!("{err:#}"),
"token resolve retry failed; pushing with the last embedded token"
);
refresh_error = Some(RefreshErrorKind::Mint);
}
}
}
let Some(desired) = self.target.as_ref().or(self.embedded.as_ref()).cloned() else {
// Managed credentials with nothing resolved or embedded:
// acquisition fails before any attempt runs, so pushes never see
// this state.
return Ok(EnsureOutcome {
action: RemoteCredentialAction::None,
token: None,
refresh_error,
});
};
let embedded_generation = self
.embedded
.as_ref()
.map(|token| token.snapshot.generation);
if !force && embedded_generation == Some(desired.snapshot.generation) {
return Ok(EnsureOutcome {
action: RemoteCredentialAction::Unchanged,
token: Some(desired.snapshot),
refresh_error,
});
}
match set_url_via_exec(sandbox, origin_url, &desired).await {
Ok(()) => {
let snapshot = desired.snapshot;
*self.embedded = Some(desired);
Ok(EnsureOutcome {
action: RemoteCredentialAction::Embedded,
token: Some(snapshot),
refresh_error,
})
}
Err(err) => {
if matches!(
&err,
crate::Error::Exec { result, .. }
if result.termination != fabro_types::CommandTermination::Exited
) {
return Err(err);
}
tracing::warn!(
error = %crate::display_for_log(&err),
"embedding push credentials in origin failed; pushing with the last embedded token"
);
Ok(EnsureOutcome {
action: RemoteCredentialAction::Unchanged,
token: self.snapshot(),
refresh_error: Some(RefreshErrorKind::SetUrl),
})
}
}
}
}
/// Rewrite `origin` with the token embedded, through the sandbox's uniform
/// exec surface.
async fn set_url_via_exec(
sandbox: &dyn crate::Sandbox,
origin_url: &str,
token: &ResolvedToken,
) -> crate::Result<()> {
let auth_url =
fabro_github::embed_token_in_url(origin_url, token.token.expose()).map_err(|err| {
crate::Error::context(
"Failed to build authenticated origin URL",
RedactedSetUrlError(fabro_redact::redact_string(&format!("{err:#}"))),
)
})?;
set_auth_url_via_exec(sandbox, auth_url).await
}
pub(crate) async fn set_auth_url_via_exec(
sandbox: &dyn crate::Sandbox,
auth_url: DisplaySafeUrl,
) -> crate::Result<()> {
let command = format!(
"git -c maintenance.auto=0 remote set-url origin {}",
crate::shell_quote(auth_url.as_raw_url().as_str())
);
let result = sandbox
.exec_command(&command, 10_000, None, None, None)
.await
.map_err(|err| {
let message = redact::redact_auth_url(&crate::display_for_log(&err), Some(&auth_url));
crate::Error::context(
"Failed to refresh push credentials: set_url_exec_failed",
RedactedSetUrlError(message),
)
})?;
if !result.is_success() {
return Err(result.into_exec_error_with_redactor(
"git remote set-url origin (refresh push credentials)",
|s| redact::redact_auth_url(s, Some(&auth_url)),
));
}
Ok(())
}
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct RedactedSetUrlError(String);
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use chrono::Utc;
use fabro_github::InstallationToken;
use fabro_github::test_support::{InstallationTokenMinter, installation_token_source};
use tokio::time::sleep;
use super::*;
use crate::sandbox::RemoteCredentialAction;
struct FixedMinter {
calls: AtomicUsize,
ttl: chrono::Duration,
}
#[async_trait::async_trait]
impl InstallationTokenMinter for FixedMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
let call = self.calls.fetch_add(1, Ordering::SeqCst) + 1;
Ok(InstallationToken {
token: format!("ghs_gen{call}"),
expires_at: Utc::now() + self.ttl,
})
}
}
struct FailingMinter;
#[async_trait::async_trait]
impl InstallationTokenMinter for FailingMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
Err(anyhow::anyhow!("mint failed"))
}
}
fn minting_state(ttl: chrono::Duration) -> PushCredentialState {
PushCredentialState::new(Some(installation_token_source(
"owner/repo",
Arc::new(FixedMinter {
calls: AtomicUsize::new(0),
ttl,
}),
)))
}
const ORIGIN: &str = "https://github.com/owner/repo";
/// Long enough for a blocked task to be observably pending on paused time.
const SHORT_WAIT: std::time::Duration = std::time::Duration::from_secs(5);
/// A refresh-ahead tick crossing the cache margin during a push waits on
/// the embed mutex until the operation releases the lease, so the remote
/// can never be swapped out from under the pinned generation.
#[tokio::test(start_paused = true)]
async fn refresh_waits_for_the_lease_to_release() {
let state = std::sync::Arc::new(minting_state(chrono::Duration::minutes(60)));
let lease = state.lease().await.expect("lease acquires");
let refresh_task = {
let state = std::sync::Arc::clone(&state);
tokio::spawn(async move {
state
.refresh(ORIGIN, |_| async { Ok(()) })
.await
.expect("refresh succeeds after the lease releases")
})
};
// The refresh must be blocked while the lease holds the embed mutex.
sleep(SHORT_WAIT).await;
assert!(
!refresh_task.is_finished(),
"refresh must wait on the embed mutex"
);
drop(lease);
let outcome = refresh_task.await.expect("refresh task completes");
// The lease's resolve minted generation 1; the deferred refresh
// reuses it (the operation never embedded, so the refresh embeds).
assert_eq!(outcome.token().unwrap().generation, 1);
}
#[tokio::test]
async fn refresh_without_managed_credentials_reports_none() {
let state = PushCredentialState::new(None);
let outcome = state
.refresh(ORIGIN, |_| async { panic!("set-url must not run") })
.await
.unwrap();
assert_eq!(outcome, RefreshOutcome::none());
}
#[tokio::test]
async fn refresh_embeds_a_new_generation_and_skips_matching_ones() {
let state = minting_state(chrono::Duration::minutes(60));
let set_url_calls = AtomicUsize::new(0);
let first = state
.refresh(ORIGIN, |auth_url| {
set_url_calls.fetch_add(1, Ordering::SeqCst);
assert!(auth_url.as_raw_url().as_str().contains("ghs_gen1"));
async { Ok(()) }
})
.await
.unwrap();
assert_eq!(first.action(), RemoteCredentialAction::Embedded);
assert_eq!(first.token().unwrap().generation, 1);
// The cached token is fresh, so the second refresh must skip set-url.
let second = state
.refresh(ORIGIN, |_| {
set_url_calls.fetch_add(1, Ordering::SeqCst);
async { Ok(()) }
})
.await
.unwrap();
assert_eq!(second.action(), RemoteCredentialAction::Unchanged);
assert_eq!(second.token().unwrap().generation, 1);
assert_eq!(set_url_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn refresh_embeds_again_when_the_source_mints_a_new_generation() {
// Tokens expire inside the margin, so every resolve re-mints.
let state = minting_state(chrono::Duration::minutes(5));
let set_url_calls = AtomicUsize::new(0);
let first = state
.refresh(ORIGIN, |_| {
set_url_calls.fetch_add(1, Ordering::SeqCst);
async { Ok(()) }
})
.await
.unwrap();
let second = state
.refresh(ORIGIN, |_| {
set_url_calls.fetch_add(1, Ordering::SeqCst);
async { Ok(()) }
})
.await
.unwrap();
assert_eq!(first.token().unwrap().generation, 1);
assert_eq!(second.action(), RemoteCredentialAction::Embedded);
assert_eq!(second.token().unwrap().generation, 2);
assert_eq!(set_url_calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn clone_seed_makes_the_first_refresh_a_no_op() {
let state = minting_state(chrono::Duration::minutes(60));
let clone_token = state.source().unwrap().mint_for_clone().await.unwrap();
state.record_embedded(clone_token).await;
let outcome = state
.refresh(ORIGIN, |_| async { panic!("set-url must not run") })
.await
.unwrap();
assert_eq!(outcome.action(), RemoteCredentialAction::Unchanged);
assert_eq!(outcome.token().unwrap().generation, 1);
}
#[tokio::test]
async fn failed_set_url_does_not_record_the_new_generation() {
let state = minting_state(chrono::Duration::minutes(60));
let err = state
.refresh(ORIGIN, |_| async {
Err(crate::Error::message("set-url failed"))
})
.await
.unwrap_err();
assert!(err.to_string().contains("set-url failed"));
// The generation was not recorded, so the retry embeds again instead
// of wrongly skipping.
let retried = state.refresh(ORIGIN, |_| async { Ok(()) }).await.unwrap();
assert_eq!(retried.action(), RemoteCredentialAction::Embedded);
assert_eq!(retried.token().unwrap().generation, 1);
}
#[tokio::test]
async fn static_credentials_seeded_at_clone_skip_set_url() {
let source = InstallationTokenSource::for_origin(
&GitHubCredentials::Pat("ghp_pat".to_string()),
ORIGIN,
serde_json::json!({ "contents": "write" }),
)
.unwrap();
let state = PushCredentialState::new(Some(source));
let clone_token = state.source().unwrap().mint_for_clone().await.unwrap();
state.record_embedded(clone_token).await;
let outcome = state
.refresh(ORIGIN, |_| async { panic!("set-url must not run") })
.await
.unwrap();
assert_eq!(outcome.action(), RemoteCredentialAction::Unchanged);
assert!(outcome.token().unwrap().is_static());
}
#[tokio::test]
async fn mint_failure_preserves_the_mint_error_chain() {
let state = PushCredentialState::new(Some(installation_token_source(
"owner/repo",
Arc::new(FailingMinter),
)));
let err = state
.refresh(ORIGIN, |_| async { panic!("set-url must not run") })
.await
.unwrap_err();
assert_eq!(err.causes(), vec![
"minting GitHub installation access token",
"mint failed"
]);
}
#[test]
fn token_source_requires_managed_credentials_and_a_github_origin() {
assert!(build_token_source(None, Some(ORIGIN)).unwrap().is_none());
let pat = GitHubCredentials::Pat("ghp_pat".to_string());
assert!(build_token_source(Some(&pat), None).unwrap().is_none());
assert!(
build_token_source(Some(&pat), Some("https://gitlab.com/owner/repo"))
.unwrap()
.is_none()
);
assert!(
build_token_source(Some(&pat), Some(ORIGIN))
.unwrap()
.is_some()
);
}
}

1059
lib/components/fabro-sandbox/src/sandbox.rs
View file

File diff suppressed because it is too large Load diff

2
lib/components/fabro-sandbox/src/sandbox_spec.rs
View file

@ -208,7 +208,7 @@ impl SandboxSpec {
} => {
let mut sandbox = DockerSandbox::new(
config.clone(),
github_app.clone(),
github_app.as_ref(),
*run_id,
clone_origin_url.clone(),
clone_branch.clone(),

14
lib/components/fabro-sandbox/src/test_support.rs
View file

@ -1,6 +1,6 @@
use std::collections::HashMap;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use async_trait::async_trait;
@ -29,6 +29,8 @@ pub struct MockSandbox {
pub os_version_str: String,
/// Captures (path, content) pairs from `write_file` calls.
pub written_files: Mutex<Vec<(String, String)>>,
/// Counts calls to `write_existing_file`.
pub existing_file_writes: AtomicUsize,
/// Captures the `timeout_ms` argument from `exec_command` calls.
pub captured_timeout: Mutex<Option<u64>>,
/// Captures the `command` argument from `exec_command` calls (last only).
@ -104,6 +106,10 @@ impl MockSandbox {
.expect("delete_calls lock poisoned")
}
pub fn existing_file_write_count(&self) -> usize {
self.existing_file_writes.load(Ordering::Relaxed)
}
pub fn set_stdio_process(&self, process: MockStdioProcess) {
*self
.stdio_process
@ -156,6 +162,7 @@ impl Default for MockSandbox {
platform_str: "darwin",
os_version_str: "Darwin 24.0.0".into(),
written_files: Mutex::new(Vec::new()),
existing_file_writes: AtomicUsize::new(0),
captured_timeout: Mutex::new(None),
captured_command: Mutex::new(None),
captured_commands: Mutex::new(Vec::new()),
@ -250,6 +257,11 @@ impl Sandbox for MockSandbox {
Ok(())
}
async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> {
self.existing_file_writes.fetch_add(1, Ordering::Relaxed);
self.write_file(path, content).await
}
async fn delete_file(&self, _path: &str) -> crate::Result<()> {
Ok(())
}

301
lib/components/fabro-store/src/blob_store.rs Normal file
View file

@ -0,0 +1,301 @@
use std::sync::Arc;
use bytes::Bytes;
use fabro_types::BlobHash;
use sqlx::SqlitePool;
use crate::record::{RawBytesCodec, Record, Repository};
use crate::{Error, Result};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Blob(pub Bytes);
impl AsRef<[u8]> for Blob {
fn as_ref(&self) -> &[u8] {
self.0.as_ref()
}
}
impl From<Bytes> for Blob {
fn from(value: Bytes) -> Self {
Self(value)
}
}
impl Record for Blob {
type Id = BlobHash;
type Codec = RawBytesCodec;
const PREFIX: &'static str = "blobs/sha256";
fn id(&self) -> Self::Id {
BlobHash::new(&self.0)
}
}
/// Which storage engine holds the blobs.
///
/// This enum is a transition vehicle, not a permanent abstraction: `Slate`
/// preserves current production behavior while the SQLite backend rolls out.
/// Once runtime blob storage switches to SQLite and legacy blobs are
/// imported, delete the `Slate` arm (and this enum) and inline the SQLite
/// implementation into [`BlobStore`]. The SQLite arm's semantics — verified
/// reads and loud failure on hash conflicts — are the intended end state.
enum BlobBackend {
Slate(Repository<Blob>),
Sqlite(SqlitePool),
}
pub struct BlobStore {
backend: BlobBackend,
}
impl std::fmt::Debug for BlobStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let backend = match &self.backend {
BlobBackend::Slate(_) => "slate",
BlobBackend::Sqlite(_) => "sqlite",
};
f.debug_struct("BlobStore")
.field("backend", &backend)
.finish_non_exhaustive()
}
}
impl BlobStore {
/// Creates a blob store backed by a SQLite pool whose migrations have run.
#[must_use]
pub fn new(pool: SqlitePool) -> Self {
Self {
backend: BlobBackend::Sqlite(pool),
}
}
pub(crate) fn from_slate(db: Arc<slatedb::Db>) -> Self {
Self {
backend: BlobBackend::Slate(Repository::new(db)),
}
}
pub async fn write(&self, bytes: &[u8]) -> Result<BlobHash> {
match &self.backend {
BlobBackend::Slate(repo) => {
let blob = Blob(Bytes::copy_from_slice(bytes));
let id = blob.id();
repo.put(&blob).await?;
Ok(id)
}
BlobBackend::Sqlite(pool) => {
let blob_hash = BlobHash::new(bytes);
let result = sqlx::query(
"INSERT INTO blobs (hash, data) VALUES (?, ?) \
ON CONFLICT(hash) DO NOTHING",
)
.bind(blob_hash.to_string())
.bind(bytes)
.execute(pool)
.await?;
if result.rows_affected() == 1 {
return Ok(blob_hash);
}
let stored: Vec<u8> = sqlx::query_scalar("SELECT data FROM blobs WHERE hash = ?")
.bind(blob_hash.to_string())
.fetch_one(pool)
.await?;
if stored == bytes {
Ok(blob_hash)
} else {
Err(Error::BlobHashConflict { blob_hash })
}
}
}
}
pub async fn read(&self, blob_hash: &BlobHash) -> Result<Option<Bytes>> {
match &self.backend {
BlobBackend::Slate(repo) => Ok(repo.get(blob_hash).await?.map(|blob| blob.0)),
BlobBackend::Sqlite(pool) => {
let stored: Option<Vec<u8>> =
sqlx::query_scalar("SELECT data FROM blobs WHERE hash = ?")
.bind(blob_hash.to_string())
.fetch_optional(pool)
.await?;
let Some(stored) = stored else {
return Ok(None);
};
if BlobHash::new(&stored) != *blob_hash {
return Err(Error::BlobIntegrity {
blob_hash: *blob_hash,
});
}
Ok(Some(Bytes::from(stored)))
}
}
}
pub async fn exists(&self, blob_hash: &BlobHash) -> Result<bool> {
match &self.backend {
BlobBackend::Slate(repo) => repo.exists(blob_hash).await,
BlobBackend::Sqlite(pool) => {
let exists: bool =
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM blobs WHERE hash = ?)")
.bind(blob_hash.to_string())
.fetch_one(pool)
.await?;
Ok(exists)
}
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use fabro_types::BlobHash;
use object_store::memory::InMemory;
use super::BlobStore;
use crate::keys::SlateKey;
use crate::{Database, Error};
type TestResult<T> = std::result::Result<T, Box<dyn std::error::Error>>;
async fn slate_store() -> Arc<BlobStore> {
let db = Database::new(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
);
db.blobs().await.unwrap()
}
async fn raw_slate_store(name: &str) -> (Arc<slatedb::Db>, BlobStore) {
let raw_db = Arc::new(
slatedb::Db::open(name, Arc::new(InMemory::new()))
.await
.unwrap(),
);
let store = BlobStore::from_slate(raw_db.clone());
(raw_db, store)
}
async fn sqlite_store() -> TestResult<(tempfile::TempDir, fabro_db::Database, BlobStore)> {
let dir = tempfile::tempdir()?;
let database = fabro_db::Database::connect(dir.path().join("fabro.sqlite3")).await?;
database.migrate().await?;
let store = BlobStore::new(database.clone_pool());
Ok((dir, database, store))
}
#[tokio::test]
async fn slate_writes_reads_and_checks_existence() {
let store = slate_store().await;
let bytes = b"hello world";
let id = store.write(bytes).await.unwrap();
assert_eq!(
store.read(&id).await.unwrap(),
Some(Bytes::from_static(bytes))
);
assert_eq!(store.write(bytes).await.unwrap(), id);
assert!(store.exists(&id).await.unwrap());
assert!(!store.exists(&BlobHash::new(b"missing")).await.unwrap());
}
#[tokio::test]
async fn slate_empty_blobs_round_trip() {
let store = slate_store().await;
let id = store.write(b"").await.unwrap();
assert_eq!(store.read(&id).await.unwrap(), Some(Bytes::new()));
}
#[tokio::test]
async fn raw_slate_db_reads_exact_blob_bytes() {
let (raw_db, store) = raw_slate_store("blob-store-tests").await;
let bytes = b"{\"ok\":true}";
let id = store.write(bytes).await.unwrap();
let saved = raw_db
.get(SlateKey::new("blobs").with("sha256").with(id))
.await
.unwrap()
.unwrap();
assert_eq!(saved.as_ref(), bytes);
}
#[tokio::test]
async fn sqlite_writes_reads_and_checks_existence() -> TestResult<()> {
let (_dir, database, store) = sqlite_store().await?;
let store = Arc::new(store);
let binary = [0_u8, 0xff, 0x80, b'a'];
let (first_write, concurrent_write) =
tokio::join!(store.write(&binary), store.write(&binary));
let binary_hash = first_write?;
assert_eq!(concurrent_write?, binary_hash);
let empty_hash = store.write(b"").await?;
assert_eq!(store.write(&binary).await?, binary_hash);
assert_eq!(
store.read(&binary_hash).await?,
Some(Bytes::copy_from_slice(&binary))
);
assert_eq!(store.read(&empty_hash).await?, Some(Bytes::new()));
assert!(store.exists(&binary_hash).await?);
let missing_hash = BlobHash::new(b"missing");
assert_eq!(store.read(&missing_hash).await?, None);
assert!(!store.exists(&missing_hash).await?);
let row_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM blobs")
.fetch_one(database.pool())
.await?;
assert_eq!(row_count, 2);
Ok(())
}
#[tokio::test]
async fn sqlite_write_rejects_conflicting_stored_bytes() -> TestResult<()> {
let (_dir, database, store) = sqlite_store().await?;
let expected = b"expected";
let blob_hash = BlobHash::new(expected);
sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)")
.bind(blob_hash.to_string())
.bind(b"different".as_slice())
.execute(database.pool())
.await?;
let error = store
.write(expected)
.await
.expect_err("conflicting bytes should fail");
assert!(
matches!(error, Error::BlobHashConflict { blob_hash: value } if value == blob_hash)
);
Ok(())
}
#[tokio::test]
async fn sqlite_read_rejects_bytes_that_do_not_match_hash() -> TestResult<()> {
let (_dir, database, store) = sqlite_store().await?;
let blob_hash = BlobHash::new(b"expected");
sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)")
.bind(blob_hash.to_string())
.bind(b"different".as_slice())
.execute(database.pool())
.await?;
let error = store
.read(&blob_hash)
.await
.expect_err("mismatched stored bytes should fail");
assert!(matches!(error, Error::BlobIntegrity { blob_hash: value } if value == blob_hash));
Ok(())
}
}

6
lib/components/fabro-store/src/error.rs
View file

@ -1,3 +1,5 @@
use fabro_types::BlobHash;
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, thiserror::Error)]
@ -10,6 +12,10 @@ pub enum Error {
Serde(#[from] serde_json::Error),
#[error("SQLite error: {0}")]
Sqlite(#[from] sqlx::Error),
#[error("stored blob {blob_hash} has bytes that conflict with its hash")]
BlobHashConflict { blob_hash: BlobHash },
#[error("stored blob data does not match requested hash {blob_hash}")]
BlobIntegrity { blob_hash: BlobHash },
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Invalid event payload: {0}")]

6
lib/components/fabro-store/src/lib.rs
View file

@ -1,6 +1,7 @@
use chrono::{DateTime, Utc};
mod artifact_store;
mod blob_store;
mod error;
mod keyed_mutex;
mod keys;
@ -18,6 +19,7 @@ pub use artifact_store::{
ArtifactKey, ArtifactStore, NodeArtifact, StageArtifactEntry, retry_storage_segment,
stage_storage_segment,
};
pub use blob_store::{Blob, BlobStore};
pub use error::{Error, Result};
pub use fabro_types::{
BlobHash, EventEnvelope, PendingInterviewRecord, Run, RunProjection, StageId, StageProjection,
@ -34,8 +36,8 @@ pub use run_summary_store::{
};
pub use serializable_projection::SerializableProjection;
pub use slate::{
AuthCode, AuthCodeStore, Blob, BlobStore, CachedRunProjection, ConsumeOutcome, Database,
RefreshToken, RefreshTokenStore, RunCatalogIndex, RunDatabase, Runs, UnreadableRun,
AuthCode, AuthCodeStore, CachedRunProjection, ConsumeOutcome, Database, RefreshToken,
RefreshTokenStore, RunCatalogIndex, RunDatabase, Runs, UnreadableRun,
};
pub use types::EventPayload;

78
lib/components/fabro-store/src/run_state.rs
View file

@ -1046,6 +1046,7 @@ fn projection_from_created(event: &EventEnvelope) -> Result<RunProjection> {
provenance: props.provenance.clone(),
manifest_blob: props.manifest_blob,
definition_blob: None,
spec_blob: props.spec_blob,
git: props.git.clone(),
fork_source_ref: props.fork_source_ref.clone(),
};
@ -1360,8 +1361,7 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
.conclusion
.as_ref()
.map(|conclusion| conclusion.timing);
let terminal_total = terminal_total_usd_micros(state);
let current_total = projected_billing(state).total_usd_micros;
let total_usd_micros = projected_billing(state).total_usd_micros;
Run {
id: *run_id,
@ -1405,10 +1405,10 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
completed_at,
},
timing: run_timing,
billing: terminal_total.map(|total_usd_micros| RunBillingSummary {
billing: total_usd_micros.map(|total_usd_micros| RunBillingSummary {
total_usd_micros: Some(total_usd_micros),
}),
size: RunSize::from_total_usd_micros(current_total),
size: RunSize::from_total_usd_micros(total_usd_micros),
ask_fabro: AskFabro::default(),
diff: diff_summary,
pull_request: state.pull_request.clone(),
@ -1421,14 +1421,6 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
}
}
fn terminal_total_usd_micros(state: &RunProjection) -> Option<i64> {
state
.conclusion
.as_ref()
.and_then(|conclusion| conclusion.billing.as_ref())
.and_then(|billing| billing.total_usd_micros)
}
pub(crate) fn projected_billing(state: &RunProjection) -> BilledTokenCounts {
if let Some(billing) = state
.conclusion
@ -1694,11 +1686,12 @@ mod tests {
CommandTermination, EventBody, FailureCategory, FailureDetail, FailureReason, Graph,
McpServerStatus, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel,
PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort,
RunApprovalState, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec, RunStatus, Speed,
StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod,
StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowWarning,
StageHandler, StageModelUsage, StageOutcome, StageState, StageTiming, SubAgentStatus,
SuccessReason, WorkflowSettings, first_event_seq, fixtures, test_support,
RunApprovalState, RunBillingSummary, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec,
RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory,
StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness,
StageContextWindowWarning, StageHandler, StageModelUsage, StageOutcome, StageState,
StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, first_event_seq, fixtures,
test_support,
};
use serde_json::json;
@ -2281,19 +2274,8 @@ mod tests {
fn test_run_spec() -> RunSpec {
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: Graph::new("test"),
graph_source: Some("digraph test {}".to_string()),
workflow_slug: None,
automation: None,
source_directory: None,
labels: HashMap::new(),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
git: None,
fork_source_ref: None,
graph_source: Some("digraph test {}".to_string()),
..test_support::test_run_spec()
}
}
@ -4086,19 +4068,9 @@ mod tests {
fn summary_synthesizes_submitted_when_run_exists_without_status() {
let mut state = initialized_projection();
state.spec = fabro_types::RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: fabro_types::Graph::new("test"),
graph_source: None,
workflow_slug: Some("test".to_string()),
automation: None,
workflow_slug: Some("test".to_string()),
source_directory: Some("/tmp/repo".to_string()),
git: None,
labels: HashMap::new(),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
..test_support::test_run_spec()
};
let summary_json = serde_json::to_value(build_summary(&state, &fixtures::RUN_1)).unwrap();
@ -4112,19 +4084,10 @@ mod tests {
fn summary_preserves_absent_workflow_name_and_reports_graph_name() {
let mut state = initialized_projection();
state.spec = fabro_types::RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: fabro_types::Graph::new("GraphName"),
graph_source: None,
workflow_slug: Some("release-flow".to_string()),
automation: None,
graph: fabro_types::Graph::new("GraphName"),
workflow_slug: Some("release-flow".to_string()),
source_directory: Some("/tmp/repo".to_string()),
git: None,
labels: HashMap::new(),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
..test_support::test_run_spec()
};
let summary = build_summary(&state, &fixtures::RUN_1);
@ -5284,7 +5247,12 @@ mod tests {
let summary = build_summary(&state, &fixtures::RUN_1);
assert_eq!(summary.size, RunSize::S);
assert_eq!(summary.billing, None);
assert_eq!(
summary.billing,
Some(RunBillingSummary {
total_usd_micros: Some(20_000_001),
})
);
}
#[test]

1
lib/components/fabro-store/src/run_summary_store.rs
View file

@ -601,6 +601,7 @@ mod tests {
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
},

184
lib/components/fabro-store/src/slate/blob_store.rs
View file

@ -1,184 +0,0 @@
use std::sync::Arc;
use bytes::Bytes;
use fabro_types::BlobHash;
use futures::StreamExt;
use tracing::warn;
use crate::record::{RawBytesCodec, Record, Repository};
use crate::{Error, Result};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Blob(pub Bytes);
impl AsRef<[u8]> for Blob {
fn as_ref(&self) -> &[u8] {
self.0.as_ref()
}
}
impl From<Bytes> for Blob {
fn from(value: Bytes) -> Self {
Self(value)
}
}
impl Record for Blob {
type Id = BlobHash;
type Codec = RawBytesCodec;
const PREFIX: &'static str = "blobs/sha256";
fn id(&self) -> Self::Id {
BlobHash::new(&self.0)
}
}
pub struct BlobStore {
repo: Repository<Blob>,
}
impl std::fmt::Debug for BlobStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BlobStore").finish_non_exhaustive()
}
}
impl BlobStore {
pub(crate) fn new(db: Arc<slatedb::Db>) -> Self {
Self {
repo: Repository::new(db),
}
}
pub async fn write(&self, bytes: &[u8]) -> Result<BlobHash> {
let blob = Blob(Bytes::copy_from_slice(bytes));
let id = blob.id();
self.repo.put(&blob).await?;
Ok(id)
}
pub async fn read(&self, id: &BlobHash) -> Result<Option<Bytes>> {
Ok(self.repo.get(id).await?.map(|blob| blob.0))
}
pub async fn exists(&self, id: &BlobHash) -> Result<bool> {
self.repo.exists(id).await
}
pub(crate) async fn list(&self) -> Result<Vec<BlobHash>> {
let mut stream = self.repo.scan_ids_stream();
let mut ids = Vec::new();
while let Some(result) = stream.next().await {
match result {
Ok(id) => ids.push(id),
Err(Error::KeyParse(err)) => {
warn!(error = %err, "Skipping malformed blob key during listing");
}
Err(err) => return Err(err),
}
}
ids.sort();
Ok(ids)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use fabro_types::BlobHash;
use object_store::memory::InMemory;
use super::BlobStore;
use crate::Database;
use crate::keys::SlateKey;
async fn store() -> Arc<BlobStore> {
let db = Database::new(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
);
db.blobs().await.unwrap()
}
async fn raw_store(name: &str) -> (Arc<slatedb::Db>, BlobStore) {
let raw_db = Arc::new(
slatedb::Db::open(name, Arc::new(InMemory::new()))
.await
.unwrap(),
);
let store = BlobStore::new(Arc::clone(&raw_db));
(raw_db, store)
}
#[tokio::test]
async fn writes_reads_and_checks_existence() {
let store = store().await;
let bytes = b"hello world";
let id = store.write(bytes).await.unwrap();
assert_eq!(
store.read(&id).await.unwrap(),
Some(Bytes::from_static(bytes))
);
assert_eq!(store.write(bytes).await.unwrap(), id);
assert!(store.exists(&id).await.unwrap());
assert!(!store.exists(&BlobHash::new(b"missing")).await.unwrap());
}
#[tokio::test]
async fn empty_blobs_round_trip() {
let store = store().await;
let id = store.write(b"").await.unwrap();
assert_eq!(store.read(&id).await.unwrap(), Some(Bytes::new()));
}
#[tokio::test]
async fn list_returns_sorted_ids_and_handles_empty_store() {
let store = store().await;
assert!(store.list().await.unwrap().is_empty());
let first_id = store.write(br#"{"z":1}"#).await.unwrap();
let second_id = store.write(br#"{"a":1}"#).await.unwrap();
let mut expected = vec![first_id, second_id];
expected.sort();
assert_eq!(store.list().await.unwrap(), expected);
}
#[tokio::test]
async fn list_skips_malformed_blob_ids() {
let (raw_db, store) = raw_store("blob-store-list-tests").await;
let id = store.write(b"valid").await.unwrap();
raw_db
.put(
SlateKey::new("blobs").with("sha256").with("not-a-blob-id"),
b"malformed",
)
.await
.unwrap();
assert_eq!(store.list().await.unwrap(), vec![id]);
}
#[tokio::test]
async fn raw_db_reads_exact_blob_bytes() {
let (raw_db, store) = raw_store("blob-store-tests").await;
let bytes = b"{\"ok\":true}";
let id = store.write(bytes).await.unwrap();
let saved = raw_db
.get(SlateKey::new("blobs").with("sha256").with(id))
.await
.unwrap()
.unwrap();
assert_eq!(saved.as_ref(), bytes);
}
}

17
lib/components/fabro-store/src/slate/mod.rs
View file

@ -1,6 +1,5 @@
mod auth_codes;
mod auth_tokens;
mod blob_store;
mod projection_cache;
mod run_catalog_index;
mod run_store;
@ -12,7 +11,6 @@ use std::time::Duration;
pub use auth_codes::{AuthCode, AuthCodeStore};
pub use auth_tokens::{ConsumeOutcome, RefreshToken, RefreshTokenStore};
pub use blob_store::{Blob, BlobStore};
use chrono::{DateTime, Utc};
use fabro_types::{Run, RunId, SessionId};
use object_store::ObjectStore;
@ -25,7 +23,7 @@ use slatedb::config::{CompressionCodec, Settings};
use tokio::sync::{Mutex, OnceCell};
use tracing::warn;
use crate::{Error, ListRunsQuery, Result, RunProjection, RunSummaryStore, keys};
use crate::{BlobStore, Error, ListRunsQuery, Result, RunProjection, RunSummaryStore, keys};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UnreadableRun {
@ -449,7 +447,7 @@ impl Database {
.blobs
.get_or_try_init(|| async {
let db = Arc::new(self.open_db().await?);
Ok::<_, Error>(Arc::new(BlobStore::new(db)))
Ok::<_, Error>(Arc::new(BlobStore::from_slate(db)))
})
.await?;
Ok(Arc::clone(store))
@ -595,6 +593,7 @@ mod tests {
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: Some(fabro_types::GitContext {
origin_url: "https://github.com/fabro-sh/fabro".to_string(),
branch: "main".to_string(),
@ -836,12 +835,12 @@ mod tests {
append_created(&run_2, "run-2", dt("2026-03-27T12:00:10Z")).await;
let shared_blob = br#"{"summary":"shared"}"#;
let shared_blob_id = run_1.write_blob(shared_blob).await.unwrap();
let shared_blob_hash = run_1.write_blob(shared_blob).await.unwrap();
store.delete_run(&test_run_id("run-1")).await.unwrap();
let reopened = store.open_run(&test_run_id("run-2")).await.unwrap();
let read = reopened.read_blob(&shared_blob_id).await.unwrap();
let read = reopened.read_blob(&shared_blob_hash).await.unwrap();
assert_eq!(read.as_deref(), Some(shared_blob.as_slice()));
}
@ -851,7 +850,7 @@ mod tests {
let run = store.create_run(&test_run_id("run-1")).await.unwrap();
append_created(&run, "run-1", dt("2026-03-27T12:00:00Z")).await;
let blob = br#"{"summary":"readable"}"#;
let blob_id = run.write_blob(blob).await.unwrap();
let blob_hash = run.write_blob(blob).await.unwrap();
// Evict the cached writer so the reader is built through the real
// `open_run_reader` construction path, not a clone of the writer.
@ -859,11 +858,9 @@ mod tests {
let reader = store.open_run_reader(&test_run_id("run-1")).await.unwrap();
assert_eq!(
reader.read_blob(&blob_id).await.unwrap().as_deref(),
reader.read_blob(&blob_hash).await.unwrap().as_deref(),
Some(blob.as_slice())
);
assert_eq!(reader.list_blobs().await.unwrap(), vec![blob_id]);
let err = reader.write_blob(b"blocked").await.unwrap_err();
assert!(matches!(err, Error::ReadOnly));

29
lib/components/fabro-store/src/slate/run_store.rs
View file

@ -11,11 +11,11 @@ use tokio::sync::{Mutex, broadcast, mpsc};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::warn;
use super::blob_store::BlobStore;
use super::projection_cache::{CachedRunProjection, RunProjectionCache};
use crate::run_state::{EventProjectionCache, RunProjectionReducer};
use crate::{
Error, EventEnvelope, EventPayload, Result, RunProjection, RunSummaryStore, StageId, keys,
BlobStore, Error, EventEnvelope, EventPayload, Result, RunProjection, RunSummaryStore, StageId,
keys,
};
const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024;
@ -561,12 +561,8 @@ impl RunDatabase {
self.inner.blob_store.write(data).await
}
pub async fn read_blob(&self, id: &BlobHash) -> Result<Option<Bytes>> {
self.inner.blob_store.read(id).await
}
pub async fn list_blobs(&self) -> Result<Vec<BlobHash>> {
self.inner.blob_store.list().await
pub async fn read_blob(&self, blob_hash: &BlobHash) -> Result<Option<Bytes>> {
self.inner.blob_store.read(blob_hash).await
}
pub async fn state(&self) -> Result<RunProjection> {
@ -896,23 +892,6 @@ mod tests {
use crate::{Database, Error, EventPayload, keys};
#[tokio::test]
async fn list_blobs_reads_global_cas_namespace() {
let object_store = Arc::new(InMemory::new());
let store = Database::new(object_store, "", Duration::from_millis(1), None);
let run_id = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap();
let run = store.create_run(&run_id).await.unwrap();
let first_blob = br#"{"a":1}"#;
let second_blob = br#"{"b":2}"#;
let first_id = run.write_blob(first_blob).await.unwrap();
let second_id = run.write_blob(second_blob).await.unwrap();
let mut blob_ids = run.list_blobs().await.unwrap();
blob_ids.sort();
assert_eq!(blob_ids, vec![first_id, second_id]);
}
fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
stage_prompt_payload_for_stage(run_id, idx, node_id, None)
}

19
lib/components/fabro-store/tests/serializable_projection.rs
View file

@ -8,30 +8,23 @@ use fabro_types::{
BilledModelUsage, BilledTokenCounts, Checkpoint, CheckpointRecord, InterviewQuestionRecord,
ParallelBranchResult, QuestionType, RunDiff, RunSandbox, RunSandboxInstance, RunSandboxPlan,
RunSandboxRuntime, RunStatus, SandboxProviderKind, StageCompletion, StageModelUsage,
StageOutcome, StartRecord, WorkflowSettings, first_event_seq, fixtures, test_support,
StageOutcome, StartRecord, first_event_seq, fixtures, test_support,
};
use serde_json::json;
fn sample_run_spec() -> RunSpec {
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: Graph::new("ship"),
graph_source: None,
workflow_slug: Some("demo".to_string()),
automation: None,
graph: Graph::new("ship"),
workflow_slug: Some("demo".to_string()),
source_directory: Some("/tmp/project".to_string()),
labels: HashMap::from([("team".to_string(), "platform".to_string())]),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
git: Some(fabro_types::GitContext {
labels: HashMap::from([("team".to_string(), "platform".to_string())]),
git: Some(fabro_types::GitContext {
origin_url: "https://github.com/fabro-sh/fabro.git".to_string(),
branch: "main".to_string(),
sha: None,
dirty: fabro_types::DirtyStatus::Clean,
}),
fork_source_ref: None,
..test_support::test_run_spec()
}
}

406
lib/components/fabro-workflow-version/src/lib.rs
View file

@ -9,7 +9,9 @@
use std::collections::{BTreeSet, HashMap, VecDeque};
use fabro_config::parse::{SettingsSource, validate_settings_source};
use fabro_config::{EnvironmentDockerfileLayer, EnvironmentImageLayer, SettingsLayer};
use fabro_config::{
EnvironmentDockerfileLayer, EnvironmentImageLayer, RunGoalLayer, SettingsLayer,
};
use fabro_graphviz::parser;
use fabro_template::{
BundleTemplateStore, GraphReference, GraphReferenceError, StaticReferenceError,
@ -17,12 +19,13 @@ use fabro_template::{
validate_static_reference, visit_graph_references,
};
use fabro_types::graph::ReferenceKind;
use fabro_types::settings::InterpString;
use fabro_types::{ManifestPath, WorkflowPath, WorkflowPathParseError, WorkflowVersion};
use thiserror::Error;
mod store;
pub use store::{WorkflowVersionStore, WorkflowVersionStoreError};
pub use store::{LoadedWorkflowVersionClosure, WorkflowVersionStore, WorkflowVersionStoreError};
#[derive(Debug, Error)]
pub enum WorkflowVersionError {
@ -88,8 +91,10 @@ pub struct ValidatedWorkflowVersion(WorkflowVersion);
impl ValidatedWorkflowVersion {
pub fn new(version: WorkflowVersion) -> Result<Self, WorkflowVersionError> {
validate_config(&version)?;
validate_graph_closure(&version)?;
let mut template_roots = TemplateRoots::new();
validate_config(&version, &mut template_roots)?;
validate_graph_closure(&version, &mut template_roots)?;
validate_template_closure(&version, template_roots.sources)?;
Ok(Self(version))
}
@ -104,7 +109,35 @@ impl ValidatedWorkflowVersion {
}
}
fn validate_config(version: &WorkflowVersion) -> Result<(), WorkflowVersionError> {
/// Template sources that anchor static dependency discovery, all rooted at
/// the workflow package root.
struct TemplateRoots {
package_root: ManifestPath,
sources: Vec<TemplateSource>,
}
impl TemplateRoots {
fn new() -> Self {
Self {
package_root: ManifestPath::from_wire(".")
.expect("the template package root must be a valid manifest path"),
sources: Vec::new(),
}
}
fn push(&mut self, path: &WorkflowPath, content: impl Into<String>) {
self.sources.push(TemplateSource::new(
manifest_path(path),
self.package_root.clone(),
content,
));
}
}
fn validate_config(
version: &WorkflowVersion,
template_roots: &mut TemplateRoots,
) -> Result<(), WorkflowVersionError> {
let config_path =
WorkflowPath::new("workflow.toml").expect("the static workflow config path must be valid");
let Some(source) = version.files().get(&config_path) else {
@ -133,9 +166,36 @@ fn validate_config(version: &WorkflowVersion) -> Result<(), WorkflowVersionError
for image in layer.environment_images() {
validate_dockerfile(version, &config_path, image)?;
}
// The run engine inlines the effective goal (file contents included) into
// the entrypoint graph and renders it under the entrypoint's template
// source, so goal includes anchor at the entrypoint for both goal forms.
match layer.run.as_ref().and_then(|run| run.goal.as_ref()) {
Some(RunGoalLayer::Inline(goal)) => {
template_roots.push(version.entrypoint(), unresolved_source(goal));
}
Some(RunGoalLayer::File { file }) => {
let (_, content) = validate_config_file_reference(
version,
&config_path,
ReferenceKind::RunGoalFile,
&unresolved_source(file),
)?;
template_roots.push(version.entrypoint(), content);
}
None => {}
}
Ok(())
}
#[expect(
clippy::disallowed_methods,
reason = "workflow-version validation preserves authored template source for dependency discovery"
)]
fn unresolved_source(value: &InterpString) -> String {
value.as_source()
}
fn validate_dockerfile(
version: &WorkflowVersion,
config_path: &WorkflowPath,
@ -144,20 +204,33 @@ fn validate_dockerfile(
let Some(EnvironmentDockerfileLayer::Path { path }) = image.dockerfile.as_ref() else {
return Ok(());
};
validate_static_reference(path, ReferenceKind::Dockerfile).map_err(|source| {
validate_config_file_reference(version, config_path, ReferenceKind::Dockerfile, path)
.map(|_| ())
}
/// Validate a static file reference in `workflow.toml` and require its target
/// to exist in the version, returning the target path and its content.
fn validate_config_file_reference<'version>(
version: &'version WorkflowVersion,
config_path: &WorkflowPath,
kind: ReferenceKind,
reference: &str,
) -> Result<(WorkflowPath, &'version str), WorkflowVersionError> {
validate_static_reference(reference, kind).map_err(|source| {
WorkflowVersionError::StaticReference {
path: config_path.clone(),
source,
}
})?;
let target = resolve_reference(config_path, ReferenceKind::Dockerfile, path)?;
require_file(version, config_path, ReferenceKind::Dockerfile, target).map(|_| ())
let target = resolve_reference(config_path, kind, reference)?;
let content = require_file(version, config_path, kind, target.clone())?;
Ok((target, content))
}
fn validate_graph_closure(version: &WorkflowVersion) -> Result<(), WorkflowVersionError> {
let template_store = template_store(version);
let template_root = ManifestPath::from_wire(".")
.expect("the template package root must be a valid manifest path");
fn validate_graph_closure(
version: &WorkflowVersion,
template_roots: &mut TemplateRoots,
) -> Result<(), WorkflowVersionError> {
let mut queue = VecDeque::from([version.entrypoint().clone()]);
let mut visited = BTreeSet::new();
let mut child_workflows = BTreeSet::new();
@ -185,10 +258,12 @@ fn validate_graph_closure(version: &WorkflowVersion) -> Result<(), WorkflowVersi
let target = resolve_reference(&path, ReferenceKind::GraphGoalFile, reference)?;
let content =
require_file(version, &path, ReferenceKind::GraphGoalFile, target.clone())?;
validate_template(&target, content, &template_store, &template_root)
template_roots.push(&target, content);
Ok(())
}
GraphReference::GoalInline { content } | GraphReference::InlinePrompt { content } => {
validate_template(&path, content, &template_store, &template_root)
template_roots.push(&path, content);
Ok(())
}
GraphReference::Import { reference } => {
let target = resolve_reference(&path, ReferenceKind::Import, reference)?;
@ -206,7 +281,7 @@ fn validate_graph_closure(version: &WorkflowVersion) -> Result<(), WorkflowVersi
let content =
require_file(version, &path, ReferenceKind::FileInline, target.clone())?;
if key == "prompt" {
validate_template(&target, content, &template_store, &template_root)?;
template_roots.push(&target, content);
}
Ok(())
}
@ -234,24 +309,24 @@ fn validate_graph_closure(version: &WorkflowVersion) -> Result<(), WorkflowVersi
Ok(())
}
fn validate_template(
path: &WorkflowPath,
content: &str,
store: &BundleTemplateStore,
root: &ManifestPath,
fn validate_template_closure(
version: &WorkflowVersion,
roots: Vec<TemplateSource>,
) -> Result<(), WorkflowVersionError> {
let manifest_path = manifest_path(path);
discover_static_dependency_closure(
[TemplateSource::new(manifest_path, root.clone(), content)],
store,
)
.map_err(|source| WorkflowVersionError::Template {
path: path.clone(),
source: Box::new(source),
discover_static_dependency_closure(roots, &template_store(version)).map_err(|source| {
WorkflowVersionError::Template {
path: template_discovery_path(&source),
source: Box::new(source),
}
})?;
Ok(())
}
fn template_discovery_path(error: &TemplateDiscoveryError) -> WorkflowPath {
WorkflowPath::new(error.source_path().to_string())
.expect("template paths sourced from a workflow version must be valid")
}
fn template_store(version: &WorkflowVersion) -> BundleTemplateStore {
BundleTemplateStore::new(
version
@ -300,6 +375,10 @@ fn manifest_path(path: &WorkflowPath) -> ManifestPath {
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use fabro_template::{TemplateDiscoveryError, TemplateLoadError};
use fabro_types::graph::ReferenceKind;
use fabro_types::{BlobHash, WorkflowPath, WorkflowVersion, WorkflowVersionId};
use super::{ValidatedWorkflowVersion, WorkflowVersionError};
@ -332,6 +411,38 @@ mod tests {
)
}
fn version_with_config(
config: impl Into<String>,
extra_files: impl IntoIterator<Item = (&'static str, &'static str)>,
) -> Result<ValidatedWorkflowVersion, WorkflowVersionError> {
let mut files = extra_files
.into_iter()
.map(|(path_value, content)| (path(path_value), content.to_owned()))
.collect::<BTreeMap<_, _>>();
files.insert(path("workflow.fabro"), "digraph W {}".to_owned());
files.insert(path("workflow.toml"), config.into());
ValidatedWorkflowVersion::new(
WorkflowVersion::new(path("workflow.fabro"), files, BTreeMap::default())
.expect("test fixtures must be structurally valid"),
)
}
fn version_with_goal_file(
reference: &str,
) -> Result<ValidatedWorkflowVersion, WorkflowVersionError> {
let reference = serde_json::to_string(reference).unwrap();
let config = format!("_version = 1\n[run.goal]\nfile = {reference}\n");
version_with_config(config, [])
}
fn version_with_inline_goal(
goal: &str,
extra_files: impl IntoIterator<Item = (&'static str, &'static str)>,
) -> Result<ValidatedWorkflowVersion, WorkflowVersionError> {
let goal = serde_json::to_string(goal).unwrap();
version_with_config(format!("_version = 1\n[run]\ngoal = {goal}\n"), extra_files)
}
#[test]
fn validates_imports_templates_file_refs_and_dependencies() {
let version = version_with(
@ -430,6 +541,245 @@ mod tests {
));
}
#[test]
fn rejects_missing_workflow_goal_file() {
let error = version_with_goal_file("prompts/goal.md").unwrap_err();
assert!(matches!(
error,
WorkflowVersionError::MissingFile {
path: source_path,
kind,
target,
}
if source_path == path("workflow.toml")
&& kind == ReferenceKind::RunGoalFile
&& target == path("prompts/goal.md")
));
}
#[test]
fn accepts_inline_workflow_goal_with_static_template_closure() {
let version = version_with_inline_goal(
r#"Review {{ vars.target }} with {{ inputs.mode }} after {{ goal }}. {% include "prompts/shared.md" %}"#,
[("prompts/shared.md", "Use {{ vars.detail }}")],
)
.unwrap();
assert_eq!(version.version().files().len(), 3);
}
#[test]
fn accepts_file_workflow_goal_with_transitive_template_closure() {
// The goal file's own includes anchor at the entrypoint's directory
// (the package root here), not at the goal file's directory; loaded
// dependencies then anchor at their own directories as usual.
let version =
version_with_config("_version = 1\n[run.goal]\nfile = \"prompts/goal.md\"\n", [
("prompts/goal.md", r#"{% include "prompts/partial.md" %}"#),
("prompts/partial.md", r#"{% include "nested/detail.md" %}"#),
("prompts/nested/detail.md", "Use {{ vars.detail }}"),
])
.unwrap();
assert_eq!(version.version().files().len(), 5);
}
#[test]
fn rejects_broken_transitive_includes_under_a_workflow_goal_file() {
// Guards the root push for file goals: without it the goal file is
// never parsed and the broken include below is silently accepted.
let error =
version_with_config("_version = 1\n[run.goal]\nfile = \"prompts/goal.md\"\n", [
("prompts/goal.md", r#"{% include "prompts/partial.md" %}"#),
("prompts/partial.md", r#"{% include "missing.md" %}"#),
])
.unwrap_err();
assert!(matches!(
error,
WorkflowVersionError::Template { path: source_path, source }
if source_path == path("prompts/partial.md")
&& matches!(
source.as_ref(),
TemplateDiscoveryError::Missing { reference, .. } if reference == "missing.md"
)
));
}
#[test]
fn anchors_workflow_goal_includes_at_the_entrypoint() {
let version_with_entrypoint = |goal_include_target: &'static str| {
ValidatedWorkflowVersion::new(
WorkflowVersion::new(
path("graphs/main.fabro"),
BTreeMap::from([
(path("graphs/main.fabro"), "digraph W {}".to_owned()),
(
path("workflow.toml"),
"_version = 1\n[run]\ngoal = \"{% include \\\"shared.md\\\" %}\"\n"
.to_owned(),
),
(path(goal_include_target), "shared".to_owned()),
]),
BTreeMap::default(),
)
.expect("test fixtures must be structurally valid"),
)
};
// The include resolves beside the entrypoint graph, matching where
// the run engine renders the inlined goal.
version_with_entrypoint("graphs/shared.md").unwrap();
let error = version_with_entrypoint("shared.md").unwrap_err();
assert!(matches!(
error,
WorkflowVersionError::Template { path: source_path, source }
if source_path == path("graphs/main.fabro")
&& matches!(
source.as_ref(),
TemplateDiscoveryError::Missing { reference, .. } if reference == "shared.md"
)
));
}
#[test]
fn rejects_non_static_or_nonportable_workflow_goal_file_references() {
for reference in ["{{ vars.NAME }}", "{% include \"goal.md\" %}"] {
let error = version_with_goal_file(reference).unwrap_err();
let WorkflowVersionError::StaticReference {
path: source_path,
source,
} = error
else {
panic!("expected static-reference error for {reference:?}");
};
assert_eq!(source_path, path("workflow.toml"));
assert_eq!(source.kind(), ReferenceKind::RunGoalFile);
}
for reference in [
"",
"/absolute.md",
"../outside.md",
"~/goal.md",
"C:/goal.md",
"prompts\\goal.md",
"prompts//goal.md",
"prompts/",
"prompts/goal\n.md",
] {
let error = version_with_goal_file(reference).unwrap_err();
assert!(
matches!(
&error,
WorkflowVersionError::InvalidReference {
path: source_path,
kind: ReferenceKind::RunGoalFile,
..
} if *source_path == path("workflow.toml")
),
"expected invalid-reference error for {reference:?}, got {error:?}"
);
}
}
#[test]
fn rejects_invalid_workflow_goal_template_closure() {
let missing = version_with_inline_goal(r#"{% include "missing.md" %}"#, []).unwrap_err();
let WorkflowVersionError::Template {
path: source_path,
source,
} = missing
else {
panic!("expected missing template dependency");
};
assert_eq!(source_path, path("workflow.fabro"));
assert!(matches!(
source.as_ref(),
TemplateDiscoveryError::Missing { parent, reference }
if parent.to_string() == "workflow.fabro" && reference == "missing.md"
));
let dynamic = version_with_inline_goal(r"{% include inputs.partial %}", []).unwrap_err();
let WorkflowVersionError::Template { source, .. } = dynamic else {
panic!("expected dynamic template dependency");
};
assert!(matches!(
source.as_ref(),
TemplateDiscoveryError::Dynamic { parent }
if parent.to_string() == "workflow.fabro"
));
let escaping =
version_with_inline_goal(r#"{% include "../outside.md" %}"#, []).unwrap_err();
let WorkflowVersionError::Template { source, .. } = escaping else {
panic!("expected escaping template dependency");
};
assert!(matches!(
source.as_ref(),
TemplateDiscoveryError::Load {
source: TemplateLoadError::EscapesRoot { parent, .. },
..
} if parent.to_string() == "workflow.fabro"
));
}
#[test]
fn validates_all_inline_graph_roots_that_share_the_graph_path() {
let error = version_with(
[(
"workflow.fabro",
r#"digraph W {
graph [goal="valid"]
step [prompt="{% include inputs.partial %}"]
}"#,
)],
[],
)
.unwrap_err();
assert!(matches!(
error,
WorkflowVersionError::Template {
source,
..
} if matches!(source.as_ref(), TemplateDiscoveryError::Dynamic { .. })
));
}
#[test]
fn validates_graph_files_included_from_goal_templates() {
// The graph file's inline prompt anchors a template root at the graph
// path; that root must not shadow the raw graph content when a goal
// template includes the graph file itself.
let error = version_with(
[
(
"workflow.fabro",
r#"digraph W {
graph [goal="@goal.md"]
step [prompt="hello", note="{% include 'missing.md' %}"]
}"#,
),
("goal.md", r#"{% include "workflow.fabro" %}"#),
],
[],
)
.unwrap_err();
assert!(matches!(
error,
WorkflowVersionError::Template { path: source_path, source }
if source_path == path("workflow.fabro")
&& matches!(
source.as_ref(),
TemplateDiscoveryError::Missing { reference, .. } if reference == "missing.md"
)
));
}
#[test]
fn accepts_root_config_and_all_dockerfile_path_sources() {
let version = version_with(

225
lib/components/fabro-workflow-version/src/store.rs
View file

@ -40,6 +40,49 @@ pub enum WorkflowVersionStoreError {
},
}
/// A fully loaded and validated workflow-version dependency graph: the
/// requested root alongside every unique transitive dependency, keyed by
/// canonical content ID.
///
/// Deliberately not `Clone`: a closure owns the full file contents of every
/// version in the graph, so copies should be explicit and deliberate.
#[derive(Debug)]
pub struct LoadedWorkflowVersionClosure {
root_id: WorkflowVersionId,
root: ValidatedWorkflowVersion,
dependencies: BTreeMap<WorkflowVersionId, ValidatedWorkflowVersion>,
}
impl LoadedWorkflowVersionClosure {
#[must_use]
pub fn root_id(&self) -> WorkflowVersionId {
self.root_id
}
#[must_use]
pub fn root(&self) -> &WorkflowVersion {
self.root.version()
}
#[must_use]
pub fn get(&self, id: &WorkflowVersionId) -> Option<&WorkflowVersion> {
if *id == self.root_id {
return Some(self.root.version());
}
self.dependencies
.get(id)
.map(ValidatedWorkflowVersion::version)
}
pub fn versions(&self) -> impl Iterator<Item = (WorkflowVersionId, &WorkflowVersion)> + '_ {
std::iter::once((self.root_id, self.root.version())).chain(
self.dependencies
.iter()
.map(|(id, version)| (*id, version.version())),
)
}
}
/// Content-addressed storage for validated workflow versions.
///
/// `put` only accepts semantically validated versions; `get` re-validates
@ -61,7 +104,7 @@ impl WorkflowVersionStore {
version: &ValidatedWorkflowVersion,
) -> Result<WorkflowVersionId, WorkflowVersionStoreError> {
let canonical = version.version().canonical_bytes()?;
self.validate_dependency_closure(version.version().workflow_dependencies())
self.walk_dependency_closure(version.version().workflow_dependencies(), |_, _| ())
.await?;
self.blobs
.write(&canonical)
@ -77,19 +120,38 @@ impl WorkflowVersionStore {
let Some(version) = self.load_one(id).await? else {
return Ok(None);
};
self.validate_dependency_closure(version.version().workflow_dependencies())
self.walk_dependency_closure(version.version().workflow_dependencies(), |_, _| ())
.await?;
Ok(Some(version))
}
pub async fn get_closure(
&self,
root_id: &WorkflowVersionId,
) -> Result<Option<LoadedWorkflowVersionClosure>, WorkflowVersionStoreError> {
let Some(root) = self.load_one(root_id).await? else {
return Ok(None);
};
let mut dependencies = BTreeMap::new();
self.walk_dependency_closure(root.version().workflow_dependencies(), |id, version| {
dependencies.insert(id, version);
})
.await?;
Ok(Some(LoadedWorkflowVersionClosure {
root_id: *root_id,
root,
dependencies,
}))
}
async fn load_one(
&self,
id: &WorkflowVersionId,
) -> Result<Option<ValidatedWorkflowVersion>, WorkflowVersionStoreError> {
let blob_id = (*id).into();
let blob_hash = (*id).into();
let Some(bytes) = self
.blobs
.read(&blob_id)
.read(&blob_hash)
.await
.map_err(|source| WorkflowVersionStoreError::Storage { source })?
else {
@ -105,9 +167,12 @@ impl WorkflowVersionStore {
Ok(Some(validated))
}
async fn validate_dependency_closure(
/// Walk the transitive dependency closure, validating every dependency
/// and handing each loaded version to `visit` exactly once.
async fn walk_dependency_closure(
&self,
dependencies: &BTreeMap<WorkflowPath, WorkflowVersionId>,
mut visit: impl FnMut(WorkflowVersionId, ValidatedWorkflowVersion),
) -> Result<(), WorkflowVersionStoreError> {
let mut pending = dependencies
.iter()
@ -128,6 +193,7 @@ impl WorkflowVersionStore {
.iter()
.map(|(path, id)| (path.clone(), *id)),
);
visit(id, dependency);
}
Ok(None) => {
return Err(WorkflowVersionStoreError::DependencyNotFound { path, id });
@ -180,6 +246,12 @@ mod tests {
.unwrap()
}
fn version_id(version: &ValidatedWorkflowVersion) -> WorkflowVersionId {
WorkflowVersionId::from(fabro_types::BlobHash::new(
&version.version().canonical_bytes().unwrap(),
))
}
async fn stores() -> (Arc<BlobStore>, WorkflowVersionStore) {
let database = Database::new(
Arc::new(InMemory::new()),
@ -197,12 +269,15 @@ mod tests {
let (blobs, store) = stores().await;
let version = version("digraph W {}", BTreeMap::new());
let expected_bytes = version.version().canonical_bytes().unwrap();
let expected_id = WorkflowVersionId::from(fabro_types::BlobHash::new(&expected_bytes));
let expected_id = version_id(&version);
let id = store.put(&version).await.unwrap();
assert_eq!(id, expected_id);
let blob_id = id.into();
assert_eq!(blobs.read(&blob_id).await.unwrap().unwrap(), expected_bytes);
let blob_hash = id.into();
assert_eq!(
blobs.read(&blob_hash).await.unwrap().unwrap(),
expected_bytes
);
assert_eq!(store.get(&id).await.unwrap(), Some(version));
}
@ -227,16 +302,12 @@ mod tests {
async fn dependency_must_be_stored_first() {
let (blobs, store) = stores().await;
let child = version("digraph Child {}", BTreeMap::new());
let child_id = WorkflowVersionId::from(fabro_types::BlobHash::new(
&child.version().canonical_bytes().unwrap(),
));
let child_id = version_id(&child);
let root = version(
r#"digraph Root { child [stack.child_workflow="child.fabro"] }"#,
BTreeMap::from([(path("child.fabro"), child_id)]),
);
let root_id = WorkflowVersionId::from(fabro_types::BlobHash::new(
&root.version().canonical_bytes().unwrap(),
));
let root_id = version_id(&root);
let error = store.put(&root).await.unwrap_err();
assert!(matches!(
@ -262,9 +333,7 @@ mod tests {
r#"digraph Root { child [stack.child_workflow="child.fabro"] }"#,
BTreeMap::from([(path("child.fabro"), child_id)]),
);
let root_id = WorkflowVersionId::from(fabro_types::BlobHash::new(
&root.version().canonical_bytes().unwrap(),
));
let root_id = version_id(&root);
assert!(matches!(
store.put(&root).await.unwrap_err(),
@ -273,12 +342,132 @@ mod tests {
));
assert!(!blobs.exists(&root_id.into()).await.unwrap());
assert!(matches!(
store.get(&child_id).await.unwrap_err(),
store.get_closure(&child_id).await.unwrap_err(),
WorkflowVersionStoreError::DependencyNotFound { id, .. }
if id == missing_grandchild_id
));
}
#[tokio::test]
async fn get_closure_returns_root_and_transitive_dependencies() {
let (_, store) = stores().await;
let grandchild = version("digraph Grandchild {}", BTreeMap::new());
let grandchild_id = store.put(&grandchild).await.unwrap();
let child = version(
r#"digraph Child { grandchild [stack.child_workflow="grandchild.fabro"] }"#,
BTreeMap::from([(path("grandchild.fabro"), grandchild_id)]),
);
let child_id = store.put(&child).await.unwrap();
let root = version(
r#"digraph Root { child [stack.child_workflow="child.fabro"] }"#,
BTreeMap::from([(path("child.fabro"), child_id)]),
);
let root_id = store.put(&root).await.unwrap();
let closure = store.get_closure(&root_id).await.unwrap().unwrap();
assert_eq!(closure.root_id(), root_id);
assert_eq!(closure.root(), root.version());
assert_eq!(closure.get(&child_id), Some(child.version()));
assert_eq!(closure.get(&grandchild_id), Some(grandchild.version()));
assert_eq!(
closure
.versions()
.map(|(id, version)| (id, version.clone()))
.collect::<BTreeMap<_, _>>(),
BTreeMap::from([
(root_id, root.version().clone()),
(child_id, child.version().clone()),
(grandchild_id, grandchild.version().clone()),
])
);
}
#[tokio::test]
async fn get_closure_deduplicates_a_diamond() {
let (_, store) = stores().await;
let leaf = version("digraph Leaf {}", BTreeMap::new());
let leaf_id = store.put(&leaf).await.unwrap();
let left = version(
r#"digraph Left { leaf [stack.child_workflow="leaf.fabro"] }"#,
BTreeMap::from([(path("leaf.fabro"), leaf_id)]),
);
let left_id = store.put(&left).await.unwrap();
let right = version(
r#"digraph Right { leaf [stack.child_workflow="leaf.fabro"] }"#,
BTreeMap::from([(path("leaf.fabro"), leaf_id)]),
);
let right_id = store.put(&right).await.unwrap();
let root = version(
r#"digraph Root {
left [stack.child_workflow="left.fabro"]
right [stack.child_workflow="right.fabro"]
}"#,
BTreeMap::from([
(path("left.fabro"), left_id),
(path("right.fabro"), right_id),
]),
);
let root_id = store.put(&root).await.unwrap();
let closure = store.get_closure(&root_id).await.unwrap().unwrap();
let ids = closure.versions().map(|(id, _)| id).collect::<Vec<_>>();
assert_eq!(ids.len(), 4);
assert_eq!(ids.iter().filter(|&&id| id == leaf_id).count(), 1);
}
#[tokio::test]
async fn get_closure_preserves_noncanonical_dependency_errors() {
let (blobs, store) = stores().await;
let dependency = version("digraph Dependency {}", BTreeMap::new());
let pretty = serde_json::to_vec_pretty(dependency.version()).unwrap();
let dependency_id = WorkflowVersionId::from(blobs.write(&pretty).await.unwrap());
let root = version(
r#"digraph Root { dependency [stack.child_workflow="dependency.fabro"] }"#,
BTreeMap::from([(path("dependency.fabro"), dependency_id)]),
);
let root_id = WorkflowVersionId::from(
blobs
.write(&root.version().canonical_bytes().unwrap())
.await
.unwrap(),
);
let error = store.get_closure(&root_id).await.unwrap_err();
let WorkflowVersionStoreError::DependencyInvalid { source, .. } = error else {
panic!("expected invalid dependency error");
};
assert!(matches!(
source.as_ref(),
WorkflowVersionStoreError::NonCanonical { id } if *id == dependency_id
));
assert!(matches!(
store.get_closure(&dependency_id).await.unwrap_err(),
WorkflowVersionStoreError::NonCanonical { id } if id == dependency_id
));
}
#[tokio::test]
async fn get_projects_the_same_validated_root_as_get_closure() {
let (_, store) = stores().await;
let child = version("digraph Child {}", BTreeMap::new());
let child_id = store.put(&child).await.unwrap();
let root = version(
r#"digraph Root { child [stack.child_workflow="child.fabro"] }"#,
BTreeMap::from([(path("child.fabro"), child_id)]),
);
let root_id = store.put(&root).await.unwrap();
let closure = store.get_closure(&root_id).await.unwrap().unwrap();
let projected = store.get(&root_id).await.unwrap().unwrap();
assert_eq!(projected.version(), closure.root());
let absent = version_id(&version("digraph Absent {}", BTreeMap::new()));
assert!(store.get_closure(&absent).await.unwrap().is_none());
assert!(store.get(&absent).await.unwrap().is_none());
}
#[tokio::test]
async fn get_rejects_arbitrary_and_noncanonical_blobs() {
let (blobs, store) = stores().await;

1
lib/components/fabro-workflow/Cargo.toml
View file

@ -76,6 +76,7 @@ toml.workspace = true
fabro-vault = { path = "../../foundation/fabro-vault" }
[dev-dependencies]
fabro-auth = { path = "../../foundation/fabro-auth", features = ["test-support"] }
fabro-github = { path = "../fabro-github", features = ["test-support"] }
base64.workspace = true
fabro-acp = { path = "../fabro-acp", features = ["test-support"] }
fabro-workflow = { path = ".", features = ["test-support"] }

176
lib/components/fabro-workflow/src/artifact.rs
View file

@ -26,7 +26,7 @@ const ARTIFACT_POINTER_PREFIX: &str = "file://";
///
/// For each entry in `updates` whose serialized JSON exceeds
/// `BLOB_OFFLOAD_THRESHOLD`, the value is persisted as a blob in `run_store`
/// and replaced with a `"blob://sha256/{blob_id}"` reference.
/// and replaced with a `"blob://sha256/{blob_hash}"` reference.
/// Small values are left untouched.
///
/// `parallel.results` is offloaded at each branch context-update boundary
@ -102,11 +102,11 @@ async fn offload_value(value: &mut Value, run_store: &RunStoreHandle) -> Result<
.map_err(|e| Error::engine_with_source("artifact serialize failed", e))?;
if bytes.len() > BLOB_OFFLOAD_THRESHOLD {
let blob_id = run_store
let blob_hash = run_store
.write_blob(&bytes)
.await
.map_err(|e| Error::engine_with_anyhow("artifact blob write failed", e))?;
*value = Value::String(format_blob_ref(&blob_id));
*value = Value::String(format_blob_ref(&blob_hash));
}
Ok(())
}
@ -204,8 +204,16 @@ pub async fn resolve_outcomes_for_execution(
run_dir: &Path,
) -> Result<HashMap<String, Outcome>> {
let mut resolved = node_outcomes.clone();
let mut locality = SandboxLocality::default();
for outcome in resolved.values_mut() {
resolve_execution_values(&mut outcome.context_updates, run_store, env, run_dir).await?;
resolve_execution_values(
&mut outcome.context_updates,
run_store,
env,
run_dir,
&mut locality,
)
.await?;
}
Ok(resolved)
}
@ -217,7 +225,8 @@ pub async fn resolved_context_snapshot(
run_dir: &Path,
) -> Result<HashMap<String, Value>> {
let mut values = context.snapshot();
resolve_execution_values(&mut values, run_store, env, run_dir).await?;
let mut locality = SandboxLocality::default();
resolve_execution_values(&mut values, run_store, env, run_dir, &mut locality).await?;
Ok(values)
}
@ -232,17 +241,17 @@ pub async fn resolve_text_or_blob_ref(value: &Value, run_store: &RunStoreHandle)
/// blob reference.
///
/// Managed `file://` references are normalized through their content-addressed
/// blob id instead of reading an execution-local path. Ordinary strings and
/// blob hash instead of reading an execution-local path. Ordinary strings and
/// ordinary file references remain unchanged for the caller to validate.
pub(crate) async fn resolve_json_value(value: Value, run_store: &RunStoreHandle) -> Result<Value> {
let blob_id = value.as_str().and_then(|reference| {
let blob_hash = value.as_str().and_then(|reference| {
parse_blob_ref(reference).or_else(|| parse_managed_blob_file_ref(reference))
});
let Some(blob_id) = blob_id else {
let Some(blob_hash) = blob_hash else {
return Ok(value);
};
let bytes = read_required_blob(&blob_id, run_store).await?;
let bytes = read_required_blob(&blob_hash, run_store).await?;
serde_json::from_slice(&bytes)
.map_err(|err| Error::engine_with_source("artifact blob was not valid JSON", err))
}
@ -267,14 +276,14 @@ pub async fn resolve_text_or_blob_ref_str(
current: &str,
run_store: &RunStoreHandle,
) -> Result<String> {
let Some(blob_id) = parse_blob_ref(current) else {
let Some(blob_hash) = parse_blob_ref(current) else {
return Ok(current.to_string());
};
let bytes = run_store
.read_blob(&blob_id)
.read_blob(&blob_hash)
.await
.map_err(|e| Error::engine_with_anyhow("text blob read failed", e))?
.ok_or_else(|| Error::engine(format!("text blob missing: {blob_id}")))?;
.ok_or_else(|| Error::engine(format!("text blob missing: {blob_hash}")))?;
serde_json::from_slice::<String>(&bytes)
.map_err(|e| Error::engine_with_source("text blob was not a JSON string", e))
}
@ -334,8 +343,8 @@ pub async fn sync_artifacts_to_env(
fn normalize_durable_value(value: &mut Value) {
match value {
Value::String(current) => {
if let Some(blob_id) = parse_managed_blob_file_ref(current) {
*current = format_blob_ref(&blob_id);
if let Some(blob_hash) = parse_managed_blob_file_ref(current) {
*current = format_blob_ref(&blob_hash);
}
}
Value::Array(items) => {
@ -357,10 +366,12 @@ fn resolve_execution_values<'a>(
run_store: &'a RunStoreHandle,
env: &'a dyn Sandbox,
run_dir: &'a Path,
locality: &'a mut SandboxLocality,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
for (key, value) in values.iter_mut() {
resolve_execution_value(Some(key.as_str()), value, run_store, env, run_dir).await?;
resolve_execution_value(Some(key.as_str()), value, run_store, env, run_dir, locality)
.await?;
}
Ok(())
})
@ -376,14 +387,16 @@ fn resolve_execution_value<'a>(
run_store: &'a RunStoreHandle,
env: &'a dyn Sandbox,
run_dir: &'a Path,
locality: &'a mut SandboxLocality,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
match value {
Value::String(current) => {
if key.is_some_and(is_text_context_key) {
*current = resolve_text_or_blob_ref_str(current, run_store).await?;
} else if let Some(blob_id) = parse_blob_ref(current) {
*current = materialize_blob_ref(&blob_id, run_store, env, run_dir).await?;
} else if let Some(blob_hash) = parse_blob_ref(current) {
*current =
materialize_blob_ref(&blob_hash, run_store, env, run_dir, locality).await?;
} else if current.starts_with(ARTIFACT_POINTER_PREFIX)
&& parse_managed_blob_file_ref(current).is_none()
{
@ -392,7 +405,7 @@ fn resolve_execution_value<'a>(
}
Value::Array(items) => {
for item in items {
resolve_execution_value(key, item, run_store, env, run_dir).await?;
resolve_execution_value(key, item, run_store, env, run_dir, locality).await?;
}
}
Value::Object(map) => {
@ -402,8 +415,15 @@ fn resolve_execution_value<'a>(
} else {
Some(child_key.as_str())
};
resolve_execution_value(child_context_key, item, run_store, env, run_dir)
.await?;
resolve_execution_value(
child_context_key,
item,
run_store,
env,
run_dir,
locality,
)
.await?;
}
}
Value::Null | Value::Bool(_) | Value::Number(_) => {}
@ -413,17 +433,18 @@ fn resolve_execution_value<'a>(
}
async fn materialize_blob_ref(
blob_id: &BlobHash,
blob_hash: &BlobHash,
run_store: &RunStoreHandle,
env: &dyn Sandbox,
run_dir: &Path,
locality: &mut SandboxLocality,
) -> Result<String> {
// Blobs are content-addressed, so an existing materialized file is always
// current — check before paying for the store read.
if is_local_execution(env, run_dir).await? {
let path = local_materialized_blob_path(run_dir, blob_id);
if locality.is_local(env, run_dir).await? {
let path = local_materialized_blob_path(run_dir, blob_hash);
if !path.exists() {
let bytes = read_required_blob(blob_id, run_store).await?;
let bytes = read_required_blob(blob_hash, run_store).await?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await.map_err(|err| {
Error::Io(format!(
@ -439,13 +460,13 @@ async fn materialize_blob_ref(
return Ok(format!("{ARTIFACT_POINTER_PREFIX}{}", path.display()));
}
let remote_path = format!("{}/.fabro/blobs/{blob_id}.json", env.working_directory());
let remote_path = format!("{}/.fabro/blobs/{blob_hash}.json", env.working_directory());
if !env
.file_exists(&remote_path)
.await
.map_err(|e| Error::engine_with_source("failed to check blob existence", e))?
{
let bytes = read_required_blob(blob_id, run_store).await?;
let bytes = read_required_blob(blob_hash, run_store).await?;
let content = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::engine_with_source("artifact blob was not valid UTF-8 JSON", e))?;
env.write_file(&remote_path, &content).await.map_err(|e| {
@ -457,14 +478,14 @@ async fn materialize_blob_ref(
}
async fn read_required_blob(
blob_id: &BlobHash,
blob_hash: &BlobHash,
run_store: &RunStoreHandle,
) -> Result<bytes::Bytes> {
run_store
.read_blob(blob_id)
.read_blob(blob_hash)
.await
.map_err(|e| Error::engine_with_anyhow("artifact blob read failed", e))?
.ok_or_else(|| Error::engine(format!("artifact blob missing: {blob_id}")))
.ok_or_else(|| Error::engine(format!("artifact blob missing: {blob_hash}")))
}
async fn resolve_explicit_file_ref(value: &str, env: &dyn Sandbox) -> Result<String> {
@ -502,17 +523,33 @@ async fn resolve_explicit_file_ref(value: &str, env: &dyn Sandbox) -> Result<Str
Ok(format!("{ARTIFACT_POINTER_PREFIX}{remote_path}"))
}
async fn is_local_execution(env: &dyn Sandbox, run_dir: &Path) -> Result<bool> {
env.file_exists(&run_dir.to_string_lossy())
.await
.map_err(|e| Error::engine_with_source("failed to inspect sandbox locality", e))
/// Memoized sandbox locality for one resolution pass. The sandbox and run
/// directory are invariant across a pass, so the (possibly remote) probe is
/// paid at most once instead of once per blob reference.
#[derive(Default)]
struct SandboxLocality {
cached: Option<bool>,
}
fn local_materialized_blob_path(run_dir: &Path, blob_id: &BlobHash) -> PathBuf {
impl SandboxLocality {
async fn is_local(&mut self, env: &dyn Sandbox, run_dir: &Path) -> Result<bool> {
if let Some(local) = self.cached {
return Ok(local);
}
let local = env
.file_exists(&run_dir.to_string_lossy())
.await
.map_err(|e| Error::engine_with_source("failed to inspect sandbox locality", e))?;
self.cached = Some(local);
Ok(local)
}
}
fn local_materialized_blob_path(run_dir: &Path, blob_hash: &BlobHash) -> PathBuf {
RunScratch::new(run_dir)
.runtime_dir()
.join("blobs")
.join(format!("{blob_id}.json"))
.join(format!("{blob_hash}.json"))
}
#[cfg(test)]
@ -549,7 +586,7 @@ mod tests {
let large_string = "x".repeat(BLOB_OFFLOAD_THRESHOLD + 1);
let serialized = serde_json::to_vec(&serde_json::json!(large_string.clone())).unwrap();
let expected_blob_id = fabro_types::BlobHash::new(&serialized);
let expected_blob_hash = fabro_types::BlobHash::new(&serialized);
let mut updates = HashMap::new();
updates.insert("response.plan".to_string(), serde_json::json!(large_string));
@ -561,11 +598,11 @@ mod tests {
let pointer = updates.get("response.plan").unwrap();
assert_eq!(
pointer,
&serde_json::json!(fabro_types::format_blob_ref(&expected_blob_id))
&serde_json::json!(fabro_types::format_blob_ref(&expected_blob_hash))
);
let blob = run_store
.read_blob(&expected_blob_id)
.read_blob(&expected_blob_hash)
.await
.unwrap()
.expect("blob should exist");
@ -591,21 +628,21 @@ mod tests {
async fn resolve_json_value_hydrates_blob_and_managed_file_references() {
let run_store = make_run_store("structured-json-resolution").await;
let value = serde_json::json!([{"name": "api"}, {"name": "web"}]);
let blob_id = run_store
let blob_hash = run_store
.write_blob(&serde_json::to_vec(&value).unwrap())
.await
.unwrap();
let handle = run_store.clone().into();
assert_eq!(
resolve_json_value(serde_json::json!(format_blob_ref(&blob_id)), &handle)
resolve_json_value(serde_json::json!(format_blob_ref(&blob_hash)), &handle)
.await
.unwrap(),
value
);
assert_eq!(
resolve_json_value(
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_id}.json")),
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_hash}.json")),
&handle,
)
.await
@ -787,15 +824,43 @@ mod tests {
);
}
#[tokio::test]
async fn resolve_context_probes_sandbox_locality_once_per_pass() {
let run_store = make_run_store("locality-probe-memoization").await;
let first_blob = run_store
.write_blob(&serde_json::to_vec(&serde_json::json!({"a": 1})).unwrap())
.await
.unwrap();
let second_blob = run_store
.write_blob(&serde_json::to_vec(&serde_json::json!({"b": 2})).unwrap())
.await
.unwrap();
let context = Context::new();
context.set("first", fabro_types::format_blob_ref(&first_blob).into());
context.set("second", fabro_types::format_blob_ref(&second_blob).into());
let env = TestSyncEnv::new(true, "/workspace");
let run_dir = tempfile::tempdir().unwrap();
resolved_context_snapshot(&context, &run_store.clone().into(), &env, run_dir.path())
.await
.unwrap();
assert_eq!(
*env.exists_calls.lock().unwrap(),
1,
"sandbox locality should be probed once per resolution pass"
);
}
#[test]
fn normalize_durable_updates_rewrites_managed_blob_file_refs_recursively() {
let blob_id = fabro_types::BlobHash::new(b"hello");
let blob_hash = fabro_types::BlobHash::new(b"hello");
let mut updates = HashMap::from([(
"nested".to_string(),
serde_json::json!({
"items": [
format!("file:///tmp/run/runtime/blobs/{blob_id}.json"),
format!("file:///sandbox/.fabro/blobs/{blob_id}.json"),
format!("file:///tmp/run/runtime/blobs/{blob_hash}.json"),
format!("file:///sandbox/.fabro/blobs/{blob_hash}.json"),
"file:///tmp/report.json",
]
}),
@ -807,8 +872,8 @@ mod tests {
updates["nested"],
serde_json::json!({
"items": [
fabro_types::format_blob_ref(&blob_id),
fabro_types::format_blob_ref(&blob_id),
fabro_types::format_blob_ref(&blob_hash),
fabro_types::format_blob_ref(&blob_hash),
"file:///tmp/report.json",
]
})
@ -870,7 +935,7 @@ mod tests {
#[test]
fn normalize_checkpoint_for_resume_converts_managed_blob_file_refs_and_drops_preamble() {
let blob_id = fabro_types::BlobHash::new(b"managed");
let blob_hash = fabro_types::BlobHash::new(b"managed");
let mut checkpoint = crate::records::Checkpoint {
timestamp: chrono::Utc::now(),
current_node: "work".to_string(),
@ -883,7 +948,7 @@ mod tests {
),
(
"response.work".to_string(),
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_id}.json")),
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_hash}.json")),
),
]),
node_outcomes: HashMap::from([(
@ -891,7 +956,7 @@ mod tests {
crate::outcome::Outcome {
context_updates: HashMap::from([(
"response.work".to_string(),
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_id}.json")),
serde_json::json!(format!("file:///sandbox/.fabro/blobs/{blob_hash}.json")),
)]),
..crate::outcome::Outcome::success()
},
@ -912,14 +977,14 @@ mod tests {
);
assert_eq!(
checkpoint.context_values.get("response.work"),
Some(&serde_json::json!(fabro_types::format_blob_ref(&blob_id)))
Some(&serde_json::json!(fabro_types::format_blob_ref(&blob_hash)))
);
assert_eq!(
checkpoint
.node_outcomes
.get("work")
.and_then(|outcome| outcome.context_updates.get("response.work")),
Some(&serde_json::json!(fabro_types::format_blob_ref(&blob_id)))
Some(&serde_json::json!(fabro_types::format_blob_ref(&blob_hash)))
);
}
@ -928,9 +993,10 @@ mod tests {
use std::sync::Mutex;
struct TestSyncEnv {
accessible: bool,
written: Mutex<Vec<(String, String)>>,
working_dir: String,
accessible: bool,
written: Mutex<Vec<(String, String)>>,
working_dir: String,
exists_calls: Mutex<usize>,
}
impl TestSyncEnv {
@ -939,6 +1005,7 @@ mod tests {
accessible,
written: Mutex::new(Vec::new()),
working_dir: working_dir.to_string(),
exists_calls: Mutex::new(0),
}
}
}
@ -962,6 +1029,7 @@ mod tests {
}
async fn file_exists(&self, _path: &str) -> fabro_sandbox::Result<bool> {
*self.exists_calls.lock().unwrap() += 1;
Ok(self.accessible)
}

17
lib/components/fabro-workflow/src/billing_rollup.rs
View file

@ -126,12 +126,10 @@ pub fn billing_rollup_from_projection(
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use fabro_model::{Catalog, ModelRef, ProviderId};
use fabro_types::{
AttrValue, BilledTokenCounts, Graph, Node, RunProjection, RunSpec, StageCompletion,
StageOutcome, WorkflowSettings, first_event_seq, fixtures, test_support,
StageOutcome, first_event_seq, test_support,
};
use super::billing_rollup_from_projection;
@ -311,19 +309,8 @@ mod tests {
});
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph,
graph_source: None,
workflow_slug: None,
automation: None,
source_directory: None,
labels: HashMap::new(),
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
git: None,
fork_source_ref: None,
..test_support::test_run_spec()
}
}
}

10
lib/components/fabro-workflow/src/command_log.rs
View file

@ -109,14 +109,14 @@ pub async fn read_json_string_blob(
run_store: &RunStoreHandle,
blob_ref: &str,
) -> Result<Option<String>> {
let Some(blob_id) = fabro_types::parse_blob_ref(blob_ref) else {
let Some(blob_hash) = fabro_types::parse_blob_ref(blob_ref) else {
return Ok(None);
};
let bytes = run_store
.read_blob(&blob_id)
.read_blob(&blob_hash)
.await
.map_err(|err| Error::engine_with_anyhow("command log blob read failed", err))?
.ok_or_else(|| Error::engine(format!("command log blob missing: {blob_id}")))?;
.ok_or_else(|| Error::engine(format!("command log blob missing: {blob_hash}")))?;
let text = serde_json::from_slice::<String>(&bytes)
.map_err(|err| Error::engine_with_source("command log blob was not a JSON string", err))?;
Ok(Some(text))
@ -155,9 +155,9 @@ async fn write_json_string_blob(run_store: &RunStoreHandle, text: &str) -> Resul
let value = Value::String(text.to_string());
let bytes = serde_json::to_vec(&value)
.map_err(|err| Error::engine_with_source("command log JSON serialization failed", err))?;
let blob_id = run_store
let blob_hash = run_store
.write_blob(&bytes)
.await
.map_err(|err| Error::engine_with_anyhow("command log blob write failed", err))?;
Ok(format_blob_ref(&blob_id))
Ok(format_blob_ref(&blob_hash))
}

93
lib/components/fabro-workflow/src/error.rs
View file

@ -84,6 +84,8 @@ const TRANSIENT_INFRA_HINTS: &[&str] = &[
"cross-device link",
"invalid cross-device link",
"os error 18",
"state change in progress",
"sandbox stop still in progress",
];
const BUDGET_EXHAUSTED_HINTS: &[&str] = &[
@ -307,6 +309,10 @@ pub enum Error {
message: String,
failure_class: FailureCategory,
exec_output_tail: Option<ExecOutputTail>,
/// Structured context lines appended after the source chain in
/// `causes()` — e.g. one line per push attempt on a publish push
/// failure.
extra_causes: Vec<String>,
#[source]
source: Option<SharedError>,
},
@ -355,6 +361,7 @@ impl Error {
message,
failure_class,
exec_output_tail,
extra_causes: Vec::new(),
source: None,
}
}
@ -366,16 +373,29 @@ impl Error {
message: impl Into<String>,
source: impl Into<anyhow::Error>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
Self::stage_with_source_details(stage, message, source, None, exec_output_tail, Vec::new())
}
fn stage_with_source_details(
stage: ErrorStage,
message: impl Into<String>,
source: impl Into<anyhow::Error>,
failure_class: Option<FailureCategory>,
exec_output_tail: Option<ExecOutputTail>,
extra_causes: Vec<String>,
) -> Self {
let message = message.into();
let source = SharedError::new(source.into());
let failure_class =
classify_failure_reason(&render_with_causes(&message, &collect_chain(&source)));
let failure_class = failure_class.unwrap_or_else(|| {
classify_failure_reason(&render_with_causes(&message, &collect_chain(&source)))
});
Self::Stage {
stage,
message,
failure_class,
exec_output_tail,
extra_causes,
source: Some(source),
}
}
@ -452,12 +472,42 @@ impl Error {
Self::stage_with_source(ErrorStage::Publish, message, source, exec_output_tail)
}
/// Build a publish error with an explicitly determined failure category,
/// for callers that know more than message sniffing can recover — e.g.
/// exhausted push retries whose attempts all classified as transient.
/// `extra_causes` lines land after the source chain in the failure
/// detail (one line per push attempt).
pub fn publish_with_source_and_class(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
failure_class: FailureCategory,
exec_output_tail: Option<ExecOutputTail>,
extra_causes: Vec<String>,
) -> Self {
Self::stage_with_source_details(
ErrorStage::Publish,
message,
source,
Some(failure_class),
exec_output_tail,
extra_causes,
)
}
#[must_use]
pub fn causes(&self) -> Vec<String> {
match self {
Self::Stage { source, .. } => source
.as_ref()
.map_or_else(Vec::new, |source| collect_chain(source)),
Self::Stage {
source,
extra_causes,
..
} => {
let mut causes = source
.as_ref()
.map_or_else(Vec::new, |source| collect_chain(source));
causes.extend(extra_causes.iter().cloned());
causes
}
Self::Template { source, .. } => collect_chain(source),
Self::ScriptInterpolation { source, .. } => collect_chain(source),
Self::Llm(err) => collect_causes(err),
@ -807,6 +857,18 @@ mod tests {
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn engine_error_with_sandbox_state_change_cause_classifies_transient() {
let source = TestOuterError {
message: "Failed to start Daytona sandbox",
source: TestCause("Sandbox state change in progress"),
};
let err = Error::engine_with_source("Pipeline lifecycle operation failed", source);
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
assert!(err.is_retryable());
}
#[test]
fn handler_error_display() {
let err = Error::handler("LLM call failed");
@ -1281,7 +1343,7 @@ mod tests {
#[test]
fn transient_infra_hints_count() {
assert_eq!(TRANSIENT_INFRA_HINTS.len(), 38);
assert_eq!(TRANSIENT_INFRA_HINTS.len(), 40);
}
#[test]
@ -1450,6 +1512,25 @@ mod tests {
);
}
#[test]
fn classify_reason_sandbox_state_change_in_progress() {
assert_eq!(
classify_failure_reason(
"Pipeline lifecycle operation failed: failed to activate sandbox after node \
attempt survey: Failed to start Daytona sandbox: Sandbox state change in progress"
),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_sandbox_stop_still_in_progress() {
assert_eq!(
classify_failure_reason("Daytona sandbox stop still in progress after 120s"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_500() {
assert_eq!(

173
lib/components/fabro-workflow/src/event/convert.rs
View file

@ -23,6 +23,44 @@ fn stage_status_from_string(status: &str) -> StageOutcome {
})
}
/// Project the sandbox layer's runtime push attempts into the durable
/// `git.push` attempt shape.
///
/// This is the only place the runtime attempt record crosses into stored
/// events: the token snapshot flattens into the three flat `token_*` fields
/// (a nested provenance enum never appears in stored events), and the retry
/// classifier's verdict becomes `classified_reason`.
fn git_push_attempt_props(
attempts: &[fabro_sandbox::PushAttempt],
) -> Vec<fabro_types::GitPushAttemptProps> {
attempts
.iter()
.map(|attempt| fabro_types::GitPushAttemptProps {
attempt: attempt.attempt,
started_at: attempt.started_at,
success: attempt.success,
classified_reason: attempt.retry_reason,
exec_output_tail: attempt.exec_output_tail.clone(),
token_generation: attempt.token.map(|token| token.generation),
token_provenance: attempt.token.map(|token| match token.provenance {
fabro_sandbox::TokenProvenance::Minted { .. } => {
fabro_types::GitTokenProvenance::Minted
}
fabro_sandbox::TokenProvenance::Reused { .. } => {
fabro_types::GitTokenProvenance::Reused
}
fabro_sandbox::TokenProvenance::Static => fabro_types::GitTokenProvenance::Static,
}),
token_age_ms: attempt
.token
.and_then(|token| token.age_at(attempt.started_at))
.map(|age| u64::try_from(age.as_millis()).unwrap_or(u64::MAX)),
credential_action: attempt.credential_action,
refresh_error: attempt.refresh_error,
})
.collect()
}
fn event_body_from_event(event: &Event) -> EventBody {
match event {
Event::RunCreated {
@ -36,6 +74,7 @@ fn event_body_from_event(event: &Event) -> EventBody {
automation,
provenance,
manifest_blob,
spec_blob,
git,
fork_source_ref,
retried_from,
@ -54,6 +93,7 @@ fn event_body_from_event(event: &Event) -> EventBody {
automation: automation.clone(),
provenance: provenance.clone(),
manifest_blob: *manifest_blob,
spec_blob: *spec_blob,
git: git.clone(),
fork_source_ref: fork_source_ref.clone(),
retried_from: *retried_from,
@ -520,10 +560,12 @@ fn event_body_from_event(event: &Event) -> EventBody {
branch,
success,
exec_output_tail,
attempts,
} => EventBody::GitPush(fabro_types::GitPushProps {
branch: branch.clone(),
success: *success,
exec_output_tail: exec_output_tail.clone(),
attempts: git_push_attempt_props(attempts),
}),
Event::GitFetch { branch, success } => EventBody::GitFetch(fabro_types::GitFetchProps {
branch: branch.clone(),
@ -2170,12 +2212,142 @@ mod tests {
}
}
/// The `git.push` attempts contract: every runtime attempt fact
/// round-trips through `GitPushAttemptProps`, the token snapshot is
/// flattened to the three flat token fields (a nested provenance enum
/// never appears in stored events), and optional failure fields are
/// omitted when absent.
#[test]
fn git_push_attempts_round_trip_through_the_durable_shape() {
let started_at = Utc::now();
let minted_at = started_at - chrono::Duration::milliseconds(180);
let expires_at = started_at + chrono::Duration::minutes(60);
let runtime_attempts = vec![
fabro_sandbox::PushAttempt {
attempt: 1,
started_at,
success: false,
retry_reason: Some(fabro_sandbox::GitRetryReason::TokenReplication),
exec_output_tail: Some(exec_tail()),
token: Some(fabro_sandbox::TokenSnapshot {
generation: 14,
provenance: fabro_sandbox::TokenProvenance::Minted {
minted_at,
expires_at,
},
}),
credential_action: Some(fabro_sandbox::RemoteCredentialAction::Embedded),
refresh_error: None,
},
// Terminal classified failure with a refresh error: the last
// attempt carries its classification too.
fabro_sandbox::PushAttempt {
attempt: 2,
started_at: started_at + chrono::Duration::seconds(3),
success: false,
retry_reason: Some(fabro_sandbox::GitRetryReason::TransientInfra),
exec_output_tail: Some(exec_tail()),
token: Some(fabro_sandbox::TokenSnapshot {
generation: 14,
provenance: fabro_sandbox::TokenProvenance::Reused {
minted_at,
expires_at,
},
}),
credential_action: Some(fabro_sandbox::RemoteCredentialAction::Unchanged),
refresh_error: Some(fabro_sandbox::RefreshErrorKind::SetUrl),
},
];
let expected_attempts = git_push_attempt_props(&runtime_attempts);
let stored = to_run_event(&fixtures::RUN_1, &Event::GitPush {
branch: "fabro/run/01M0DH033P2XSTHAGVBHG6922F".to_string(),
success: false,
exec_output_tail: Some(exec_tail()),
attempts: runtime_attempts,
});
let json = serde_json::to_value(&stored).unwrap();
let serialized = &json["properties"]["attempts"];
assert_eq!(serialized[0]["attempt"], 1);
assert_eq!(serialized[0]["classified_reason"], "token_replication");
assert_eq!(serialized[0]["token_generation"], 14);
assert_eq!(serialized[0]["token_provenance"], "minted");
assert_eq!(serialized[0]["token_age_ms"], 180);
assert_eq!(serialized[0]["credential_action"], "embedded");
assert!(serialized[0].get("refresh_error").is_none());
assert_eq!(serialized[1]["classified_reason"], "transient_infra");
assert_eq!(serialized[1]["token_provenance"], "reused");
assert_eq!(serialized[1]["refresh_error"], "set_url");
// The provenance enum never nests in stored events.
assert!(serialized[0].get("token").is_none());
let round_tripped: ::fabro_types::RunEvent = serde_json::from_value(json).unwrap();
match round_tripped.body {
EventBody::GitPush(props) => {
assert!(!props.success);
assert_eq!(props.attempts, expected_attempts);
}
other => panic!("expected GitPush body, got {other:?}"),
}
}
#[test]
fn successful_single_attempt_push_omits_failure_fields() {
let attempts = vec![fabro_sandbox::PushAttempt {
attempt: 1,
started_at: Utc::now(),
success: true,
retry_reason: None,
exec_output_tail: None,
token: Some(fabro_sandbox::TokenSnapshot {
generation: 0,
provenance: fabro_sandbox::TokenProvenance::Static,
}),
credential_action: Some(fabro_sandbox::RemoteCredentialAction::Unchanged),
refresh_error: None,
}];
let stored = to_run_event(&fixtures::RUN_1, &Event::GitPush {
branch: "fabro/run/run-1".to_string(),
success: true,
exec_output_tail: None,
attempts,
});
let json = serde_json::to_value(&stored).unwrap();
let attempt = &json["properties"]["attempts"][0];
assert_eq!(attempt["success"], true);
assert_eq!(attempt["token_provenance"], "static");
for absent in [
"classified_reason",
"exec_output_tail",
"token_age_ms",
"refresh_error",
] {
assert!(attempt.get(absent).is_none(), "{absent} should be omitted");
}
}
/// Events stored before attempts were recorded deserialize with the field
/// absent; the pre-existing three fields are untouched.
#[test]
fn stored_git_push_without_attempts_still_deserializes() {
let json = serde_json::json!({
"branch": "fabro/run/old",
"success": true
});
let props: fabro_types::GitPushProps = serde_json::from_value(json).unwrap();
assert!(props.attempts.is_empty());
assert!(props.exec_output_tail.is_none());
}
#[test]
fn git_push_maps_exec_output_tail_to_props() {
let stored = to_run_event(&fixtures::RUN_1, &Event::GitPush {
branch: "refs/heads/run:refs/heads/run".to_string(),
success: false,
exec_output_tail: Some(exec_tail()),
attempts: Vec::new(),
});
match stored.body {
@ -2669,6 +2841,7 @@ mod tests {
automation: Some(automation.clone()),
provenance,
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

9
lib/components/fabro-workflow/src/event/events.rs
View file

@ -41,6 +41,8 @@ pub enum Event {
#[serde(default, skip_serializing_if = "Option::is_none")]
manifest_blob: Option<BlobHash>,
#[serde(default, skip_serializing_if = "Option::is_none")]
spec_blob: Option<BlobHash>,
#[serde(default, skip_serializing_if = "Option::is_none")]
git: Option<GitContext>,
#[serde(default, skip_serializing_if = "Option::is_none")]
fork_source_ref: Option<ForkSourceRef>,
@ -432,6 +434,9 @@ pub enum Event {
success: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
exec_output_tail: Option<fabro_types::ExecOutputTail>,
/// Per-attempt history of the push operation.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
attempts: Vec<fabro_sandbox::PushAttempt>,
},
GitFetch {
branch: String,
@ -1230,14 +1235,16 @@ impl Event {
branch,
success,
exec_output_tail,
attempts,
} => {
if *success {
debug!(branch, "Git push succeeded");
debug!(branch, attempts = attempts.len(), "Git push succeeded");
} else {
let tail =
fabro_types::ExecOutputTail::trace_summary(exec_output_tail.as_ref());
warn!(
branch,
attempts = attempts.len(),
exec_output_tail_present = tail.present,
exec_stdout_tail_bytes = tail.stdout_bytes,
exec_stderr_tail_bytes = tail.stderr_bytes,

1
lib/components/fabro-workflow/src/event/sink.rs
View file

@ -290,6 +290,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

1
lib/components/fabro-workflow/src/git.rs
View file

@ -364,6 +364,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

274
lib/components/fabro-workflow/src/github_token_source.rs
View file

@ -1,274 +0,0 @@
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context as _;
use fabro_github::{GitHubAppCredentials, InstallationToken};
use tokio::sync::Mutex;
use tracing::warn;
const REFRESH_THRESHOLD: Duration = Duration::from_mins(15);
#[async_trait::async_trait]
pub trait IatMinter: Send + Sync {
async fn mint(&self) -> anyhow::Result<InstallationToken>;
}
pub struct AppIatMinter {
creds: GitHubAppCredentials,
http: fabro_http::HttpClient,
owner: String,
repo: String,
api_base: String,
install_url: Option<String>,
permissions: serde_json::Value,
}
impl AppIatMinter {
#[must_use]
pub fn new(
creds: GitHubAppCredentials,
http: fabro_http::HttpClient,
owner: String,
repo: String,
api_base: String,
install_url: Option<String>,
permissions: serde_json::Value,
) -> Self {
Self {
creds,
http,
owner,
repo,
api_base,
install_url,
permissions,
}
}
}
#[async_trait::async_trait]
impl IatMinter for AppIatMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.creds
.mint_installation_token(
&self.http,
&self.owner,
&self.repo,
&self.api_base,
self.permissions.clone(),
self.install_url.as_deref(),
)
.await
}
}
pub struct GitHubTokenSource {
state: SourceState,
}
enum SourceState {
Pat(String),
StaticIat(InstallationToken),
Mintable {
minter: Arc<dyn IatMinter>,
cache: Mutex<Option<InstallationToken>>,
},
}
impl GitHubTokenSource {
#[must_use]
pub fn pat(token: String) -> Self {
Self {
state: SourceState::Pat(token),
}
}
#[must_use]
pub fn static_iat(token: InstallationToken) -> Self {
Self {
state: SourceState::StaticIat(token),
}
}
#[must_use]
pub fn mintable(minter: Arc<dyn IatMinter>) -> Self {
Self {
state: SourceState::Mintable {
minter,
cache: Mutex::new(None),
},
}
}
#[must_use]
pub fn is_refreshable(&self) -> bool {
matches!(self.state, SourceState::Mintable { .. })
}
pub async fn current_token(&self) -> anyhow::Result<String> {
match &self.state {
SourceState::Pat(token) => Ok(token.clone()),
SourceState::StaticIat(token) => token.valid_token().map(str::to_owned),
SourceState::Mintable { minter, cache } => {
let mut cache = cache.lock().await;
let cached_is_fresh = cache
.as_ref()
.is_some_and(|token| !token.near_expiry(REFRESH_THRESHOLD));
if !cached_is_fresh {
match minter.mint().await {
Ok(token) => *cache = Some(token),
Err(err) => {
if let Some(token) = cache.as_ref() {
if let Ok(value) = token.valid_token() {
warn!(
error = %err,
"GitHub installation token refresh failed; using cached token"
);
return Ok(value.to_owned());
}
}
return Err(err)
.context("failed to mint GitHub installation access token");
}
}
}
let token = cache
.as_ref()
.ok_or_else(|| anyhow::anyhow!("mintable token source has no cached token"))?;
token.valid_token().map(str::to_owned)
}
}
}
}
#[cfg(test)]
mod tests {
use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::anyhow;
use super::*;
enum MintAction {
Token(&'static str, chrono::DateTime<chrono::Utc>),
Error(&'static str),
}
struct MockMinter {
calls: AtomicUsize,
script: Mutex<VecDeque<MintAction>>,
}
impl MockMinter {
fn new(script: Vec<MintAction>) -> Self {
Self {
calls: AtomicUsize::new(0),
script: Mutex::new(script.into()),
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
#[async_trait::async_trait]
impl IatMinter for MockMinter {
async fn mint(&self) -> anyhow::Result<InstallationToken> {
self.calls.fetch_add(1, Ordering::SeqCst);
match self.script.lock().await.pop_front().expect("mint script") {
MintAction::Token(token, expires_at) => Ok(InstallationToken {
token: token.to_string(),
expires_at,
}),
MintAction::Error(message) => Err(anyhow!(message)),
}
}
}
#[tokio::test]
async fn pat_returns_same_token_without_minting() {
let source = GitHubTokenSource::pat("ghp_pat".to_string());
assert_eq!(source.current_token().await.unwrap(), "ghp_pat");
assert_eq!(source.current_token().await.unwrap(), "ghp_pat");
assert!(!source.is_refreshable());
}
#[tokio::test]
async fn static_iat_returns_valid_token_and_rejects_expired_token() {
let valid = GitHubTokenSource::static_iat(InstallationToken {
token: "ghs_valid".to_string(),
expires_at: chrono::Utc::now() + chrono::Duration::minutes(30),
});
assert_eq!(valid.current_token().await.unwrap(), "ghs_valid");
assert!(!valid.is_refreshable());
let expired = GitHubTokenSource::static_iat(InstallationToken {
token: "ghs_expired".to_string(),
expires_at: chrono::Utc::now() - chrono::Duration::seconds(1),
});
assert!(expired.current_token().await.is_err());
}
#[tokio::test]
async fn mintable_reuses_cached_token_until_refresh_threshold() {
let minter = Arc::new(MockMinter::new(vec![MintAction::Token(
"ghs_cached",
chrono::Utc::now() + chrono::Duration::minutes(30),
)]));
let source = GitHubTokenSource::mintable(minter.clone());
assert!(source.is_refreshable());
assert_eq!(source.current_token().await.unwrap(), "ghs_cached");
assert_eq!(source.current_token().await.unwrap(), "ghs_cached");
assert_eq!(minter.calls(), 1);
}
#[tokio::test]
async fn mintable_refreshes_cached_token_near_expiry() {
let minter = Arc::new(MockMinter::new(vec![
MintAction::Token(
"ghs_first",
chrono::Utc::now() + chrono::Duration::minutes(10),
),
MintAction::Token(
"ghs_second",
chrono::Utc::now() + chrono::Duration::minutes(30),
),
]));
let source = GitHubTokenSource::mintable(minter.clone());
assert_eq!(source.current_token().await.unwrap(), "ghs_first");
assert_eq!(source.current_token().await.unwrap(), "ghs_second");
assert_eq!(minter.calls(), 2);
}
#[tokio::test]
async fn mintable_uses_valid_cached_token_when_refresh_fails() {
let minter = Arc::new(MockMinter::new(vec![
MintAction::Token(
"ghs_cached",
chrono::Utc::now() + chrono::Duration::minutes(10),
),
MintAction::Error("mint failed"),
]));
let source = GitHubTokenSource::mintable(minter.clone());
assert_eq!(source.current_token().await.unwrap(), "ghs_cached");
assert_eq!(source.current_token().await.unwrap(), "ghs_cached");
assert_eq!(minter.calls(), 2);
}
#[tokio::test]
async fn mintable_errors_when_no_cached_token_can_cover_mint_failure() {
let minter = Arc::new(MockMinter::new(vec![MintAction::Error("mint failed")]));
let source = GitHubTokenSource::mintable(minter);
let err = format!("{:#}", source.current_token().await.unwrap_err());
assert!(err.contains("mint failed"), "got: {err}");
}
}

1
lib/components/fabro-workflow/src/handler/agent.rs
View file

@ -501,6 +501,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

22
lib/components/fabro-workflow/src/handler/command.rs
View file

@ -374,6 +374,7 @@ mod tests {
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
},
@ -390,16 +391,19 @@ mod tests {
}
async fn write_blob(&self, data: &[u8]) -> anyhow::Result<fabro_types::BlobHash> {
let blob_id = fabro_types::BlobHash::new(data);
let blob_hash = fabro_types::BlobHash::new(data);
self.blobs
.lock()
.await
.insert(blob_id, Bytes::copy_from_slice(data));
Ok(blob_id)
.insert(blob_hash, Bytes::copy_from_slice(data));
Ok(blob_hash)
}
async fn read_blob(&self, id: &fabro_types::BlobHash) -> anyhow::Result<Option<Bytes>> {
Ok(self.blobs.lock().await.get(id).cloned())
async fn read_blob(
&self,
blob_hash: &fabro_types::BlobHash,
) -> anyhow::Result<Option<Bytes>> {
Ok(self.blobs.lock().await.get(blob_hash).cloned())
}
async fn read_run_log(&self) -> anyhow::Result<Option<Vec<u8>>> {
@ -471,6 +475,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -1690,7 +1695,7 @@ mod tests {
}
#[async_trait::async_trait]
impl crate::github_token_source::IatMinter for RefreshingMinter {
impl fabro_github::test_support::InstallationTokenMinter for RefreshingMinter {
async fn mint(&self) -> anyhow::Result<fabro_github::InstallationToken> {
let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
Ok(fabro_github::InstallationToken {
@ -1831,8 +1836,9 @@ mod tests {
calls: std::sync::atomic::AtomicUsize::new(0),
});
let mut services = make_sandbox_services(spy.clone());
services.github_token = Some(std::sync::Arc::new(
crate::github_token_source::GitHubTokenSource::mintable(minter.clone()),
services.github_token = Some(fabro_github::test_support::installation_token_source(
"owner/repo",
minter.clone(),
));
let handler = CommandHandler;

465
lib/components/fabro-workflow/src/handler/llm/acp.rs
View file

@ -13,6 +13,7 @@ use fabro_acp::{
use fabro_agent::{
AgentEvent, RefreshOutcome, Sandbox, StaticEnvProvider, SteeringItem, ToolEnvProvider,
};
use fabro_github::token_source::REFRESH_MARGIN;
use fabro_graphviz::graph::Node;
use fabro_static::EnvVars;
use fabro_types::{
@ -32,8 +33,13 @@ use crate::handler::NodeTimeoutPolicy;
use crate::steering_hub::{ActiveControlHandle, SteeringHub};
/// Default refresh-ahead interval — comfortably under the ~60-min GitHub App
/// installation-token TTL.
/// installation-token TTL. Used as the loop cadence when a tick reports no
/// managed credentials; ticks that see a real token reschedule from its
/// expiry instead.
const REFRESH_INTERVAL_DEFAULT: Duration = Duration::from_mins(45);
/// Floor for expiry-driven rescheduling, so a token already inside the cache
/// margin cannot pin the loop in a hot cycle.
const REFRESH_RESCHEDULE_FLOOR: Duration = Duration::from_secs(30);
/// Upper bound on a single push-credential refresh (token mint + `git remote
/// set-url` exec). The turn-entry refresh runs before the ACP process spawns
/// and the ACP node uses `NodeTimeoutPolicy::HandlerManaged`, so without this
@ -100,19 +106,39 @@ fn push_cred_refresh_interval() -> Option<Duration> {
)
}
/// Background loop that re-mints the sandbox's push credentials every
/// `interval` for the duration of one ACP turn, so a single turn that outlives
/// the installation-token TTL still pushes with a fresh token. Bounded by
/// `cancel` (the drop-guard cancels it at turn end). A failed or timed-out tick
/// retries after a shorter delay so a transient error does not leave a
/// longer-than-interval window with an expired token.
/// Delay until the next refresh-ahead tick after a successful refresh.
///
/// With a cached token source, a fixed interval is unsafe: a tick landing
/// just outside the cache margin returns a reused token, and a fixed
/// 45-minute sleep would leave the embedded token expired until the next
/// tick. Schedule from the token's own `expires_at` instead: wake when the
/// cache margin opens, so that tick re-mints. `None` disables the loop —
/// static credentials cannot be re-minted by waiting.
fn next_refresh_delay(outcome: &RefreshOutcome) -> Option<Duration> {
let token = outcome.token()?;
let expires_at = token.expires_at()?;
let margin = chrono::Duration::from_std(REFRESH_MARGIN).unwrap_or(chrono::Duration::MAX);
let until_margin = ((expires_at - margin) - chrono::Utc::now())
.to_std()
.unwrap_or(Duration::ZERO);
Some(until_margin.max(REFRESH_RESCHEDULE_FLOOR))
}
/// Background loop that keeps the sandbox's push credentials fresh for the
/// duration of one ACP turn, so a single turn that outlives the
/// installation-token TTL still pushes with a fresh token. Bounded by
/// `cancel` (the drop-guard cancels it at turn end). Each successful tick
/// reschedules from the embedded token's expiry ([`next_refresh_delay`]); a
/// failed or timed-out tick retries after a shorter delay so a transient
/// error does not leave a longer-than-interval window with an expired token.
async fn refresh_ahead_loop(
sandbox: Arc<dyn Sandbox>,
cancel: CancellationToken,
interval: Duration,
initial_delay: Duration,
) {
let retry_delay = interval.min(Duration::from_mins(1));
let mut delay = interval;
let mut delay = initial_delay;
loop {
tokio::select! {
() = cancel.cancelled() => break,
@ -120,19 +146,34 @@ async fn refresh_ahead_loop(
match timeout(REFRESH_MINT_TIMEOUT, sandbox.refresh_push_credentials())
.await
{
Ok(Ok(RefreshOutcome::Refreshed)) => {
tracing::info!(
interval_secs = interval.as_secs(),
"refresh-ahead re-minted push credentials mid-turn"
);
delay = interval;
}
Ok(Ok(RefreshOutcome::Skipped)) => {
tracing::debug!(
interval_secs = interval.as_secs(),
"refresh-ahead tick: no managed push credentials to refresh"
);
delay = interval;
Ok(Ok(outcome)) => {
match outcome {
RefreshOutcome::Embedded(token) => {
tracing::info!(
generation = token.generation,
"refresh-ahead re-embedded push credentials mid-turn"
);
}
RefreshOutcome::Unchanged(token) => {
tracing::debug!(
generation = token.generation,
"refresh-ahead tick: embedded push credentials still fresh"
);
}
RefreshOutcome::None => {
tracing::debug!(
"refresh-ahead tick: no managed push credentials to refresh"
);
}
}
if let Some(next) = next_refresh_delay(&outcome) {
delay = next;
} else {
tracing::debug!(
"refresh-ahead loop stopped: static credentials cannot be re-minted"
);
break;
}
}
Ok(Err(e)) => {
tracing::warn!(
@ -267,64 +308,57 @@ impl AgentAcpBackend {
}) as Arc<dyn Fn(String, Option<Principal>) + Send + Sync>
});
// Keep the sandbox's push credentials fresh for the duration of this ACP
// turn so the agent's own `git push` uses a live token instead of the one
// baked into the clone at run start.
//
// Part 2 (turn-entry): re-mint + rewrite the origin URL before the ACP
// process spawns, covering a push early in the turn. Non-fatal and
// timeout-bounded — a stalled mint must neither fail nor hang node entry.
// Part 3 (loop): a background task re-mints every ~45 min so a single turn
// that itself outlives the ~60-min installation-token TTL still pushes
// with a fresh token; a normal sub-interval turn never ticks (the
// drop-guard aborts the task at turn end before the first tick).
//
// FABRO_PUSH_CRED_REFRESH_AHEAD=0 (or false/off/no/empty, case-
// insensitive) disables the WHOLE feature — turn-entry re-mint AND loop —
// so an operator who manages `origin` themselves can opt out of all
// fabro-side origin rewriting. FABRO_PUSH_CRED_REFRESH_INTERVAL_SECONDS
// overrides the loop interval; 0 disables just the loop.
//
// Known limitations tracked as follow-ups (not addressed here): (a)
// resumed/parked runs reconnect the sandbox with no GitHub App creds, so
// refresh no-ops until those creds are threaded through the reconnect
// path; (b) the turn-entry re-mint has no freshness check, so it mints
// once per node entry even when the current token is still fresh; (c) the
// background `git remote set-url` can contend with the agent's own git on
// `.git/config.lock`; (d) parallel ACP branches each run their own loop;
// (e) this refresh lives in the ACP handler only, though the stale-origin
// problem is stage-type-agnostic (native/command stages that push are not
// covered); (f) refresh failures are logged via tracing but not surfaced
// as a RunNotice event on the run stream.
// Refresh before launch for early pushes. Schedule later refreshes from
// token expiry so the loop cannot sleep past the cache margin.
let refresh_enabled = push_cred_refresh_enabled();
if refresh_enabled {
let refresh_interval = refresh_enabled.then(push_cred_refresh_interval).flatten();
let refresh_schedule = if refresh_enabled {
match timeout(REFRESH_MINT_TIMEOUT, sandbox.refresh_push_credentials()).await {
Ok(Ok(RefreshOutcome::Refreshed)) => {
tracing::debug!("refreshed sandbox push credentials at ACP turn entry");
Ok(Ok(outcome)) => {
match outcome {
RefreshOutcome::Embedded(token) => {
tracing::debug!(
generation = token.generation,
"refreshed sandbox push credentials at ACP turn entry"
);
}
RefreshOutcome::Unchanged(token) => {
tracing::debug!(
generation = token.generation,
"sandbox push credentials already fresh at ACP turn entry"
);
}
RefreshOutcome::None => {}
}
refresh_interval.zip(next_refresh_delay(&outcome))
}
Ok(Ok(RefreshOutcome::Skipped)) => {}
Ok(Err(e)) => {
tracing::warn!(
error = %fabro_sandbox::display_for_log(&e),
"node-entry push-credential refresh failed (non-fatal)"
);
refresh_interval
.map(|interval| (interval, interval.min(Duration::from_mins(1))))
}
Err(_elapsed) => {
tracing::warn!(
timeout_secs = REFRESH_MINT_TIMEOUT.as_secs(),
"node-entry push-credential refresh timed out (non-fatal)"
);
refresh_interval
.map(|interval| (interval, interval.min(Duration::from_mins(1))))
}
}
}
let _refresh_ahead_guard: Option<AbortOnDrop> = refresh_enabled
.then(push_cred_refresh_interval)
.flatten()
.map(|interval| {
} else {
None
};
let _refresh_ahead_guard: Option<AbortOnDrop> =
refresh_schedule.map(|(interval, initial_delay)| {
AbortOnDrop(tokio::spawn(refresh_ahead_loop(
Arc::clone(sandbox),
cancel_token.child_token(),
interval,
initial_delay,
)))
});
@ -611,14 +645,18 @@ mod tests {
use fabro_acp::test_support::fake_acp_agent_script;
use fabro_acp::{AcpError, AcpProcessExit};
use fabro_agent::{LocalSandbox, RefreshOutcome, Sandbox, shell_quote};
use fabro_agent::{
LocalSandbox, RefreshOutcome, RemoteCredentialAction, Sandbox, TokenProvenance,
TokenSnapshot, shell_quote,
};
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_sandbox::test_support::MockSandbox;
use fabro_types::{CommandTermination, EventBody, ExecOutputTail};
use tokio_util::sync::CancellationToken;
use super::{
AgentAcpBackend, acp_error_to_workflow, parse_refresh_enabled, parse_refresh_interval,
AgentAcpBackend, REFRESH_RESCHEDULE_FLOOR, acp_error_to_workflow, next_refresh_delay,
parse_refresh_enabled, parse_refresh_interval, refresh_ahead_loop,
};
use crate::context::Context;
use crate::event::Emitter;
@ -671,17 +709,312 @@ mod tests {
}
#[tokio::test]
async fn refresh_reports_skipped_without_managed_credentials() {
// MockSandbox uses the trait default (no GitHub App creds), so refresh is
// a no-op that must report Skipped — the signal the refresh-ahead loop
// relies on to log at debug rather than falsely claim a re-mint.
async fn refresh_reports_no_action_without_managed_credentials() {
// MockSandbox uses the trait default (no GitHub App creds), so refresh
// is a no-op that must report no remote action and no token — the
// signal the refresh-ahead loop relies on to log at debug rather than
// falsely claim a re-embed.
let sandbox = MockSandbox::linux();
assert_eq!(
sandbox.refresh_push_credentials().await.unwrap(),
RefreshOutcome::Skipped
RefreshOutcome::none()
);
}
fn minted_outcome(
action: RemoteCredentialAction,
generation: u64,
minted_ago: chrono::Duration,
expires_in: chrono::Duration,
reused: bool,
) -> RefreshOutcome {
let now = chrono::Utc::now();
let minted_at = now - minted_ago;
let expires_at = now + expires_in;
let provenance = if reused {
TokenProvenance::Reused {
minted_at,
expires_at,
}
} else {
TokenProvenance::Minted {
minted_at,
expires_at,
}
};
let token = TokenSnapshot {
generation,
provenance,
};
match action {
RemoteCredentialAction::Embedded => RefreshOutcome::embedded(token),
RemoteCredentialAction::Unchanged => RefreshOutcome::unchanged(token),
RemoteCredentialAction::None => RefreshOutcome::none(),
}
}
fn static_outcome() -> RefreshOutcome {
RefreshOutcome::unchanged(TokenSnapshot {
generation: 0,
provenance: TokenProvenance::Static,
})
}
#[test]
fn next_refresh_delay_schedules_from_token_expiry_minus_margin() {
let outcome = minted_outcome(
RemoteCredentialAction::Embedded,
1,
chrono::Duration::zero(),
chrono::Duration::minutes(60),
false,
);
let delay = next_refresh_delay(&outcome).unwrap();
// Expiry minus the 10-minute refresh margin: ~50 minutes out.
assert!(delay > Duration::from_mins(49), "{delay:?}");
assert!(delay <= Duration::from_mins(50), "{delay:?}");
}
#[test]
fn next_refresh_delay_floors_when_the_margin_is_already_open() {
let outcome = minted_outcome(
RemoteCredentialAction::Unchanged,
1,
chrono::Duration::minutes(55),
chrono::Duration::minutes(5),
true,
);
assert_eq!(next_refresh_delay(&outcome), Some(REFRESH_RESCHEDULE_FLOOR));
}
#[test]
fn next_refresh_delay_disables_the_loop_for_static_credentials() {
assert_eq!(next_refresh_delay(&static_outcome()), None);
}
#[test]
fn next_refresh_delay_disables_the_loop_without_managed_credentials() {
assert_eq!(next_refresh_delay(&RefreshOutcome::none()), None);
}
/// Sandbox stub whose refresh outcomes are scripted, recording when each
/// refresh tick lands on the (paused) tokio clock.
struct ScriptedRefreshSandbox {
script: Mutex<std::collections::VecDeque<RefreshOutcome>>,
ticks: Mutex<Vec<tokio::time::Instant>>,
}
impl ScriptedRefreshSandbox {
fn new(script: Vec<RefreshOutcome>) -> Arc<Self> {
Arc::new(Self {
script: Mutex::new(script.into()),
ticks: Mutex::new(Vec::new()),
})
}
fn ticks(&self) -> Vec<tokio::time::Instant> {
self.ticks.lock().expect("ticks lock").clone()
}
}
#[async_trait::async_trait]
impl Sandbox for ScriptedRefreshSandbox {
async fn refresh_push_credentials(&self) -> fabro_sandbox::Result<RefreshOutcome> {
self.ticks
.lock()
.expect("ticks lock")
.push(tokio::time::Instant::now());
Ok(self
.script
.lock()
.expect("script lock")
.pop_front()
.expect("refresh script exhausted"))
}
async fn read_file_bytes(&self, _path: &str) -> fabro_sandbox::Result<Vec<u8>> {
unimplemented!("refresh loop only calls refresh_push_credentials")
}
async fn write_file(&self, _path: &str, _content: &str) -> fabro_sandbox::Result<()> {
unimplemented!()
}
async fn delete_file(&self, _path: &str) -> fabro_sandbox::Result<()> {
unimplemented!()
}
async fn file_exists(&self, _path: &str) -> fabro_sandbox::Result<bool> {
unimplemented!()
}
async fn list_directory(
&self,
_path: &str,
_depth: Option<usize>,
) -> fabro_sandbox::Result<Vec<fabro_sandbox::DirEntry>> {
unimplemented!()
}
async fn exec_command(
&self,
_command: &str,
_timeout_ms: u64,
_working_dir: Option<&str>,
_env_vars: Option<&HashMap<String, String>>,
_cancel_token: Option<CancellationToken>,
) -> fabro_sandbox::Result<fabro_sandbox::ExecResult> {
unimplemented!()
}
async fn grep(
&self,
_pattern: &str,
_path: &str,
_options: &fabro_sandbox::GrepOptions,
) -> fabro_sandbox::Result<Vec<String>> {
unimplemented!()
}
async fn download_file_to_local(
&self,
_remote_path: &str,
_local_path: &std::path::Path,
) -> fabro_sandbox::Result<()> {
unimplemented!()
}
async fn upload_file_from_local(
&self,
_local_path: &std::path::Path,
_remote_path: &str,
) -> fabro_sandbox::Result<()> {
unimplemented!()
}
async fn initialize(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn cleanup(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
fn working_directory(&self) -> &str {
"/workspace"
}
fn platform(&self) -> &str {
"linux"
}
fn os_version(&self) -> String {
"linux".to_string()
}
}
/// Long-turn timeline: the clone/turn-entry mint happened at minute 0 with
/// a 60-minute TTL. The loop's first tick at minute 45 sees the cached
/// token reused with ~15 minutes left and must NOT sleep another fixed 45
/// minutes (that would cross expiry at minute 60) — it reschedules for the
/// margin opening (~5 minutes out). That margin-crossing tick re-mints and
/// reschedules from the fresh token's expiry (~50 minutes out).
#[tokio::test(start_paused = true)]
async fn refresh_ahead_reschedules_from_token_expiry_across_a_long_turn() {
let interval = Duration::from_mins(45);
let sandbox = ScriptedRefreshSandbox::new(vec![
// Minute 45: cache still fresh (expires minute 60, margin opens
// minute 50).
minted_outcome(
RemoteCredentialAction::Unchanged,
1,
chrono::Duration::minutes(45),
chrono::Duration::minutes(15),
true,
),
// Minute ~50: margin open → the source minted generation 2.
minted_outcome(
RemoteCredentialAction::Embedded,
2,
chrono::Duration::zero(),
chrono::Duration::minutes(60),
false,
),
// Minute ~100: generation 2 still fresh.
minted_outcome(
RemoteCredentialAction::Unchanged,
2,
chrono::Duration::minutes(50),
chrono::Duration::minutes(10),
true,
),
]);
let cancel = CancellationToken::new();
let start = tokio::time::Instant::now();
let loop_task = tokio::spawn(refresh_ahead_loop(
Arc::clone(&sandbox) as Arc<dyn Sandbox>,
cancel.clone(),
interval,
interval,
));
while sandbox.ticks().len() < 3 {
tokio::time::sleep(Duration::from_secs(1)).await;
}
cancel.cancel();
loop_task.await.expect("refresh loop should exit cleanly");
let ticks = sandbox.ticks();
assert_eq!(ticks[0] - start, interval, "first tick uses the interval");
// Reused token expiring in 15 minutes → next tick when the 10-minute
// margin opens, ~5 minutes later (never another fixed 45 minutes).
let second_gap = ticks[1] - ticks[0];
assert!(second_gap <= Duration::from_mins(5), "{second_gap:?}");
assert!(second_gap > Duration::from_mins(4), "{second_gap:?}");
// Fresh 60-minute token → next tick ~50 minutes out.
let third_gap = ticks[2] - ticks[1];
assert!(third_gap <= Duration::from_mins(50), "{third_gap:?}");
assert!(third_gap > Duration::from_mins(49), "{third_gap:?}");
}
#[tokio::test(start_paused = true)]
async fn refresh_ahead_honors_the_expiry_based_initial_delay() {
let interval = Duration::from_mins(45);
let entry_outcome = minted_outcome(
RemoteCredentialAction::Unchanged,
1,
chrono::Duration::minutes(45),
chrono::Duration::minutes(15),
true,
);
let initial_delay = next_refresh_delay(&entry_outcome).unwrap();
let sandbox = ScriptedRefreshSandbox::new(vec![minted_outcome(
RemoteCredentialAction::Embedded,
2,
chrono::Duration::zero(),
chrono::Duration::minutes(60),
false,
)]);
let cancel = CancellationToken::new();
let start = tokio::time::Instant::now();
let loop_task = tokio::spawn(refresh_ahead_loop(
Arc::clone(&sandbox) as Arc<dyn Sandbox>,
cancel.clone(),
interval,
initial_delay,
));
while sandbox.ticks().is_empty() {
tokio::time::sleep(Duration::from_secs(1)).await;
}
cancel.cancel();
loop_task.await.expect("refresh loop should exit cleanly");
let first_tick = sandbox.ticks()[0] - start;
assert!(first_tick <= Duration::from_mins(5), "{first_tick:?}");
assert!(first_tick > Duration::from_mins(4), "{first_tick:?}");
}
#[tokio::test]
async fn acp_backend_run_sends_prompt_and_returns_text() {
let tempdir = tempfile::tempdir().unwrap();

5
lib/components/fabro-workflow/src/handler/parallel.rs
View file

@ -956,6 +956,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -1917,7 +1918,7 @@ mod tests {
"name": "large-item",
"body": "x".repeat(101 * 1024)
}]);
let blob_id = run_store
let blob_hash = run_store
.write_blob(&serde_json::to_vec(&items).unwrap())
.await
.unwrap();
@ -1933,7 +1934,7 @@ mod tests {
)));
let (node, graph) = for_each_graph("items", 1);
let context = test_context();
context.set("items", serde_json::json!(format_blob_ref(&blob_id)));
context.set("items", serde_json::json!(format_blob_ref(&blob_hash)));
let outcome = ParallelHandler
.execute(&node, &context, &graph, sandbox_dir.path(), &services)

1
lib/components/fabro-workflow/src/handler/prompt.rs
View file

@ -279,6 +279,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

1
lib/components/fabro-workflow/src/lib.rs
View file

@ -293,7 +293,6 @@ pub mod error;
pub mod event;
pub mod file_resolver;
pub mod git;
pub mod github_token_source;
pub(crate) mod graph;
pub mod handler;
mod hook_context;

1
lib/components/fabro-workflow/src/lifecycle/event.rs
View file

@ -493,6 +493,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
branch: push.branch.clone(),
success: push.success,
exec_output_tail: push.exec_output_tail.clone(),
attempts: push.attempts.clone(),
});
}
}

76
lib/components/fabro-workflow/src/lifecycle/git.rs
View file

@ -18,7 +18,10 @@ use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::lifecycle::event::stage_scope_for;
use crate::outcome::BilledModelUsage;
use crate::run_metadata::{MetadataSnapshot, RunMetadataRuntime, RunMetadataWriterHandle};
use crate::run_metadata::{
MetadataSnapshot, RunMetadataRuntime, RunMetadataWriterHandle,
metadata_push_failure_is_transient,
};
use crate::run_options::RunOptions;
use crate::runtime_store::RunStoreHandle;
use crate::sandbox_git::{
@ -72,18 +75,22 @@ pub(crate) struct PushResult {
pub branch: String,
pub success: bool,
pub exec_output_tail: Option<fabro_types::ExecOutputTail>,
pub attempts: Vec<fabro_sandbox::PushAttempt>,
}
/// Push a run branch to its remote counterpart.
///
/// Owns the refspec convention so the checkpoint push and the terminal publish
/// push cannot drift apart.
/// push cannot drift apart. The caller picks the retry budget: cheap for
/// checkpoint pushes (the next checkpoint re-pushes the same branch anyway),
/// generous for the terminal publish push.
pub(crate) async fn push_run_branch(
sandbox: &dyn fabro_sandbox::Sandbox,
branch: &str,
) -> fabro_sandbox::Result<()> {
plan: &fabro_sandbox::RetryPlan,
) -> Result<fabro_sandbox::PushReport, fabro_sandbox::PushError> {
sandbox
.git_push_ref(&format!("refs/heads/{branch}:refs/heads/{branch}"))
.git_push_ref(&format!("refs/heads/{branch}:refs/heads/{branch}"), plan)
.await
}
@ -117,7 +124,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
"git lifecycle mutex should not be poisoned: no code panics while holding this lock",
) = None;
if let Some(meta_branch) = self.metadata_branch().map(str::to_string) {
if self.metadata_writer.is_none() || self.metadata_runtime.metadata_degraded() {
if self.metadata_writer.is_none() || self.metadata_runtime.metadata_suspended() {
return Ok(());
}
let phase = MetadataSnapshotPhase::Init;
@ -154,6 +161,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
false,
);
}
},
@ -174,6 +182,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
false,
);
}
}
@ -208,7 +217,9 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
None,
);
let shadow_sha = if let Some(meta_branch) = self.metadata_branch().map(str::to_string) {
if self.metadata_writer.is_none() || self.metadata_runtime.metadata_degraded() {
if self.metadata_writer.is_none()
|| self.metadata_runtime.metadata_checkpoint_suspended()
{
None
} else {
let phase = MetadataSnapshotPhase::Checkpoint;
@ -253,6 +264,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
false,
);
None
}
@ -276,6 +288,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
false,
);
None
}
@ -320,30 +333,41 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
.as_ref()
.and_then(|g| g.run_branch.as_ref())
{
let (push_ok, exec_output_tail) =
match push_run_branch(self.sandbox.as_ref(), branch).await {
Ok(()) => (true, None),
Err(err) => {
let plan = fabro_sandbox::RetryPlan::checkpoint_push();
let (push_ok, exec_output_tail, attempts) =
match push_run_branch(self.sandbox.as_ref(), branch, &plan).await {
Ok(report) => {
self.sandbox_git.record_successful_push();
(true, None, report.attempts)
}
Err(push_error) => {
let exec_output_tail =
fabro_sandbox::default_redacted_output_tail(&err);
fabro_sandbox::default_redacted_output_tail(
&push_error.error,
);
tracing::warn!(
branch = %branch,
error = %fabro_sandbox::display_for_log(&err),
attempts = push_error.report.attempts.len(),
error = %fabro_sandbox::display_for_log(&push_error.error),
"git push from run lifecycle failed"
);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::GitPushFailed,
format!("Failed to push run branch {branch}: {err}"),
format!(
"Failed to push run branch {branch}: {}",
push_error.error
),
exec_output_tail.clone(),
);
(false, exec_output_tail)
(false, exec_output_tail, push_error.report.attempts)
}
};
git_result.push_results.push(PushResult {
branch: branch.clone(),
success: push_ok,
exec_output_tail,
attempts,
});
}
}
@ -452,7 +476,10 @@ impl GitLifecycle {
message: &str,
scope: Option<&StageScope>,
) -> Option<String> {
if self.metadata_runtime.metadata_degraded() {
if self.metadata_runtime.metadata_suspended()
|| (phase == MetadataSnapshotPhase::Checkpoint
&& self.metadata_runtime.metadata_checkpoint_suspended())
{
return None;
}
let writer = self.metadata_writer.as_ref()?;
@ -477,8 +504,12 @@ impl GitLifecycle {
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataPushFailed,
message,
metadata_push_failure_is_transient(detail, snapshot.token.as_ref()),
);
} else {
// One good snapshot ends the degradation; a later
// independent failure warns again.
self.metadata_runtime.clear_metadata_degraded();
self.emit_metadata_snapshot_completed(
phase,
meta_branch,
@ -503,7 +534,11 @@ impl GitLifecycle {
None,
scope,
);
self.emit_metadata_warning(RunNoticeCode::CheckpointMetadataWriteFailed, message);
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
false,
);
None
}
}
@ -588,8 +623,8 @@ impl GitLifecycle {
}
}
fn emit_metadata_warning(&self, code: RunNoticeCode, message: String) {
if self.metadata_runtime.mark_metadata_degraded() {
fn emit_metadata_warning(&self, code: RunNoticeCode, message: String, transient: bool) {
if self.metadata_runtime.mark_metadata_degraded(transient) {
self.emitter.notice(RunNoticeLevel::Warn, code, message);
}
}
@ -750,6 +785,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
@ -1202,7 +1238,7 @@ mod tests {
let repo_dir = tempfile::tempdir().unwrap();
init_git_repo(repo_dir.path());
let runtime = Arc::new(RunMetadataRuntime::new());
runtime.mark_metadata_degraded();
runtime.mark_metadata_degraded(false);
let emitter = Arc::new(Emitter::new(fixtures::RUN_1));
let events = record_events(&emitter);
let lifecycle = git_lifecycle(
@ -1328,7 +1364,7 @@ mod tests {
Ok(BlobHash::new(data))
}
async fn read_blob(&self, _id: &BlobHash) -> Result<Option<Bytes>> {
async fn read_blob(&self, _blob_hash: &BlobHash) -> Result<Option<Bytes>> {
Ok(None)
}

1
lib/components/fabro-workflow/src/operations/archive.rs
View file

@ -227,6 +227,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,

48
lib/components/fabro-workflow/src/operations/create.rs
View file

@ -13,10 +13,11 @@ use std::sync::Arc;
use fabro_config::Storage;
use fabro_graphviz::graph::{AttrValue, Graph};
use fabro_model::{Catalog, ProviderId};
use fabro_store::Database;
use fabro_store::{Database, RunDatabase};
use fabro_template::TemplateContext;
use fabro_types::{
AutomationRef, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance, WorkflowSettings,
AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance,
WorkflowSettings,
};
use fabro_util::json::normalize_json_value;
use tokio::task::spawn_blocking;
@ -470,6 +471,7 @@ pub async fn persist_create_run(
provenance,
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git,
fork_source_ref,
};
@ -516,18 +518,17 @@ async fn persist_created_run(
.create_run(&record.run_id)
.await
.map_err(|err| Error::engine_with_source("failed to create run store", err))?;
let manifest_blob = match submitted_manifest_bytes {
Some(bytes) => Some(run_store.write_blob(bytes).await.map_err(store_error)?),
None => None,
};
let definition_blob = match accepted_definition {
Some(definition) => {
let bytes =
serde_json::to_vec(definition).map_err(|err| Error::engine(err.to_string()))?;
Some(run_store.write_blob(&bytes).await.map_err(store_error)?)
}
None => None,
};
let definition_bytes = accepted_definition
.map(serde_json::to_vec)
.transpose()
.map_err(|err| Error::engine_with_source("failed to serialize run definition", err))?;
let spec_bytes = serde_json::to_vec(record)
.map_err(|err| Error::engine_with_source("failed to serialize run spec", err))?;
let (manifest_blob, definition_blob, spec_blob) = tokio::try_join!(
write_optional_blob(&run_store, submitted_manifest_bytes),
write_optional_blob(&run_store, definition_bytes.as_deref()),
async { run_store.write_blob(&spec_bytes).await.map_err(store_error) },
)?;
let title = explicit_title.unwrap_or_else(|| fabro_types::infer_run_title(record.graph.goal()));
let stored = to_run_event_at(
@ -554,6 +555,7 @@ async fn persist_created_run(
automation: record.automation.clone(),
provenance: record.provenance.clone(),
manifest_blob,
spec_blob: Some(spec_blob),
git: record.git.clone(),
fork_source_ref: record.fork_source_ref.clone(),
retried_from: None,
@ -580,8 +582,22 @@ async fn persist_created_run(
.map_err(store_error)
}
fn store_error(err: impl std::fmt::Display) -> Error {
Error::engine(err.to_string())
async fn write_optional_blob(
run_store: &RunDatabase,
bytes: Option<&[u8]>,
) -> Result<Option<BlobHash>, Error> {
match bytes {
Some(bytes) => run_store
.write_blob(bytes)
.await
.map(Some)
.map_err(store_error),
None => Ok(None),
}
}
fn store_error(err: impl Into<anyhow::Error>) -> Error {
Error::engine_with_source("run store operation failed", err)
}
/// Parse, transform, and validate `dot_source`.

4
lib/components/fabro-workflow/src/operations/fork.rs
View file

@ -162,6 +162,9 @@ async fn persist_forked_run(
automation: spec.automation.clone(),
provenance: spec.provenance.clone(),
manifest_blob: spec.manifest_blob,
// Content-addressed, so the forked run reads the source run's
// unredacted spec bytes through the same id.
spec_blob: spec.spec_blob,
git: spec.git.clone(),
fork_source_ref: spec.fork_source_ref.clone(),
retried_from: None,
@ -381,6 +384,7 @@ mod tests {
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: Some(fabro_types::GitContext {
origin_url: "https://github.com/example/repo.git".to_string(),
branch: "main".to_string(),

5
lib/components/fabro-workflow/src/operations/retry.rs
View file

@ -54,6 +54,7 @@ pub async fn retry_run(
provenance: _,
manifest_blob,
definition_blob,
spec_blob,
git,
fork_source_ref,
} = source.spec;
@ -78,6 +79,9 @@ pub async fn retry_run(
automation,
provenance: input.provenance.clone(),
manifest_blob,
// Blobs are content-addressed, so the retried run reads the source
// run's unredacted spec bytes through the same id.
spec_blob,
git,
fork_source_ref,
retried_from: Some(source_run_id),
@ -185,6 +189,7 @@ mod tests {
automation: None,
provenance: provenance("source-user"),
manifest_blob,
spec_blob: None,
git: Some(git_context()),
fork_source_ref,
retried_from: None,

4
lib/components/fabro-workflow/src/operations/source.rs
View file

@ -109,8 +109,8 @@ pub(crate) fn resolve_workflow(request: ResolveWorkflowInput) -> anyhow::Result<
/// Resolve the `run.goal` override for a direct (non-manifest) workflow
/// run. Reads the file from disk if the goal layer is the `file` variant.
/// Relative paths that survived config load (e.g. env-interpolated ones)
/// are anchored at `working_directory`.
/// Relative paths that survived config load are anchored at
/// `working_directory`.
fn resolve_goal_override(
settings: &WorkflowSettings,
working_directory: &Path,

10
lib/components/fabro-workflow/src/operations/start.rs
View file

@ -359,8 +359,8 @@ impl RunSession {
let git = git_checkpoint_options_from_start(settings, &record.run_id, state.start);
let definition_blob = state.spec.definition_blob;
let accepted_definition = match definition_blob {
Some(blob_id) => {
Some(load_accepted_run_definition(&services.run_store, blob_id).await?)
Some(blob_hash) => {
Some(load_accepted_run_definition(&services.run_store, blob_hash).await?)
}
None => None,
};
@ -572,15 +572,15 @@ fn vault_token_lookup(vault: &Vault, name: &str) -> Option<String> {
async fn load_accepted_run_definition(
run_store: &RunStoreHandle,
blob_id: fabro_types::BlobHash,
blob_hash: fabro_types::BlobHash,
) -> Result<RunDefinition, Error> {
let bytes = run_store
.read_blob(&blob_id)
.read_blob(&blob_hash)
.await
.map_err(|err| Error::engine(err.to_string()))?
.ok_or_else(|| {
Error::engine(format!(
"run definition blob is missing from the run store: {blob_id}"
"run definition blob is missing from the run store: {blob_hash}"
))
})?;
serde_json::from_slice(&bytes).map_err(|err| Error::Parse(err.to_string()))

1
lib/components/fabro-workflow/src/operations/timeline.rs
View file

@ -252,6 +252,7 @@ mod tests {
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
},

6
lib/components/fabro-workflow/src/pipeline/execute/tests.rs
View file

@ -173,6 +173,7 @@ fn persisted_workflow(graph: Graph, source: String, run_dir: &Path, run_id: RunI
provenance: test_support::test_run_provenance(),
manifest_blob: None,
definition_blob: None,
spec_blob: None,
fork_source_ref: None,
},
)
@ -218,6 +219,7 @@ async fn seed_created_and_starting(
automation: None,
provenance: test_support::test_run_provenance(),
manifest_blob: None,
spec_blob: None,
git: run_options.pre_run_git.clone(),
fork_source_ref: run_options.fork_source_ref.clone(),
retried_from: None,
@ -751,11 +753,11 @@ impl HandlerTrait for BlobCommandOutputHandler {
services: &crate::handler::EngineServices,
) -> std::result::Result<Outcome, Error> {
let blob = serde_json::to_vec("routed-ok").unwrap();
let blob_id = services.run.run_store.write_blob(&blob).await.unwrap();
let blob_hash = services.run.run_store.write_blob(&blob).await.unwrap();
let mut outcome = Outcome::success();
outcome.context_updates.insert(
context::keys::COMMAND_OUTPUT.to_string(),
serde_json::json!(format_blob_ref(&blob_id)),
serde_json::json!(format_blob_ref(&blob_hash)),
);
Ok(outcome)
}

Some files were not shown because too many files have changed in this diff Show more