diff --git a/AGENTS.md b/AGENTS.md index 59712966b..acc2f2dec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,18 @@ macOS note: if `cargo nextest run` fails with `Too many open files (os error 24) - Docker is the default runtime sandbox provider from `defaults.toml`. The Fabro process must have a working Docker client environment (`DOCKER_HOST`, socket access, Docker Desktop behavior, TLS settings, groups/permissions, and any remote daemon policy are operator responsibilities). - The packaged compose service mounts `/var/run/docker.sock` so the server can create sibling run containers on the host daemon. This is host-root-equivalent under Docker's security model; only use it in the trusted, single-tenant deployment model described by the sandbox code/docs. - Docker and Daytona are clone-based providers. When a run manifest has a GitHub origin, they clone it into the provider workspace. Present non-GitHub origins fail unless the provider has `skip_clone = true`; absent origins or `skip_clone = true` create an empty workspace without repository files. +- The sandbox layer also accepts an optional exact commit for future admitted + runs. An exact commit always requires a non-empty branch. Docker initializes + an empty repository, shallow-fetches the SHA at the same depth as a branch + clone, and checks it out; Daytona uses its official SDK clone with both + `branch` and `commit_id`. Both providers then point the admitted branch at + the commit and verify HEAD, so the workspace still reports the admitted + branch name. Keep those provider transports distinct, never fall back to a + newer branch HEAD, and do not wire this capability directly from legacy + `GitContext.sha`. The sandbox layer does not verify that the commit is + reachable from the branch; admission owns that check. Current production + callers remain branch-only until the RunIntent admission cutover supplies a + validated branch/SHA pair. ### Release automation - `cargo dev release` — creates the next stable release tag. Use `cargo dev release --nightly` for a nightly prerelease. Use `--dry-run` to print planned commands without mutating git or running Cargo, `--skip-tests` only after running the release-mode smoke yourself, and `--release-date YYYY-MM-DD` or `FABRO_RELEASE_DATE` for deterministic version computation. diff --git a/Cargo.lock b/Cargo.lock index c09095c1d..eda80ee66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,7 +1870,7 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "daytona-api-client" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "reqwest 0.13.2", "reqwest-middleware", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "daytona-sdk" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "daytona-api-client", "daytona-toolbox-client", @@ -1904,13 +1904,15 @@ dependencies = [ [[package]] name = "daytona-toolbox-client" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "reqwest 0.13.2", "reqwest-middleware", "serde", "serde_json", "serde_repr", + "tokio", + "tokio-util", "url", ] @@ -2072,7 +2074,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2199,7 +2201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2664,6 +2666,7 @@ dependencies = [ "fabro-static", "fabro-test", "fabro-types", + "futures", "jsonwebtoken", "serde", "serde_json", @@ -4389,6 +4392,22 @@ dependencies = [ "webpki-roots 1.0.6", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -4441,7 +4460,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -5316,6 +5335,23 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.2.1", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -5390,7 +5426,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6387,7 +6423,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6672,11 +6708,13 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", "mime", "mime_guess", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -6688,6 +6726,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -6861,7 +6900,7 @@ dependencies = [ "errno 0.3.14", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6920,7 +6959,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7444,7 +7483,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno 0.3.14", + "errno 0.2.8", "libc", ] @@ -8034,7 +8073,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8080,7 +8119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8232,6 +8271,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -9133,7 +9182,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4a6437546..ebf4764e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,8 +97,8 @@ twin-openai = { path = "test/twin/openai" } twin-github = { path = "test/twin/github" } tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] } futures-util = "0.3" -daytona-sdk = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "73c9c458dd1a1d096afd3521175637af82afd8d8", package = "daytona-sdk" } -daytona-api-client = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "73c9c458dd1a1d096afd3521175637af82afd8d8", package = "daytona-api-client" } +daytona-sdk = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "be2c7b7272740d47c023cac8abc9f63c1a51a511", package = "daytona-sdk" } +daytona-api-client = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "be2c7b7272740d47c023cac8abc9f63c1a51a511", package = "daytona-api-client" } sentry = { version = "0.35", default-features = false, features = ["backtrace", "contexts", "ureq", "rustls"] } fork = "0.2" exec = "0.3" diff --git a/docs/internal/events-strategy.md b/docs/internal/events-strategy.md index 8059687c3..0d62bcaf6 100644 --- a/docs/internal/events-strategy.md +++ b/docs/internal/events-strategy.md @@ -186,4 +186,8 @@ Do not rebuild or mutate the `RunEvent` in downstream listeners. Any JSONL sink, the run store, and SSE should reflect the same canonical envelope bytes after redaction. +An active workflow treats any run-event sink write failure as fatal. It cancels execution and +attempts to persist `run.failed` through the direct sink path. Persistence-error logs must include +the full source chain so an HTTP status or transport failure remains visible. + `status.json` remains the authoritative completion signal for detached runs. Terminal run status should only be written after all post-run work is finished. diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 546b9d106..89016b90a 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -11434,6 +11434,11 @@ components: type: ["string", "null"] workflow_slug: type: ["string", "null"] + workflow_version_id: + description: Exact immutable root workflow version from which the run was admitted, when applicable. + oneOf: + - $ref: "#/components/schemas/WorkflowVersionId" + - type: "null" automation: oneOf: - $ref: "#/components/schemas/AutomationRef" @@ -14442,6 +14447,17 @@ components: type: object additionalProperties: type: string + additional_repositories: + type: array + description: | + Additional GitHub repositories, beyond the implicit run origin, + that the minted GITHUB_TOKEN must cover. Each entry is a full + `owner/repository` slug; every repository must share one owner + with the run origin. Omitted when empty; settings persisted + before this field existed deserialize to an empty set. + items: + type: string + uniqueItems: true RunGoal: oneOf: @@ -14628,6 +14644,12 @@ components: properties: enabled: type: boolean + depth: + type: integer + format: int32 + minimum: 0 + default: 100 + description: Git history depth. Set to 0 to clone full history. RunBranchSettings: type: object diff --git a/docs/public/changelog/2026-08-21.mdx b/docs/public/changelog/2026-08-21.mdx index 71308d8f3..557fe7346 100644 --- a/docs/public/changelog/2026-08-21.mdx +++ b/docs/public/changelog/2026-08-21.mdx @@ -1,8 +1,24 @@ --- -title: "Venice search backend" +title: "Additional GitHub repositories and Venice search" date: "2026-08-21" --- +## One token for the whole repository set + +A run can now declare additional GitHub repositories that its stages may access through the managed `GITHUB_TOKEN`: + +```toml +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +The run origin stays implicit, and Fabro mints one installation token scoped to the origin plus every declared repository with the shared permission map. Inside stages, `gh` commands, raw GitHub API calls, plain Git over HTTPS, and the common SSH URL spellings (`git@github.com:owner/repo` and `ssh://git@github.com/owner/repo`) all work against the declared set — the SSH forms are transparently rewritten to authenticated HTTPS with no secret placed in Git configuration. + +Every repository must share one owner and be reachable by the origin's GitHub App installation. Preflight resolves each repository's installation, mints the scoped token once, and probes every repository with `git ls-remote`, naming the exact repository when something is not accessible; run initialization enforces the same checks. A declared-but-inaccessible repository fails the run before its first stage. + +Declaring additional repositories requires `contents = "read"` or `contents = "write"`. With `contents = "write"`, any stage can push to any declared repository — declare the smallest set and weakest permissions that work. See [Additional repositories](/integrations/github#additional-repositories) for details, including layering rules and `GH_TOKEN` precedence. + ## Venice search backend for `web_search` The built-in `web_search` tool now supports Venice as an automatic alternative to direct Brave Search. Fabro uses `BRAVE_SEARCH_API_KEY` when present. Otherwise it uses `VENICE_API_KEY` with Venice's Brave search engine. If neither key is present, the tool is not registered. Failed calls do not fall back between providers. diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index 9bdeadce1..d04232b55 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -59,13 +59,18 @@ Fabro performs this selection once when creating a run and persists the chosen p | `gemini-3.1-flash-lite` | gemini | `gemini-flash-lite`, `gemini-3.1-flash-lite-preview` | 1M | $0.25 / $1.50 | 200 tok/s | | `kimi-k2.5` | moonshot | | 262K | $0.60 / $3.00 | 50 tok/s | | `kimi-k3` | moonshot | `kimi` | 1M | $3.00 / $15.00 | n/a | +| `kimi-k3-fast` | venice | `kimi-fast` | 1M | $4.50 / $22.50 | n/a | | `deepseek-v4-flash` | deepseek | `deepseek`, `deepseek-v4`, `deepseek-flash` | 1,048,576 | $0.14 / $0.28 | n/a | | `deepseek-v4-pro` | deepseek | | 1,048,576 | $0.435 / $0.87 | n/a | +| `grok-4.6` | venice | `grok`, `grok46`, `grok-46` | 500K | $2.27 / $6.80 | n/a | | `laguna-s-2.1` | poolside | `laguna`, `laguna-s` | 1M | $0.10 / $0.20 | n/a | | `laguna-xs-2.1` | poolside | `laguna-xs` | 262K | $0.10 / $0.20 | n/a | | `glm-5.2` | zai | `glm`, `glm5`, `glm52`, `glm5.2` | 1M | $1.40 / $4.40 | n/a | +| `glm-5.3` | venice | `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` | 1M | $1.75 / $5.50 | n/a | | `minimax-m2.5` | minimax | `minimax` | 197K | $0.30 / $1.20 | 45 tok/s | | `mercury-2` | inception | `mercury` | 131K | $0.25 / $0.75 | 1000 tok/s | +| `qwen3.8-max` | venice | `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` | 1M | $2.50 / $7.50 | n/a | +| `qwen3.8-27b` | venice | `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` | 262K | $0.45 / $3.20 | n/a | Each provider requires its own API key. Server-backed workflows read provider credentials from the server vault (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPSEEK_API_KEY`, or `POOLSIDE_API_KEY` set with `fabro secret set` or `fabro provider login`). Standalone SDK/CLI flows can opt into env-backed credential sources explicitly. See the [Quick Start](/getting-started/quick-start) for setup. @@ -169,6 +174,10 @@ Provider `billing_policy` defaults from `adapter` and controls usage-cost estima Provider fields in configuration, APIs, and model routing are provider ID strings. Built-in names like `anthropic`, `openai`, and `gemini` still work, but custom IDs like `proxy` work anywhere a provider ID is accepted. +### Venice + +Fabro ships a built-in [Venice](/integrations/venice) provider with a curated catalog of Venice-hosted Kimi, Grok, GLM, DeepSeek, and Qwen models. Store its API key with `fabro provider login --provider venice`. Pin `provider = "venice"` when a shared model slug must use Venice instead of a higher-priority direct provider. + ### Poolside Fabro ships a built-in [Poolside](/integrations/poolside) provider for Laguna S 2.1 and Laguna XS 2.1 over Poolside's OpenAI-compatible API. Store a direct API key with `fabro provider login --provider poolside`. The same model slugs are also available through the opt-in OpenRouter provider; its vendor-namespaced strings remain provider-only `api_id` values. @@ -232,6 +241,7 @@ When no model or provider is specified, Fabro chooses the default offering on th | `moonshot` | `kimi-k3` | | `poolside` | `laguna-s-2.1` | | `zai` | `glm-5.2` | +| `venice` | `deepseek-v4-flash` | | `minimax` | `minimax-m2.5` | | `inception` | `mercury-2` | diff --git a/docs/public/docs.json b/docs/public/docs.json index 9faa178a8..1082706cb 100644 --- a/docs/public/docs.json +++ b/docs/public/docs.json @@ -97,6 +97,7 @@ "integrations/litellm", "integrations/bedrock", "integrations/deepseek", + "integrations/venice", "integrations/poolside", "integrations/openrouter", "integrations/modal", diff --git a/docs/public/execution/environments.mdx b/docs/public/execution/environments.mdx index 82a72d5ed..a97509bd9 100644 --- a/docs/public/execution/environments.mdx +++ b/docs/public/execution/environments.mdx @@ -258,7 +258,7 @@ memory = "4GB" mode = "block" ``` -Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace. Set `[run.clone] enabled = false` to start with an empty workspace. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. +Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace with a history depth of 100. Set `[run.clone] enabled = false` to start with an empty workspace. Set `[run.clone] depth = 0` to clone full history. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. The image must provide `/bin/bash`; Fabro evaluates every sandbox command with it and has no `sh` fallback. Commands run in a **non-login** shell, so login profiles (`/etc/profile.d/*.sh`, `~/.bash_profile`, and `nvm`/`rbenv`/`sdkman` initializers) are not sourced — put anything they set into the Dockerfile's `ENV` instead. Fabro verifies Bash during initialization and again on resume, and fails with remediation rather than reporting the sandbox ready. diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index d2ce9bd5e..5e7a0b8b0 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -241,10 +241,16 @@ Configure whether clone-based sandboxes clone the run's GitHub origin before exe ```toml title="run.toml" [run.clone] enabled = true +depth = 100 ``` Set `enabled = false` to start Docker and Daytona runs with an empty provider workspace. Use [prepare steps](#runprepare) to clone or create any files the workflow needs. +| Field | Description | +|---|---| +| `enabled` | When `false`, Fabro skips the repository clone. Defaults to `true`. | +| `depth` | Git history depth for Docker and Daytona. Defaults to `100`. Set it to `0` to clone full history. | + ### `[run.run_branch]` Configure Fabro's managed `fabro/run/` checkpoint branch. @@ -363,6 +369,24 @@ Only requested permissions are included. The upper bound is the permission set g This table follows the normal settings precedence order. A higher-precedence layer can set `permissions = {}` to clear inherited permissions and run without a GitHub token. +### `[run.integrations.github].additional_repositories` + +Declare extra GitHub repositories, beyond the implicit run origin, that the minted `GITHUB_TOKEN` must cover. The one `permissions` map applies to the origin and every declared repository. + +```toml title="run.toml" +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +Each entry is a full `owner/repository` slug. Every repository in the effective set must share one owner and be reachable by the origin repository's GitHub App installation. A non-empty list requires `contents = "read"` or `contents = "write"`. Malformed slugs, case-insensitive duplicates, cross-owner sets, and sets larger than 499 entries fail configuration validation with indexed error paths such as `run.integrations.github.additional_repositories[1]`. + +Unlike permissions-only configuration, declared additional repositories are a hard requirement: missing credentials, a missing origin, or an inaccessible declared repository fails preflight and run initialization with the repository named. + +The higher-precedence list replaces the lower one wholesale — no union and no `...` splice — and `additional_repositories = []` explicitly clears an inherited list. `additional_repositories` and `permissions` resolve independently; if layering leaves repositories declared while permissions were cleared, resolution reports the invalid combination instead of dropping either field. + +See [Additional repositories](/integrations/github#additional-repositories) for what works inside stages (`gh`, GitHub API, plain Git over HTTPS and the common SSH spellings) and for the security boundary. + ### `[run.notifications]` Define named notification routes for run events. Slack lifecycle notifications are configured here, not in server config. diff --git a/docs/public/integrations/daytona.mdx b/docs/public/integrations/daytona.mdx index 1a2f25790..828debc0d 100644 --- a/docs/public/integrations/daytona.mdx +++ b/docs/public/integrations/daytona.mdx @@ -141,6 +141,15 @@ provider = "daytona" enabled = false ``` +Daytona clones 100 commits by default. To keep only the newest commit, set a smaller clone depth: + +```toml title="run.toml" +[run.clone] +depth = 1 +``` + +Set `depth = 0` to clone the full repository history. + If the clone fails without GitHub access configured, Fabro suggests running the setup flow: ``` diff --git a/docs/public/integrations/github.mdx b/docs/public/integrations/github.mdx index a08185561..9550c436c 100644 --- a/docs/public/integrations/github.mdx +++ b/docs/public/integrations/github.mdx @@ -62,6 +62,7 @@ When you choose the GitHub App strategy, the CLI opens GitHub with a pre-filled | Emails | Read | Read verified email for OAuth login | | Dependabot alerts | Write | Read and manage repository vulnerability alerts | | Organization projects | Write | Read and update organization Projects V2 | + | Packages | Read | Download private GitHub Packages (e.g. npm registry) with the sandbox `GITHUB_TOKEN` | These permissions are included when Fabro registers a new app. For an existing GitHub App, add the missing permissions in the app's settings, then approve the permission update on each installation before workflows can use them. @@ -228,7 +229,7 @@ For public repositories, the clone works without credentials. The token is still ### GITHUB_TOKEN injection -When any settings layer declares `[run.integrations.github.permissions]`, Fabro prepares a scoped GitHub App token source and exposes it as the `GITHUB_TOKEN` environment variable in sandbox command and agent execution. Agents running inside the sandbox can use this token for GitHub API calls, cloning additional private repos, or pushing to branches. The GitHub CLI (`gh`) reads `GITHUB_TOKEN` automatically, so command stages can run `gh pr list`, `gh issue create`, and similar commands without an explicit `gh auth login`. +When any settings layer declares `[run.integrations.github.permissions]`, Fabro prepares a scoped GitHub App token source and exposes it as the `GITHUB_TOKEN` environment variable in sandbox command and agent execution. Agents running inside the sandbox can use this token for GitHub API calls and pushes within the granted permissions. The GitHub CLI (`gh`) reads `GITHUB_TOKEN` automatically, so command stages can run `gh pr list`, `gh issue create`, and similar commands without an explicit `gh auth login`. ```toml title="workflow.toml" [run.integrations.github.permissions] @@ -238,6 +239,46 @@ pull_requests = "write" Only the listed permissions are requested — the token is scoped to the minimum access needed. If the GitHub App isn't configured or the repository lacks an installation, the run logs a warning and continues without the token. +In App mode, the token covers only the run's origin repository unless the run declares [additional repositories](#additional-repositories). Injecting `GITHUB_TOKEN` alone does not make other private repositories reachable. + +### Additional repositories + +A run can declare extra GitHub repositories that its stages may access through the same `GITHUB_TOKEN`: + +```toml title="workflow.toml" +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +The run origin stays implicit — never list it. Each entry is a full `owner/repository` slug (no scheme, host, ref, or extra path component). Fabro mints **one** installation token scoped to the origin plus every declared repository, with the one shared `permissions` map applying to all of them. + +What works against every declared repository, within the granted permissions: + +- **`gh` CLI and raw GitHub API calls** through `GITHUB_TOKEN`. +- **Plain Git over HTTPS** (`git clone https://github.com/owner/repo`), through a secret-free credential helper that reads `$GITHUB_TOKEN` at invocation time. +- **The common SSH spellings** `git@github.com:owner/repo[.git]` and `ssh://git@github.com/owner/repo[.git]`, through per-repository SSH-to-HTTPS rewrites injected into the stage environment. + +Fabro does not clone additional repositories for you; a workflow that needs one on disk adds its own clone step (`git clone https://github.com/owner/repo` or `gh repo clone owner/repo`). + +Requirements and validation: + +- Every repository in the effective set must share **one owner** and be reachable by the origin repository's GitHub App installation, because one App installation covers one account. Cross-owner declarations fail configuration validation; a same-owner repository outside the installation fails preflight and run initialization with the repository named. +- A non-empty `additional_repositories` requires `contents = "read"` or `contents = "write"` in the permission map. +- Malformed slugs, duplicates (repository identity is case-insensitive), and sets larger than 499 entries fail configuration validation with indexed error paths. +- Unlike permissions-only configuration, declared additional repositories are a hard requirement: missing GitHub credentials, a missing origin, or an inaccessible declared repository fails preflight and run initialization instead of continuing without the token. +- Layering: the higher-precedence `additional_repositories` list replaces the lower one wholesale (no union, no `...` splice), and `additional_repositories = []` explicitly clears an inherited list. `permissions` keeps its existing whole-map replacement behavior. If layering leaves repositories declared with permissions cleared, configuration resolution reports the invalid combination. + +Behavior notes: + +- **Token strategy (PAT):** the configured PAT is used as-is. The repository list drives validation and preflight probes, but it cannot narrow the PAT's inherent GitHub scope — App mode remains the least-authority option. +- **`GH_TOKEN` precedence:** `gh` checks `GH_TOKEN` before `GITHUB_TOKEN`. If the resolved run environment defines `GH_TOKEN`, `gh` uses it instead of the managed token; Fabro never sets or removes `GH_TOKEN`, and preflight warns when additional repositories are declared alongside one. +- **SSH rewrites match by prefix.** With `owner/repo` declared, the SSH spelling of `owner/repo-other` is also rewritten to HTTPS. The scoped token is invalid for undeclared repositories at GitHub, so authority is unchanged — but a private undeclared repository fails with a GitHub authorization error instead of a missing-credential or SSH error. + +#### Security boundary + +Workflow authors may name any repository reachable by the server's GitHub App installation; Fabro applies no second server-side repository intersection. The token is scoped server-side to exactly the declared set — a request to an undeclared repository fails at GitHub, and Fabro never mints an unscoped installation-wide token. With `contents = "write"`, **any stage can push to any declared repository**. Declare the smallest repository set and the weakest permissions that work. + Installation Access Tokens are short-lived. Fabro refreshes its own credentials before checkpoint pushes. For ACP/CLI agent turns launched with GitHub App push credentials, Fabro also re-mints the token and rewrites the sandbox's `origin` URL before the ACP process starts, then every 45 minutes for the lifetime of that turn. Refresh failures are logged and do not fail the stage. `FABRO_PUSH_CRED_REFRESH_AHEAD` defaults to enabled; set it to `0`, `false`, `off`, `no`, or an empty value to disable both turn-entry and background refresh. `FABRO_PUSH_CRED_REFRESH_INTERVAL_SECONDS` overrides the background interval, and `0` disables only the background loop. This refresh loop is ACP-specific; command and native/API agent stages do not run it. Reconnected sandboxes for resumed or parked runs currently lack the App credentials needed for ACP refresh, so the refresh is skipped there. diff --git a/docs/public/integrations/venice.mdx b/docs/public/integrations/venice.mdx new file mode 100644 index 000000000..f172ccb34 --- /dev/null +++ b/docs/public/integrations/venice.mdx @@ -0,0 +1,125 @@ +--- +title: "Venice" +description: "Run Kimi, Grok, GLM, DeepSeek, and Qwen models through Venice" +--- + +[Venice](https://venice.ai/) provides an OpenAI-compatible API for hosted text models. Fabro enables the `venice` provider in its built-in catalog and maps stable Fabro model slugs to Venice's API model IDs. + +## Prerequisites + +- A Venice account +- An inference API key from [venice.ai/settings/api](https://venice.ai/settings/api) +- A running Fabro server + +## Configure credentials + +Store the API key in the target Fabro server vault: + +```bash +fabro provider login --provider venice + +# For a non-default remote server: +fabro provider login --server https://your-fabro.example --provider venice + +# Or set the vault token directly: +fabro secret set VENICE_API_KEY +fabro secret --server https://your-fabro.example set VENICE_API_KEY +``` + +Standalone SDK usage outside a Fabro server can use an env-backed credential source explicitly: + +```bash +export VENICE_API_KEY= +``` + +Fabro sends bearer-authenticated Chat Completions requests to `https://api.venice.ai/api/v1`. + +## Included models + +| Fabro model slug | Venice API ID | Context | Max output | Role and aliases | +|---|---|---:|---:|---| +| `kimi-k3` | `kimi-k3` | 1,000,000 | 131,072 | Alias `kimi` | +| `kimi-k3-fast` | `kimi-k3-fast-api` | 1,000,000 | 131,072 | Alias `kimi-fast` | +| `grok-4.6` | `grok-4-6` | 500,000 | 32,000 | Aliases `grok`, `grok46`, `grok-46` | +| `glm-5.3` | `z-ai-glm-5-3` | 1,000,000 | 131,072 | Aliases `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` | +| `deepseek-v4-flash` | `deepseek-v4-flash-0731` | 1,000,000 | 32,768 | Provider default; aliases `deepseek`, `deepseek-v4`, `deepseek-flash` | +| `deepseek-v4-pro` | `deepseek-v4-pro-0813` | 1,000,000 | 32,768 | Alias `deepseek-pro` | +| `qwen3.8-max` | `qwen-3-8-max` | 1,000,000 | 131,072 | Aliases `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` | +| `qwen3.8-27b` | `qwen-3-8-27b` | 262,144 | 131,072 | Aliases `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` | + +Venice API IDs are also valid provider-scoped selectors. Fabro persists the stable Fabro slug and the selected provider when it creates a run. + +## Select Venice explicitly + +Some Venice models use the same stable slugs as direct providers. An unqualified selector chooses the highest-priority ready provider. For example, `deepseek` can select the direct DeepSeek provider when both API keys are configured. + +Pin Venice when the run must use Venice: + +```bash +fabro model list --provider venice +fabro model test --provider venice --model deepseek-v4-flash --deep +fabro run workflow.fabro --provider venice --model deepseek-v4-flash +``` + +In a workflow stylesheet: + +```dot title="workflow.fabro" +digraph Example { + graph [ + model_stylesheet=" + * { provider: venice; model: deepseek-v4-flash; } + .complex { provider: venice; model: qwen; } + .fast { provider: venice; model: kimi-fast; } + " + ] + + start [shape=Mdiamond, label="Start"] + work [label="Implement", class="complex"] + check [label="Check", class="fast"] + exit [shape=Msquare, label="Exit"] + + start -> work -> check -> exit +} +``` + +The generic Qwen aliases `qwen` and `qwen3.8` select Qwen 3.8 Max. Use a size-specific alias such as `qwen-27b` to select Qwen 3.8 27B. + +## Capabilities and reasoning + +All included models support tool calling and reasoning. Kimi K3, Kimi K3 Fast, Grok 4.6, Qwen 3.8 Max, and Qwen 3.8 27B also accept image input. + +Fabro exposes native reasoning-effort controls only when Venice supports them: + +| Model | Reasoning effort values | +|---|---| +| `grok-4.6` | `low`, `medium`, `high`, `xhigh` | +| `glm-5.3` | `low`, `high`, `max` | +| `deepseek-v4-flash` | `low`, `high`, `max` | +| `qwen3.8-27b` | `low`, `medium`, `xhigh` | + +The other models reason by default but do not expose a Venice reasoning-effort control. Fabro omits sampling parameters for Kimi and DeepSeek because those routes do not use them with their configured reasoning behavior. + +## Pricing and prompt caching + +The built-in catalog uses Venice's published prices per million tokens: + +| Model | Uncached input | Cache hit | Output | +|---|---:|---:|---:| +| `kimi-k3` | $3.75 | $0.375 | $18.75 | +| `kimi-k3-fast` | $4.50 | $0.45 | $22.50 | +| `grok-4.6` | $2.27 | $0.57 | $6.80 | +| `glm-5.3` | $1.75 | $0.325 | $5.50 | +| `deepseek-v4-flash` | $0.175 | $0.035 | $0.35 | +| `deepseek-v4-pro` | $1.65 | $0.165 | $4.95 | +| `qwen3.8-max` | $2.50 | $0.3125 | $7.50 | +| `qwen3.8-27b` | $0.45 | n/a | $3.20 | + +Fabro reports cached input separately when Venice returns cache usage for the selected model. Prices and model availability can change upstream; use `fabro model list --provider venice` to inspect the catalog shipped with your Fabro version and the [Venice model catalog](https://docs.venice.ai/models/overview) for the current upstream service. + +## Troubleshooting + +**"No credential was found for provider 'venice'"** — Store `VENICE_API_KEY` in the server vault with `fabro provider login --provider venice`. Pass `--server` when configuring a remote Fabro server. + +**A shared model used another provider** — Pin Venice with `--provider venice` or `provider: venice` in the workflow stylesheet. Unqualified selectors use provider priority. + +**A Venice API model ID is rejected without a provider** — Use the stable Fabro slug for portable selection, or qualify the API ID with the provider, such as `venice:qwen-3-8-max`. diff --git a/docs/public/reference/user-configuration.mdx b/docs/public/reference/user-configuration.mdx index cf60ea2a4..e68f36896 100644 --- a/docs/public/reference/user-configuration.mdx +++ b/docs/public/reference/user-configuration.mdx @@ -34,7 +34,7 @@ Files that omit `_version` are treated as version `1`. The legacy top-level `ver | Scope | Examples | |---|---| | CLI-only | `[cli.target]`, `[cli.auth]`, `[cli.exec]`, `[cli.output]`, `[cli.updates]`, `[cli.logging]` | -| Shared run defaults | `[run.model]`, `[run.environment]`, `[environments.]`, `[run.checkpoint]`, `[run.inputs]`, `[run.prepare]`, `[run.pull_request]`, `[run.integrations.github.permissions]`, `[run.hooks]`, `[run.agent.mcps]` | +| Shared run defaults | `[run.model]`, `[run.environment]`, `[environments.]`, `[run.checkpoint]`, `[run.inputs]`, `[run.prepare]`, `[run.pull_request]`, `[run.integrations.github]`, `[run.hooks]`, `[run.agent.mcps]` | | Shared LLM catalog | `[llm.providers.]`, provider-scoped `[llm.providers..models.]` offerings, limits, features, controls, and costs | | Server-only | `[server.listen]`, `[server.api]`, `[server.web]`, `[server.auth]`, `[server.storage]`, `[server.artifacts]`, `[server.slatedb]`, `[server.scheduler]`, `[server.logging]`, `[server.integrations]` | diff --git a/lib/apps/fabro-cli/src/commands/install.rs b/lib/apps/fabro-cli/src/commands/install.rs index 973524899..2a3f401b4 100644 --- a/lib/apps/fabro-cli/src/commands/install.rs +++ b/lib/apps/fabro-cli/src/commands/install.rs @@ -950,7 +950,8 @@ fn build_github_app_manifest(app_name: &str, port: u16, web_url: &str) -> serde_ "issues": "write", "emails": "read", "vulnerability_alerts": "write", - "organization_projects": "write" + "organization_projects": "write", + "packages": "read" }, "default_events": [] }) @@ -2682,6 +2683,10 @@ client_id = "client-id" manifest["default_permissions"]["organization_projects"], serde_json::json!("write"), ); + assert_eq!( + manifest["default_permissions"]["packages"], + serde_json::json!("read"), + ); } #[test] diff --git a/lib/apps/fabro-cli/src/commands/run/attach.rs b/lib/apps/fabro-cli/src/commands/run/attach.rs index 7c1cf93b7..bc110029d 100644 --- a/lib/apps/fabro-cli/src/commands/run/attach.rs +++ b/lib/apps/fabro-cli/src/commands/run/attach.rs @@ -843,6 +843,7 @@ mod tests { graph: fabro_types::Graph::new("test"), graph_source: None, workflow_slug: None, + workflow_version_id: None, automation: None, source_directory: None, labels: std::collections::HashMap::default(), diff --git a/lib/apps/fabro-cli/src/commands/run/runner.rs b/lib/apps/fabro-cli/src/commands/run/runner.rs index 713838555..9d5bb5388 100644 --- a/lib/apps/fabro-cli/src/commands/run/runner.rs +++ b/lib/apps/fabro-cli/src/commands/run/runner.rs @@ -161,13 +161,13 @@ pub(crate) async fn execute( artifact_sink, run_control: Some(run_control), github_app, - github_permissions: run_spec + github_integration: run_spec .settings .run .integrations .github - .resolve_permissions() - .context("failed to resolve github permissions")?, + .resolve_integration() + .context("failed to resolve github integration")?, vault, catalog, on_node: None, @@ -1762,7 +1762,10 @@ mod tests { .parse::() .expect("test provider should parse"); run.integrations = RunIntegrationsSettings { - github: RunIntegrationsGithubSettings { permissions }, + github: RunIntegrationsGithubSettings { + permissions, + ..RunIntegrationsGithubSettings::default() + }, }; run } diff --git a/lib/apps/fabro-cli/tests/it/cmd/attach.rs b/lib/apps/fabro-cli/tests/it/cmd/attach.rs index 9f388c14b..93cd58c83 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/attach.rs @@ -924,6 +924,7 @@ fn attach_json_errors_without_prompting_for_human_input() { "skip_git_hooks": false }, "clone": { + "depth": 100, "enabled": true }, "environment": { diff --git a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs index d01e477e1..56821fe45 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs @@ -152,7 +152,8 @@ fn inspect_resolves_selector_via_server_endpoint() { "commit_timeout_ms": 30000 }, "clone": { - "enabled": true + "enabled": true, + "depth": 100 }, "run_branch": { "enabled": true, diff --git a/lib/apps/fabro-cli/tests/it/support/mod.rs b/lib/apps/fabro-cli/tests/it/support/mod.rs index 57e4a98e5..2994b3c0a 100644 --- a/lib/apps/fabro-cli/tests/it/support/mod.rs +++ b/lib/apps/fabro-cli/tests/it/support/mod.rs @@ -47,6 +47,7 @@ pub(crate) fn run_projection_json(run_id: &str, status: &serde_json::Value) -> s graph: Graph::new("Remote Workflow"), graph_source: None, workflow_slug: Some("remote-workflow".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/srv/repo".to_string()), labels: std::collections::HashMap::default(), diff --git a/lib/apps/fabro-server/src/automation_materializer.rs b/lib/apps/fabro-server/src/automation_materializer.rs index b57d53b22..b2f1751b7 100644 --- a/lib/apps/fabro-server/src/automation_materializer.rs +++ b/lib/apps/fabro-server/src/automation_materializer.rs @@ -12,8 +12,7 @@ use fabro_util::error::collect_chain; use tokio::{fs, task}; use crate::git_checkout::{ - GitCheckoutError, GitRepoCache, WorktreePrepareInput, github_metadata_url, - resolve_git_auth_config, + GitCheckoutError, GitRepoCache, WorktreePrepareInput, resolve_git_auth_config, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -199,7 +198,7 @@ fn build_manifest_from_checkout( let mut manifest = built.manifest; manifest.git = Some(GitContext { - origin_url: github_metadata_url(&git_context.repo), + origin_url: git_context.repo.https_url(), branch: git_context.ref_selector, sha: Some(git_context.checked_out_sha), dirty: DirtyStatus::Clean, diff --git a/lib/apps/fabro-server/src/git_checkout.rs b/lib/apps/fabro-server/src/git_checkout.rs index 8bdb4f869..3648c6b4c 100644 --- a/lib/apps/fabro-server/src/git_checkout.rs +++ b/lib/apps/fabro-server/src/git_checkout.rs @@ -165,11 +165,9 @@ async fn bare_clone_may_be_corrupt(bare_dir: &Path) -> bool { } fn github_clone_url(repo: &GitHubRepositorySlug) -> String { - format!("https://github.com/{}/{}.git", repo.owner(), repo.repo()) -} - -pub(crate) fn github_metadata_url(repo: &GitHubRepositorySlug) -> String { - format!("https://github.com/{}/{}", repo.owner(), repo.repo()) + let mut url = repo.https_url(); + url.push_str(".git"); + url } #[derive(Clone, Debug, PartialEq, Eq)] @@ -478,10 +476,7 @@ mod tests { github_clone_url(&repo), "https://github.com/fabro-sh/fabro.git" ); - assert_eq!( - github_metadata_url(&repo), - "https://github.com/fabro-sh/fabro" - ); + assert_eq!(repo.https_url(), "https://github.com/fabro-sh/fabro"); assert!(!github_clone_url(&repo).contains('@')); } @@ -491,10 +486,7 @@ mod tests { assert_eq!(repo.owner(), "owner"); assert_eq!(repo.repo(), ".github"); - assert_eq!( - github_metadata_url(&repo), - "https://github.com/owner/.github" - ); + assert_eq!(repo.https_url(), "https://github.com/owner/.github"); } #[test] diff --git a/lib/apps/fabro-server/src/install.rs b/lib/apps/fabro-server/src/install.rs index 5f7d9b05a..0fae34bbb 100644 --- a/lib/apps/fabro-server/src/install.rs +++ b/lib/apps/fabro-server/src/install.rs @@ -2059,7 +2059,8 @@ fn build_github_app_manifest( "issues": "write", "emails": "read", "vulnerability_alerts": "write", - "organization_projects": "write" + "organization_projects": "write", + "packages": "read" }, "default_events": [] }) @@ -2375,6 +2376,21 @@ mod tests { ); } + #[test] + fn github_app_manifest_includes_packages_read_permission() { + let manifest = build_github_app_manifest( + "Fabro Test", + "https://fabro.example/setup", + "https://fabro.example/auth/callback/github", + "https://fabro.example/setup", + ); + + assert_eq!( + manifest["default_permissions"]["packages"], + serde_json::json!("read"), + ); + } + #[test] fn token_validation_accepts_any_matching_source() { let state = InstallAppState::for_test("expected"); diff --git a/lib/apps/fabro-server/src/run_compiler.rs b/lib/apps/fabro-server/src/run_compiler.rs index a018f46d0..b5358f1a3 100644 --- a/lib/apps/fabro-server/src/run_compiler.rs +++ b/lib/apps/fabro-server/src/run_compiler.rs @@ -38,6 +38,7 @@ use fabro_types::settings::interp::{InterpString, ResolveError}; use fabro_types::settings::run::{McpServerSettings, RunGoal}; use fabro_types::{ AutomationRef, GitContext, ManifestPath, RunId, RunProvenance, WorkflowSettings, + WorkflowVersionId, }; use fabro_util::workspace_glob::{WorkspaceGlob, WorkspaceGlobError}; use fabro_workflow::Error as WorkflowError; @@ -91,6 +92,7 @@ pub(crate) struct RawRunCompilerInput { pub(crate) git: Option, pub(crate) storage_root: PathBuf, pub(crate) workflow_slug: Option, + pub(crate) workflow_version_id: Option, pub(crate) provenance: RunProvenance, pub(crate) web_url: Option, pub(crate) submitted_manifest_bytes: Option>, @@ -121,6 +123,7 @@ struct RunMetadata { run_id: Option, storage_root: PathBuf, workflow_slug: Option, + workflow_version_id: Option, submitted_manifest_bytes: Option>, title: Option, automation: Option, @@ -314,6 +317,7 @@ pub(crate) fn normalize_source(input: RawRunCompilerInput) -> Result Result CreateRunPersistenceInput { run_id, storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -545,6 +551,7 @@ pub(crate) fn assemble_run(pinned: PinnedRun) -> CreateRunPersistenceInput { run_id: run_id.expect("run ID should be resolved before compilation"), storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -711,6 +718,7 @@ mod tests { git: None, storage_root: PathBuf::from("/tmp/fabro-storage"), workflow_slug: None, + workflow_version_id: None, provenance: provenance(), web_url: None, submitted_manifest_bytes: None, @@ -978,11 +986,13 @@ include = ["reports/{{ vars.path }}/*.json"] trigger_id: Some("schedule".to_string()), }; let submitted = b"submitted manifest".to_vec(); + let workflow_version_id = fabro_types::test_support::test_workflow_version_id(); let mut input = raw_input(None, HashMap::new()); input.run_id = Some(run_id); input.parent_id = Some(parent_id); input.title = Some("Compiler boundary".to_string()); input.workflow_slug = Some("compiler-boundary".to_string()); + input.workflow_version_id = Some(workflow_version_id); input.web_url = Some(format!("https://fabro.test/runs/{run_id}")); input.submitted_manifest_bytes = Some(submitted.clone()); input.automation = Some(automation.clone()); @@ -1005,6 +1015,7 @@ include = ["reports/{{ vars.path }}/*.json"] assert_eq!(persistence.run_id(), run_id); assert_eq!(persistence.workflow_slug(), Some("compiler-boundary")); + assert_eq!(persistence.workflow_version_id(), Some(workflow_version_id)); assert_eq!( persistence.submitted_manifest_bytes(), Some(submitted.as_slice()) diff --git a/lib/apps/fabro-server/src/run_files.rs b/lib/apps/fabro-server/src/run_files.rs index 2343933b6..78bc85ff8 100644 --- a/lib/apps/fabro-server/src/run_files.rs +++ b/lib/apps/fabro-server/src/run_files.rs @@ -2376,20 +2376,21 @@ index 1111111..2222222 160000 let mut projection = fabro_store::RunProjection::new( "Test run".to_string(), fabro_types::RunSpec { - run_id: fabro_types::fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: fabro_types::Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::default(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fabro_types::fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: fabro_types::Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::default(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), ); diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 0ee804a3a..30cac5b0d 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -12,6 +12,7 @@ use fabro_config::{ CliLayer, CliOutputLayer, EnvironmentLayer, MergeMap, RunLayer, SettingsLayer, WorkflowSettingsBuilder, parse_input_overrides, parse_labels, project, }; +use fabro_github::token_source::{InstallationTokenSource, ResolvedToken, TokenSnapshot}; use fabro_graphviz::graph::{Graph, is_llm_handler_type}; use fabro_graphviz::render::apply_direction; use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe}; @@ -669,11 +670,11 @@ pub(crate) fn effective_sandbox_provider(settings: &RunNamespace) -> SandboxProv } fn resolve_daytona_config(settings: &RunNamespace) -> DaytonaConfig { - daytona_config_from_environment(&settings.environment, !settings.clone.enabled) + daytona_config_from_environment(&settings.environment, &settings.clone) } fn resolve_docker_config(settings: &RunNamespace) -> DockerSandboxOptions { - docker_config_from_environment(&settings.environment, !settings.clone.enabled) + docker_config_from_environment(&settings.environment, &settings.clone) } #[derive(Clone, Debug, PartialEq, Eq)] @@ -872,6 +873,18 @@ async fn check_git_remote_ref( command.arg(branch); } + run_ls_remote(command) + .await + .map_err(|message| redact_auth_url(&message, auth_url.as_ref())) +} + +/// Run a prepared `git ls-remote` invocation with a 10s timeout, reducing a +/// failure to its most useful message: stderr, then stdout, then the exit +/// status. +async fn run_ls_remote(mut command: Command) -> std::result::Result<(), String> { + // Dropping a timed-out `Command::output` future does not stop the child + // unless kill-on-drop is enabled. + command.kill_on_drop(true); let output = time::timeout(Duration::from_secs(10), command.output()) .await .map_err(|_| "git ls-remote timed out after 10s".to_string())? @@ -883,14 +896,13 @@ async fn check_git_remote_ref( let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let message = if !stderr.is_empty() { + Err(if !stderr.is_empty() { stderr } else if !stdout.is_empty() { stdout } else { format!("git ls-remote exited with status {}", output.status) - }; - Err(redact_auth_url(&message, auth_url.as_ref())) + }) } fn preflight_sandbox_spec( @@ -923,6 +935,7 @@ fn preflight_sandbox_spec( run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, } } SandboxProviderKind::Daytona => { @@ -934,6 +947,7 @@ fn preflight_sandbox_spec( run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, api_key: daytona_api_key, } } @@ -1198,32 +1212,233 @@ async fn run_github_token_check( resolved_run: &RunNamespace, github_app: Option, ) -> bool { - if !resolved_run.integrations.github.is_token_requested() { + run_github_token_check_with( + checks, + prepared, + resolved_run, + github_app, + mint_scoped_github_token, + probe_github_repository, + ) + .await +} + +async fn run_github_token_check_with( + checks: &mut Vec, + prepared: &PreparedManifest, + resolved_run: &RunNamespace, + github_app: Option, + mint_scoped_token: M, + probe_repository: P, +) -> bool +where + M: FnOnce(fabro_github::GitHubRepositoryAccess, fabro_github::GitHubCredentials) -> MFut, + MFut: Future>, + P: Fn(fabro_types::GitHubRepositorySlug, ResolvedToken) -> PFut, + PFut: Future>, +{ + let github = &resolved_run.integrations.github; + if !github.is_token_requested() { return true; } // Resolve InterpString permission values eagerly for token minting and // for display in the preflight report. - let github_permissions = match resolved_run.integrations.github.resolve_permissions() { - Ok(permissions) => permissions, + let integration = match github.resolve_integration() { + Ok(integration) => integration, Err(err) => { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "invalid permissions".into(), - details: vec![], - remediation: Some(format!("Failed to resolve GitHub permissions: {err}")), - }); - return false; + return fail_github_token_check( + checks, + Vec::new(), + "invalid permissions", + format!("Failed to resolve GitHub permissions: {err}"), + ); } }; - - let perm_details = github_permissions + let perm_details = integration + .permissions .iter() .map(|(key, value)| CheckDetail::new(format!("{key}: {value}"))) .collect::>(); + + if !integration.has_additional_repositories() { + // Primary-only behavior is unchanged: a mint check when credentials + // and an origin exist, a warning otherwise, and no Git-content probe + // (permissions-only workflows may request non-contents permissions). + return check_primary_only_github_token( + checks, + prepared, + github_app, + &integration.permissions, + perm_details, + ) + .await; + } + + // `gh` checks GH_TOKEN before GITHUB_TOKEN, so a user-defined GH_TOKEN + // bypasses the managed scoped token for gh commands. Warn without + // failing; the value is the workflow author's responsibility. + if resolved_run.environment.env.contains_key(EnvVars::GH_TOKEN) { + checks.push(CheckResult { + name: "GH_TOKEN Override".into(), + status: CheckStatus::Warning, + summary: "gh will not use the managed token".into(), + details: vec![], + remediation: Some( + "The resolved run environment defines GH_TOKEN, which the gh CLI prefers over \ + the managed GITHUB_TOKEN; gh commands will not use the token scoped to the \ + declared repositories." + .to_string(), + ), + }); + } + + let Some(origin_url) = prepared + .git + .as_ref() + .map(|git| git.origin_url.trim()) + .filter(|url| !url.is_empty()) + else { + return fail_github_token_check( + checks, + perm_details, + "missing origin", + "run.integrations.github.additional_repositories requires a GitHub run origin, but \ + this run has no repository origin URL" + .to_string(), + ); + }; + let Some(creds) = github_app else { + return fail_github_token_check( + checks, + perm_details, + "missing credentials", + "run.integrations.github.additional_repositories requires GitHub credentials, but \ + none are configured on the server" + .to_string(), + ); + }; + // The same validated access value runtime initialization constructs, so + // preflight and runtime cannot disagree about the effective set. + let access = match fabro_github::GitHubRepositoryAccess::new( + Some(origin_url), + &integration.additional_repositories, + integration.permissions.clone(), + ) { + Ok(Some(access)) => access, + // `new` returns `Ok(None)` only when nothing is declared, and the + // declared set is non-empty here. Fail closed instead of panicking. + Ok(None) => { + return fail_github_token_check( + checks, + perm_details, + "missing origin", + "run.integrations.github.additional_repositories requires a GitHub run origin, \ + but this run has no repository origin URL" + .to_string(), + ); + } + Err(err) => { + return fail_github_token_check( + checks, + perm_details, + "invalid repository set", + format!("{err:#}"), + ); + } + }; + + // One mint scoped to the whole effective set. In App mode the minter + // first resolves every repository's installation so a failure names the + // repository the App cannot see. + let token = match mint_scoped_token(access.clone(), creds).await { + Ok(token) => token, + Err(err) => { + return fail_github_token_check(checks, perm_details, "failed", err); + } + }; + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Pass, + summary: "minted".into(), + details: perm_details.clone(), + remediation: None, + }); + + // Probe every effective repository with bounded concurrency, then report + // in deterministic primary-first order. Possession of a scoped token is + // not proof of access; the probe also verifies PAT/static credentials. + let targets: Vec = + access.targets().into_iter().cloned().collect(); + let probe_repository = &probe_repository; + let mut results: Vec<(usize, CheckResult)> = + stream::iter(targets.into_iter().enumerate().map(|(index, slug)| { + let token = token.clone(); + let perm_details = perm_details.clone(); + async move { + let check = match probe_repository(slug.clone(), token).await { + Ok(()) => CheckResult { + name: format!("GitHub Repository ({slug})"), + status: CheckStatus::Pass, + summary: "reachable".into(), + details: perm_details, + remediation: None, + }, + Err(err) => CheckResult { + name: format!("GitHub Repository ({slug})"), + status: CheckStatus::Error, + summary: "failed".into(), + details: perm_details, + remediation: Some(format!("Failed to verify repository access: {err}")), + }, + }; + (index, check) + } + })) + .buffer_unordered(REPOSITORY_PROBE_CONCURRENCY) + .collect() + .await; + results.sort_by_key(|(index, _)| *index); + + let mut ok = true; + for (_, check) in results { + if check.status != CheckStatus::Pass { + ok = false; + } + checks.push(check); + } + ok +} + +/// Report one "GitHub Token" preflight failure and fail the check. +fn fail_github_token_check( + checks: &mut Vec, + perm_details: Vec, + summary: &str, + remediation: String, +) -> bool { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: summary.into(), + details: perm_details, + remediation: Some(remediation), + }); + false +} + +/// Bounded concurrency for per-repository `git ls-remote` probes. +const REPOSITORY_PROBE_CONCURRENCY: usize = 4; + +async fn check_primary_only_github_token( + checks: &mut Vec, + prepared: &PreparedManifest, + github_app: Option, + permissions: &HashMap, + perm_details: Vec, +) -> bool { if let (Some(creds), Some(git)) = (&github_app, prepared.git.as_ref()) { - match mint_github_token(creds, &git.origin_url, &github_permissions).await { + match mint_github_token(creds, &git.origin_url, permissions).await { Ok(_) => { checks.push(CheckResult { name: "GitHub Token".into(), @@ -1257,6 +1472,62 @@ async fn run_github_token_check( } } +/// Production minter for the multi-repository path. The source owns the +/// effective set. In App mode its first resolve checks every repository's +/// installation before minting the scoped token. +async fn mint_scoped_github_token( + access: fabro_github::GitHubRepositoryAccess, + creds: fabro_github::GitHubCredentials, +) -> std::result::Result { + let source = + InstallationTokenSource::for_access(&creds, &access).map_err(|err| format!("{err:#}"))?; + source.resolve().await.map_err(|err| format!("{err:#}")) +} + +/// Production per-repository probe: a non-interactive +/// `git ls-remote HEAD` authenticated through +/// [`fabro_github::GITHUB_CREDENTIAL_HELPER`] reading `GITHUB_TOKEN` from +/// the child process environment, so the token never appears in the URL, +/// argv, or rendered errors — exactly what the runtime `git_bridge` +/// configures in `fabro-workflow`. +async fn probe_github_repository( + slug: fabro_types::GitHubRepositorySlug, + token: ResolvedToken, +) -> std::result::Result<(), String> { + let url = slug.https_url(); + probe_with_replication_retry(token.snapshot, || run_probe_ls_remote(&url, &token)).await +} + +/// Retry auth-shaped failures with the SAME token: replication of a given +/// token only makes progress, while re-minting would restart the replication +/// clock. The sandbox git retry executor owns attempt limits, +/// classification, and pacing. +async fn probe_with_replication_retry( + snapshot: TokenSnapshot, + mut run: F, +) -> std::result::Result<(), String> +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let credential_context = fabro_sandbox::CredentialContext::from_snapshot(Some(&snapshot)); + fabro_sandbox::retry_git_operation( + SandboxProviderKind::Local, + "repository probe", + &fabro_sandbox::RetryPlan::repository_probe(), + |_attempt| run(), + |message| fabro_sandbox::classify_failure(message, credential_context), + ) + .await +} + +async fn run_probe_ls_remote(url: &str, token: &ResolvedToken) -> std::result::Result<(), String> { + let mut command = Command::new("git"); + fabro_github::apply_probe_git_env(&mut command, token.token.expose()); + command.args(["ls-remote", url, "HEAD"]); + run_ls_remote(command).await +} + async fn mint_github_token( creds: &fabro_github::GitHubCredentials, origin_url: &str, @@ -2972,4 +3243,336 @@ dockerfile = { path = "Dockerfile" } ); } } + + mod github_additional_repository_checks { + //! Seam-injected tests for the declared-additional-repositories + //! preflight path: one scoped mint, per-repository probes with + //! deterministic primary-first reporting, GH_TOKEN warning, and the + //! replication-lag retry policy. + + use std::sync::Mutex as StdMutex; + use std::sync::atomic::{AtomicU64, Ordering}; + + use fabro_github::token_source::{ + ResolvedToken, SecretString, TokenProvenance, TokenSnapshot, + }; + use fabro_types::settings::run::RunIntegrationsGithubSettings; + + use super::*; + + fn static_token(secret: &str) -> ResolvedToken { + ResolvedToken { + token: SecretString::new(secret.to_string()), + snapshot: TokenSnapshot { + generation: 0, + provenance: TokenProvenance::Static, + }, + refresh_failed: false, + } + } + + fn fresh_minted_snapshot() -> TokenSnapshot { + let now = chrono::Utc::now(); + TokenSnapshot { + generation: 1, + provenance: TokenProvenance::Minted { + minted_at: now, + expires_at: now + chrono::Duration::minutes(60), + }, + } + } + + fn declared(origin: &str, additional: &[&str]) -> (PreparedManifest, RunNamespace) { + let (prepared, mut resolved) = prepared_and_resolved_for_sandbox( + SandboxProviderKind::Local, + true, + Some(git_context(origin, "main")), + ); + resolved.integrations.github = RunIntegrationsGithubSettings { + permissions: HashMap::from([( + "contents".to_string(), + InterpString::parse("read"), + )]), + additional_repositories: additional + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect(), + }; + (prepared, resolved) + } + + fn pat_creds() -> fabro_github::GitHubCredentials { + fabro_github::GitHubCredentials::Pat("ghp_test".to_string()) + } + + #[tokio::test(start_paused = true)] + async fn reports_each_repository_primary_first_despite_probe_completion_order() { + let (prepared, resolved) = declared("https://github.com/acme/widgets", &[ + "acme/zeta", + "acme/alpha", + ]); + let minted = Arc::new(StdMutex::new(Vec::new())); + let minted_for_seam = Arc::clone(&minted); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + move |access, _creds| { + minted_for_seam.lock().unwrap().push(access); + async { Ok(static_token("scoped-token")) } + }, + |slug, _token| async move { + // Invert completion order: the primary finishes last. + let delay = match slug.repo() { + "widgets" => 30, + "alpha" => 20, + _ => 10, + }; + time::sleep(Duration::from_millis(delay)).await; + Ok(()) + }, + ) + .await; + + assert!(ok); + // One mint listing every repository with the shared permissions. + let minted = minted.lock().unwrap(); + assert_eq!(minted.len(), 1); + assert_eq!(minted[0].repository_names(), vec![ + "widgets", "alpha", "zeta" + ]); + assert_eq!( + minted[0].permissions().get("contents").map(String::as_str), + Some("read") + ); + + let names: Vec<&str> = checks.iter().map(|check| check.name.as_str()).collect(); + assert_eq!(names, vec![ + "GitHub Token", + "GitHub Repository (acme/widgets)", + "GitHub Repository (acme/alpha)", + "GitHub Repository (acme/zeta)", + ]); + assert!(checks.iter().all(|check| check.status == CheckStatus::Pass)); + // The token never reaches check output. + for check in &checks { + let rendered = format!("{check:?}"); + assert!(!rendered.contains("scoped-token"), "{rendered}"); + } + } + + #[tokio::test] + async fn installation_resolution_failure_names_only_the_inaccessible_repository() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let probes = Arc::new(AtomicU64::new(0)); + let probes_for_seam = Arc::clone(&probes); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { + Err( + "the GitHub App installation cannot see repository acme/keystone; add \ + it to the installation's repository access" + .to_string(), + ) + }, + move |_slug, _token| { + probes_for_seam.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }, + ) + .await; + + assert!(!ok); + assert_eq!( + probes.load(Ordering::SeqCst), + 0, + "no probes after a failed mint" + ); + assert_eq!(checks.last().unwrap().name, "GitHub Token"); + assert_eq!(checks.last().unwrap().status, CheckStatus::Error); + let remediation = checks.last().unwrap().remediation.as_deref().unwrap(); + assert!(remediation.contains("acme/keystone"), "{remediation}"); + assert!(!remediation.contains("acme/widgets"), "{remediation}"); + } + + #[tokio::test] + async fn successful_mint_with_failed_probe_still_fails() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |slug, _token| async move { + if slug.repo() == "keystone" { + Err("remote: Repository not found.".to_string()) + } else { + Ok(()) + } + }, + ) + .await; + + assert!(!ok); + let keystone = checks + .iter() + .find(|check| check.name == "GitHub Repository (acme/keystone)") + .expect("keystone probe result should be reported"); + assert_eq!(keystone.status, CheckStatus::Error); + assert!( + !keystone + .remediation + .as_deref() + .unwrap_or_default() + .contains("scoped-token") + ); + let widgets = checks + .iter() + .find(|check| check.name == "GitHub Repository (acme/widgets)") + .expect("primary probe result should be reported"); + assert_eq!(widgets.status, CheckStatus::Pass); + } + + #[tokio::test] + async fn resolved_gh_token_warns_without_failing() { + let (prepared, mut resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + resolved + .environment + .env + .insert("GH_TOKEN".to_string(), InterpString::parse("user-token")); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(ok, "a GH_TOKEN override warns but does not fail preflight"); + let warning = checks + .iter() + .find(|check| check.name == "GH_TOKEN Override") + .expect("GH_TOKEN warning should be reported"); + assert_eq!(warning.status, CheckStatus::Warning); + } + + #[tokio::test] + async fn missing_origin_fails_for_declared_repositories() { + let (prepared, resolved) = { + let (mut prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + prepared.git = None; + (prepared, resolved) + }; + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(!ok); + let check = checks.last().unwrap(); + assert_eq!(check.summary, "missing origin"); + assert!( + check + .remediation + .as_deref() + .unwrap_or_default() + .contains("requires a GitHub run origin"), + "{:?}", + check.remediation + ); + } + + #[tokio::test] + async fn missing_credentials_fail_for_declared_repositories() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + None, + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(!ok); + assert_eq!(checks.last().unwrap().summary, "missing credentials"); + } + + #[tokio::test(start_paused = true)] + async fn replication_lag_failure_retries_with_the_same_token_and_succeeds() { + let attempts = Arc::new(AtomicU64::new(0)); + let attempts_for_run = Arc::clone(&attempts); + + let result = probe_with_replication_retry(fresh_minted_snapshot(), move || { + let attempts = Arc::clone(&attempts_for_run); + async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err("remote: Repository not found.".to_string()) + } else { + Ok(()) + } + } + }) + .await; + + assert!(result.is_ok()); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } + + #[tokio::test(start_paused = true)] + async fn static_credential_auth_failures_do_not_retry() { + let attempts = Arc::new(AtomicU64::new(0)); + let attempts_for_run = Arc::clone(&attempts); + let static_snapshot = TokenSnapshot { + generation: 0, + provenance: TokenProvenance::Static, + }; + + let result = probe_with_replication_retry(static_snapshot, move || { + let attempts = Arc::clone(&attempts_for_run); + async move { + attempts.fetch_add(1, Ordering::SeqCst); + Err("remote: Repository not found.".to_string()) + } + }) + .await; + + assert!(result.is_err()); + assert_eq!( + attempts.load(Ordering::SeqCst), + 1, + "a 404 with a static credential cannot become valid by waiting" + ); + } + } } diff --git a/lib/apps/fabro-server/src/server.rs b/lib/apps/fabro-server/src/server.rs index 8c7d5514d..3c6935f51 100644 --- a/lib/apps/fabro-server/src/server.rs +++ b/lib/apps/fabro-server/src/server.rs @@ -4081,15 +4081,15 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { return; } }; - let github_permissions = match persisted + let github_integration = match persisted .run_spec() .settings .run .integrations .github - .resolve_permissions() + .resolve_integration() { - Ok(permissions) => permissions, + Ok(integration) => integration, Err(err) => { tracing::error!( run_id = %run_id, @@ -4131,7 +4131,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { artifact_sink: Some(ArtifactSink::Store(state.artifact_store.clone())), run_control: None, github_app, - github_permissions, + github_integration, vault: Arc::new(AsyncRwLock::new(vault.into_vault())), catalog: state.catalog(), on_node: None, diff --git a/lib/apps/fabro-server/src/server/handler/events.rs b/lib/apps/fabro-server/src/server/handler/events.rs index 5fd655f1d..926fefdca 100644 --- a/lib/apps/fabro-server/src/server/handler/events.rs +++ b/lib/apps/fabro-server/src/server/handler/events.rs @@ -616,23 +616,24 @@ mod stage_events_tests { async fn append_run_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { workflow_event::append_event(run_store, run_id, &workflow_event::Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: *run_id, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("run.created should append"); diff --git a/lib/apps/fabro-server/src/server/handler/pair.rs b/lib/apps/fabro-server/src/server/handler/pair.rs index 6e244bcac..c3c3166ee 100644 --- a/lib/apps/fabro-server/src/server/handler/pair.rs +++ b/lib/apps/fabro-server/src/server/handler/pair.rs @@ -1024,6 +1024,7 @@ mod tests { labels: std::collections::BTreeMap::new(), source_directory: None, workflow_slug: None, + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/apps/fabro-server/src/server/handler/runs.rs b/lib/apps/fabro-server/src/server/handler/runs.rs index 56ae1c754..8a8efc259 100644 --- a/lib/apps/fabro-server/src/server/handler/runs.rs +++ b/lib/apps/fabro-server/src/server/handler/runs.rs @@ -730,6 +730,7 @@ pub(crate) async fn create_run_from_manifest( git: manifest.git.clone(), storage_root: state.server_storage_dir(), workflow_slug: None, + workflow_version_id: None, provenance: run_provenance(&headers, &actor), web_url: None, submitted_manifest_bytes: Some(submitted_manifest_bytes), diff --git a/lib/apps/fabro-server/src/server/handler/sessions.rs b/lib/apps/fabro-server/src/server/handler/sessions.rs index e9bacdfff..835b9ed51 100644 --- a/lib/apps/fabro-server/src/server/handler/sessions.rs +++ b/lib/apps/fabro-server/src/server/handler/sessions.rs @@ -1917,6 +1917,7 @@ reasoning = false graph, graph_source: None, workflow_slug: None, + workflow_version_id: None, automation: None, source_directory: None, labels: HashMap::default(), diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index dc656c32e..dc21a12e6 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -4649,6 +4649,7 @@ async fn append_default_run_created(run_store: &fabro_store::RunDatabase, run_id labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: None, + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -4701,6 +4702,7 @@ async fn create_slack_notification_run( labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: workflow_slug.map(str::to_string), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -5775,6 +5777,7 @@ async fn list_run_stages_distinguishes_visits() { labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -5912,6 +5915,7 @@ async fn list_run_stages_exposes_execution_identity_for_resumed_stage() { labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -7096,6 +7100,7 @@ async fn create_completed_run_ready_for_pull_request( graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/tmp/project".to_string()), git: git.clone(), @@ -7117,6 +7122,7 @@ async fn create_completed_run_ready_for_pull_request( labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: run_spec.workflow_version_id, automation: None, provenance: run_spec.provenance.clone(), manifest_blob: None, @@ -14090,6 +14096,7 @@ async fn create_preserved_local_sandbox_run(state: &Arc, run_id: RunId labels: std::collections::BTreeMap::default(), source_directory: Some("/tmp/fabro-run".to_string()), workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -14840,6 +14847,7 @@ async fn delete_run_retry_after_missing_provider_resource_removes_metadata() { labels: std::collections::BTreeMap::default(), source_directory: Some("/tmp/fabro-run".to_string()), workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/apps/fabro-server/tests/it/api/run_files.rs b/lib/apps/fabro-server/tests/it/api/run_files.rs index 307c4a574..b8aad6410 100644 --- a/lib/apps/fabro-server/tests/it/api/run_files.rs +++ b/lib/apps/fabro-server/tests/it/api/run_files.rs @@ -56,24 +56,26 @@ async fn append_completed_run_with_final_patch( ) { let run_store = store.create_run(run_id).await.expect("create run store"); workflow_event::append_event(&run_store, run_id, &workflow_event::Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()) + run_id: *run_id, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()) .expect("workflow settings should serialize"), - graph: serde_json::to_value(Graph::new("test")).expect("graph should serialize"), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(Graph::new("test")) + .expect("graph should serialize"), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("append RunCreated"); diff --git a/lib/components/fabro-agent/tests/it/docker_shell.rs b/lib/components/fabro-agent/tests/it/docker_shell.rs index 48c0d25b3..6f50423ba 100644 --- a/lib/components/fabro-agent/tests/it/docker_shell.rs +++ b/lib/components/fabro-agent/tests/it/docker_shell.rs @@ -28,6 +28,7 @@ async fn shell_reports_real_docker_process_outcome() { None, None, None, + None, ) else { return; }; diff --git a/lib/components/fabro-github/Cargo.toml b/lib/components/fabro-github/Cargo.toml index 8f755dd79..a2d6d7abd 100644 --- a/lib/components/fabro-github/Cargo.toml +++ b/lib/components/fabro-github/Cargo.toml @@ -25,6 +25,7 @@ fabro-http.workspace = true fabro-redact.workspace = true fabro-static.workspace = true fabro-types = { path = "../../foundation/fabro-types" } +futures.workspace = true jsonwebtoken.workspace = true chrono.workspace = true tracing.workspace = true diff --git a/lib/components/fabro-github/src/access.rs b/lib/components/fabro-github/src/access.rs new file mode 100644 index 000000000..774ecc445 --- /dev/null +++ b/lib/components/fabro-github/src/access.rs @@ -0,0 +1,587 @@ +//! The validated effective repository set for one run's GitHub access. +//! +//! [`GitHubRepositoryAccess`] is the single value both server preflight and +//! workflow initialization construct from the run origin, the declared +//! additional repositories, and the resolved shared permissions — so the two +//! paths cannot disagree about which repositories a run's `GITHUB_TOKEN` +//! covers. It carries no token or key material. + +use std::collections::{BTreeSet, HashMap}; + +use anyhow::{Context as _, bail}; +use fabro_types::GitHubRepositorySlug; +use fabro_types::settings::run::RunIntegrationsGithubSettings; +use futures::stream::{self, StreamExt as _}; + +use crate::{GitHubAppCredentials, HttpClient, InstallationLookup, InstallationToken}; + +/// Keep GitHub installation lookups bounded while avoiding one network round +/// trip at a time for large declared repository sets. +const INSTALLATION_LOOKUP_CONCURRENCY: usize = 4; + +/// The validated effective repository set for a run: the primary origin +/// repository plus zero or more distinct additional repositories, all with +/// one shared owner, and the shared permission map that scopes the token. +/// +/// Secret-free by construction: `Debug` may render everywhere the run +/// pipeline logs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitHubRepositoryAccess { + primary: GitHubRepositorySlug, + /// Sorted, deduplicated, primary excluded. + additional: Vec, + permissions: HashMap, +} + +impl GitHubRepositoryAccess { + /// Build the effective access request. + /// + /// Returns `Ok(None)` when no origin URL is available and no additional + /// repositories are declared — the legacy "nothing to scope" state whose + /// handling stays with the caller. Every declared-additional invariant is + /// enforced here: + /// + /// - a declared additional set requires a GitHub origin, + /// - the additional set cannot contain the primary repository, + /// - every additional repository shares the primary's owner + /// (case-insensitive) because one App installation covers one account, + /// - a declared additional set requires interpolated permissions with + /// `contents = "read"` or `contents = "write"`. + pub fn new( + origin_url: Option<&str>, + additional_repositories: &BTreeSet, + permissions: HashMap, + ) -> anyhow::Result> { + let origin_url = origin_url.map(str::trim).filter(|url| !url.is_empty()); + let Some(origin_url) = origin_url else { + if additional_repositories.is_empty() { + return Ok(None); + } + bail!( + "run.integrations.github.additional_repositories requires a GitHub run origin; \ + this run has no repository origin URL" + ); + }; + + let normalized = crate::normalize_repo_origin_url(origin_url); + let (owner, repo) = crate::parse_github_owner_repo(&normalized) + .context("parsing GitHub origin for repository access")?; + let Some(primary) = GitHubRepositorySlug::try_new(&format!("{owner}/{repo}")) else { + bail!("run origin does not name a valid GitHub `owner/repository`: {owner}/{repo}"); + }; + + if !additional_repositories.is_empty() { + validate_additional_permissions(&permissions)?; + } + + let mut additional = Vec::with_capacity(additional_repositories.len()); + for slug in additional_repositories { + if *slug == primary { + bail!( + "run.integrations.github.additional_repositories must not repeat the run \ + origin repository `{primary}` — the origin is always included" + ); + } + if !slug.same_owner(&primary) { + bail!( + "additional repository `{slug}` has owner `{}` but the run origin `{primary}` \ + has owner `{}`; all repositories must share one owner because one GitHub App \ + installation covers one account", + slug.owner(), + primary.owner() + ); + } + additional.push(slug.clone()); + } + + Ok(Some(Self { + primary, + additional, + permissions, + })) + } + + #[must_use] + pub fn primary(&self) -> &GitHubRepositorySlug { + &self.primary + } + + /// Every repository in the effective set, primary first, then the + /// additional repositories in their deterministic sorted order. + #[must_use] + pub fn targets(&self) -> Vec<&GitHubRepositorySlug> { + std::iter::once(&self.primary) + .chain(self.additional.iter()) + .collect() + } + + /// Project each validated slug to its repository-name component for the + /// installation-token mint request, which accepts names within the + /// selected installation. Every target shares the primary's owner, so the + /// projection loses nothing. + #[must_use] + pub fn repository_names(&self) -> Vec { + self.targets() + .into_iter() + .map(|slug| slug.repo().to_string()) + .collect() + } + + #[must_use] + pub fn owner(&self) -> &str { + self.primary.owner() + } + + #[must_use] + pub fn permissions(&self) -> &HashMap { + &self.permissions + } + + pub fn permissions_json(&self) -> anyhow::Result { + serde_json::to_value(&self.permissions).context("serializing GitHub permissions") + } + + #[must_use] + pub fn has_additional_repositories(&self) -> bool { + !self.additional.is_empty() + } + + /// Resolve every target's App installation and require one shared + /// installation ID, so a repository the App cannot see — or one that + /// resolves to a different installation — is named before any token is + /// minted. Targets are checked in deterministic primary-first order. + async fn resolve_shared_installation_with_jwt( + &self, + client: &impl HttpClient, + jwt: &str, + base_url: &str, + ) -> anyhow::Result { + let targets: Vec = self.targets().into_iter().cloned().collect(); + let mut lookups = stream::iter(targets.into_iter().map(|slug| async move { + let lookup = + crate::lookup_installation(client, jwt, base_url, slug.owner(), slug.repo()) + .await + .with_context(|| format!("looking up the GitHub App installation for {slug}")); + (slug, lookup) + })) + .buffered(INSTALLATION_LOOKUP_CONCURRENCY); + + let mut shared: Option<(u64, GitHubRepositorySlug)> = None; + while let Some((slug, lookup)) = lookups.next().await { + let lookup = lookup?; + let id = match lookup { + InstallationLookup::Found(id) => id, + InstallationLookup::NotFound => bail!( + "the GitHub App installation cannot see repository {slug}; add it to the \ + installation's repository access" + ), + InstallationLookup::Failed(status) => bail!( + "unexpected status {status} looking up the GitHub App installation for {slug}" + ), + }; + match &shared { + None => shared = Some((id, slug)), + Some((shared_id, first)) if *shared_id != id => bail!( + "repository {slug} belongs to GitHub App installation {id} but {first} \ + belongs to installation {shared_id}; all repositories must share one \ + installation" + ), + Some(_) => {} + } + } + let (id, _) = shared.expect("the effective repository set always contains the primary"); + Ok(id) + } + + /// Resolve every target to one installation, then mint one token scoped + /// to this exact repository set without looking up the primary twice. + pub(crate) async fn mint_installation_token( + &self, + creds: &GitHubAppCredentials, + client: &impl HttpClient, + base_url: &str, + ) -> anyhow::Result { + let jwt = crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem)?; + let installation_id = self + .resolve_shared_installation_with_jwt(client, &jwt, base_url) + .await + .context("resolving the shared GitHub App installation")?; + crate::mint_installation_token_for_id_with_jwt( + client, + &jwt, + installation_id, + &self.repository_names(), + base_url, + self.permissions_json()?, + ) + .await + } +} + +/// A non-empty additional set needs a token that can reach repository +/// contents. Configuration resolution already checked literal values; this +/// is the runtime re-check after `{{ vars.* }}` interpolation. +fn validate_additional_permissions(permissions: &HashMap) -> anyhow::Result<()> { + let Some(contents) = permissions.get("contents") else { + bail!( + "run.integrations.github.additional_repositories requires the `contents` permission \ + (`read` or `write`)" + ); + }; + if !RunIntegrationsGithubSettings::contents_permission_allows_repository_access(contents) { + bail!( + "run.integrations.github.additional_repositories requires `contents = \"read\"` or \ + `contents = \"write\"`, got `{contents}`" + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn slugs(values: &[&str]) -> BTreeSet { + values + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect() + } + + fn contents_read() -> HashMap { + HashMap::from([("contents".to_string(), "read".to_string())]) + } + + fn access( + origin: &str, + additional: &[&str], + permissions: HashMap, + ) -> anyhow::Result> { + GitHubRepositoryAccess::new(Some(origin), &slugs(additional), permissions) + } + + #[test] + fn https_and_both_ssh_origin_forms_normalize_to_the_same_primary() { + let origins = [ + "https://github.com/fabro-sh/fabro.git", + "git@github.com:fabro-sh/fabro.git", + "ssh://git@github.com/fabro-sh/fabro.git", + "https://github.com/fabro-sh/fabro", + ]; + for origin in origins { + let access = access(origin, &[], HashMap::new()) + .expect(origin) + .expect("origin should produce an access value"); + assert_eq!(access.primary().to_string(), "fabro-sh/fabro", "{origin}"); + } + } + + #[test] + fn no_origin_and_no_additional_repositories_is_none() { + let access = GitHubRepositoryAccess::new(None, &BTreeSet::new(), HashMap::new()).unwrap(); + assert!(access.is_none()); + + let blank = + GitHubRepositoryAccess::new(Some(" "), &BTreeSet::new(), HashMap::new()).unwrap(); + assert!(blank.is_none()); + } + + #[test] + fn additional_repositories_require_an_origin() { + let err = + GitHubRepositoryAccess::new(None, &slugs(&["fabro-sh/keystone"]), contents_read()) + .unwrap_err(); + assert!( + err.to_string().contains("requires a GitHub run origin"), + "{err:#}" + ); + } + + #[test] + fn additional_repositories_require_a_github_origin() { + let err = access( + "https://gitlab.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap_err(); + assert!(err.to_string().contains("repository access"), "{err:#}"); + } + + #[test] + fn rejects_primary_duplication_regardless_of_url_spelling_or_case() { + let origins = [ + "https://github.com/Fabro-SH/Fabro.git", + "git@github.com:fabro-sh/fabro.git", + "ssh://git@github.com/fabro-sh/fabro", + ]; + for origin in origins { + let err = access(origin, &["fabro-sh/FABRO"], contents_read()).unwrap_err(); + assert!( + err.to_string().contains("must not repeat the run origin"), + "{origin}: {err:#}" + ); + } + } + + #[test] + fn rejects_an_additional_repository_with_a_different_owner() { + let err = access( + "https://github.com/fabro-sh/fabro", + &["lithoscomputer/conveyor"], + contents_read(), + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("lithoscomputer/conveyor"), "{message}"); + assert!(message.contains("share one owner"), "{message}"); + } + + #[test] + fn rejects_a_declared_set_without_a_contents_permission() { + let missing = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + HashMap::new(), + ) + .unwrap_err(); + assert!( + missing.to_string().contains("`contents` permission"), + "{missing:#}" + ); + + let wrong_level = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + HashMap::from([("contents".to_string(), "admin".to_string())]), + ) + .unwrap_err(); + assert!( + wrong_level.to_string().contains("got `admin`"), + "{wrong_level:#}" + ); + } + + #[test] + fn targets_retain_every_full_slug_exactly_once_primary_first() { + let access = access( + "git@github.com:fabro-sh/fabro.git", + &["fabro-sh/keystone", "fabro-sh/arc"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let targets: Vec = access.targets().iter().map(ToString::to_string).collect(); + assert_eq!(targets, vec![ + "fabro-sh/fabro", + "fabro-sh/arc", + "fabro-sh/keystone", + ]); + assert_eq!(access.repository_names(), vec!["fabro", "arc", "keystone"]); + assert_eq!(access.owner(), "fabro-sh"); + assert!(access.has_additional_repositories()); + } + + #[test] + fn debug_output_contains_only_repositories_and_permissions() { + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/arc"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let rendered = format!("{access:?}"); + assert!(rendered.contains("fabro-sh"), "{rendered}"); + assert!(rendered.contains("contents"), "{rendered}"); + // The value carries no token or key material by construction; its + // fields are exactly the repository slugs and the permission map. + assert!(!rendered.to_lowercase().contains("token"), "{rendered}"); + assert!(!rendered.to_lowercase().contains("key"), "{rendered}"); + } + + #[tokio::test] + async fn resolve_shared_installation_names_the_invisible_repository() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 404, + "{}", + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let err = access + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) + .await + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("fabro-sh/keystone"), "{message}"); + assert!(message.contains("cannot see"), "{message}"); + } + + #[tokio::test] + async fn resolve_shared_installation_requires_one_installation_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 8}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let err = access + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) + .await + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("installation 8"), "{message}"); + assert!(message.contains("fabro-sh/keystone"), "{message}"); + } + + #[tokio::test] + async fn resolve_shared_installation_returns_the_shared_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 7}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let id = access + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) + .await + .unwrap(); + assert_eq!(id, 7); + } + + #[tokio::test] + async fn access_mint_reuses_the_resolved_installation_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Post, + "/app/installations/7/access_tokens", + 201, + r#"{"token":"scoped","expires_at":"2099-01-01T00:00:00Z"}"#, + ) + .with_req_body( + r#"{"repositories":["fabro","keystone"],"permissions":{"contents":"read"}}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let token = access + .mint_installation_token(&creds, &mock, "") + .await + .unwrap(); + + assert_eq!(token.token, "scoped"); + assert_eq!( + mock.request_count(), + 3, + "each repository should be looked up once before the mint" + ); + } +} diff --git a/lib/components/fabro-github/src/lib.rs b/lib/components/fabro-github/src/lib.rs index a38c6c8ce..87990e757 100644 --- a/lib/components/fabro-github/src/lib.rs +++ b/lib/components/fabro-github/src/lib.rs @@ -9,13 +9,45 @@ use fabro_types::settings::run::MergeStrategy; use serde::Deserialize; use tokio::process::Command; +pub mod access; pub mod token_source; #[cfg(any(test, feature = "test-support"))] pub mod test_support; +#[cfg(test)] +pub(crate) mod tests_mock; + +pub use access::GitHubRepositoryAccess; pub const GITHUB_API_BASE_URL: &str = "https://api.github.com"; +/// Git config key that routes github.com HTTPS credentials through +/// [`GITHUB_CREDENTIAL_HELPER`]. +pub const GITHUB_CREDENTIAL_HELPER_KEY: &str = "credential.https://github.com.helper"; + +/// Secret-free git credential helper: reads `$GITHUB_TOKEN` from the +/// invoking git process's environment at invocation time, so the token never +/// lands in git configuration, argv, or rendered errors. Non-`get` +/// operations (`store`, `erase`) are ignored. The one definition shared by +/// the runtime git bridge, server preflight probes, and the live contract +/// test, so the probes always exercise exactly what the bridge configures. +pub const GITHUB_CREDENTIAL_HELPER: &str = r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#; + +/// Configure a `git` invocation to authenticate to github.com through +/// [`GITHUB_CREDENTIAL_HELPER`] with `token`, isolated from user/system git +/// configuration and terminal prompts. The token is passed only through the +/// child process environment. +pub fn apply_probe_git_env(command: &mut Command, token: &str) { + command + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env(EnvVars::GITHUB_TOKEN, token) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", GITHUB_CREDENTIAL_HELPER_KEY) + .env("GIT_CONFIG_VALUE_0", GITHUB_CREDENTIAL_HELPER); +} + /// Returns the GitHub API base URL, allowing override via `GITHUB_BASE_URL` env /// var. #[expect( @@ -151,6 +183,29 @@ impl GitHubAppCredentials { base_url: &str, permissions: serde_json::Value, install_url: Option<&str>, + ) -> anyhow::Result { + self.mint_installation_token_for_repositories( + client, + owner, + &[repo.to_string()], + base_url, + permissions, + install_url, + ) + .await + } + + /// Mint one installation token scoped to every repository in + /// `repository_names` (names within `owner`'s installation, primary + /// first) with the shared `permissions`. + pub async fn mint_installation_token_for_repositories( + &self, + client: &impl HttpClient, + owner: &str, + repository_names: &[String], + base_url: &str, + permissions: serde_json::Value, + install_url: Option<&str>, ) -> anyhow::Result { let jwt = sign_app_jwt(&self.app_id, &self.private_key_pem)?; let default_install_url = self.installation_url(owner); @@ -159,7 +214,7 @@ impl GitHubAppCredentials { client, &jwt, owner, - repo, + repository_names, base_url, permissions, install_url, @@ -475,47 +530,88 @@ pub async fn create_installation_access_token_with_permissions_and_install_url( permissions: serde_json::Value, install_url: Option<&str>, ) -> anyhow::Result { - mint_installation_token_with_jwt(client, jwt, owner, repo, base_url, permissions, install_url) - .await - .map(|token| token.token) + mint_installation_token_with_jwt( + client, + jwt, + owner, + &[repo.to_string()], + base_url, + permissions, + install_url, + ) + .await + .map(|token| token.token) +} + +/// Outcome of one GitHub App installation lookup +/// (`GET /repos/{owner}/{repo}/installation`). +/// +/// Status interpretation stays with callers because their user guidance +/// differs: the mint path speaks about the owner's App installation, the +/// multi-repository resolution names the specific repository. +pub(crate) enum InstallationLookup { + Found(u64), + /// 404: the App cannot see the repository (or is not installed at all). + NotFound, + /// Any other non-200 status. + Failed(u16), +} + +/// Look up the App installation covering `owner/repo` and parse its id. +/// Transport and parse failures carry no caller-specific context; callers +/// attach their own. +pub(crate) async fn lookup_installation( + client: &impl HttpClient, + jwt: &str, + base_url: &str, + owner: &str, + repo: &str, +) -> anyhow::Result { + #[derive(Deserialize)] + struct Installation { + id: u64, + } + + let endpoint = format!("{base_url}/repos/{owner}/{repo}/installation"); + let auth = format!("Bearer {jwt}"); + let resp = client + .request(HttpMethod::Get, &endpoint, &github_headers(&auth), None) + .await?; + match resp.status { + 200 => { + let installation: Installation = resp + .json() + .context("Failed to parse installation response")?; + Ok(InstallationLookup::Found(installation.id)) + } + 404 => Ok(InstallationLookup::NotFound), + status => Ok(InstallationLookup::Failed(status)), + } } async fn mint_installation_token_with_jwt( client: &impl HttpClient, jwt: &str, owner: &str, - repo: &str, + repos: &[String], base_url: &str, permissions: serde_json::Value, install_url: Option<&str>, ) -> anyhow::Result { - #[derive(Deserialize)] - struct Installation { - id: u64, - } + let Some(primary_repo) = repos.first() else { + bail!("installation token mint requires at least one repository"); + }; - #[derive(Deserialize)] - struct AccessToken { - token: String, - expires_at: DateTime, - } - - // Step 1: Find the installation for this repo - let installation_endpoint = format!("{base_url}/repos/{owner}/{repo}/installation"); - let auth = format!("Bearer {jwt}"); - let resp = client - .request( - HttpMethod::Get, - &installation_endpoint, - &github_headers(&auth), - None, - ) + // Step 1: Find the installation via the primary repository. Multi- + // repository callers resolve every repository's installation up front + // (`GitHubRepositoryAccess::resolve_shared_installation`), so the + // primary stands for the whole set here. + let installation_id = match lookup_installation(client, jwt, base_url, owner, primary_repo) .await - .context("Failed to look up GitHub App installation")?; - - match resp.status { - 200 => {} - 404 => { + .context("Failed to look up GitHub App installation")? + { + InstallationLookup::Found(id) => id, + InstallationLookup::NotFound => { let install_url = install_url.map_or_else( || format!("https://github.com/organizations/{owner}/settings/installations"), str::to_string, @@ -525,37 +621,54 @@ async fn mint_installation_token_with_jwt( Install it at {install_url}" ); } - 403 => { + InstallationLookup::Failed(403) => { bail!( "GitHub App installation is suspended. \ Re-enable it in your organization's GitHub App settings." ); } - 401 => { + InstallationLookup::Failed(401) => { bail!( "GitHub App authentication failed. \ Check that app_id and GITHUB_APP_PRIVATE_KEY are correct." ); } - _ => { - bail!( - "Unexpected status {} looking up GitHub App installation", - resp.status - ); + InstallationLookup::Failed(status) => { + bail!("Unexpected status {status} looking up GitHub App installation"); } + }; + + mint_installation_token_for_id_with_jwt( + client, + jwt, + installation_id, + repos, + base_url, + permissions, + ) + .await +} + +/// Create a repository-scoped token for an installation already resolved by +/// the caller. +pub(crate) async fn mint_installation_token_for_id_with_jwt( + client: &impl HttpClient, + jwt: &str, + installation_id: u64, + repos: &[String], + base_url: &str, + permissions: serde_json::Value, +) -> anyhow::Result { + #[derive(Deserialize)] + struct AccessToken { + token: String, + expires_at: DateTime, } - let installation: Installation = resp - .json() - .context("Failed to parse installation response")?; - - // Step 2: Create a scoped access token - let token_url = format!( - "{base_url}/app/installations/{}/access_tokens", - installation.id - ); + let auth = format!("Bearer {jwt}"); + let token_url = format!("{base_url}/app/installations/{installation_id}/access_tokens"); let body = serde_json::json!({ - "repositories": [repo], + "repositories": repos, "permissions": permissions, }); @@ -573,8 +686,9 @@ async fn mint_installation_token_with_jwt( 201 => {} 422 => { bail!( - "GitHub App does not have access to repository {repo}. \ - Update the installation's repository permissions to include it." + "GitHub App does not have access to every requested repository ({}). \ + Update the installation's repository permissions to include them.", + repos.join(", ") ); } 401 => { @@ -1065,25 +1179,24 @@ pub async fn check_app_installed( repo: &str, base_url: &str, ) -> anyhow::Result { - let url = format!("{base_url}/repos/{owner}/{repo}/installation"); - let auth = format!("Bearer {jwt}"); - let resp = client - .request(HttpMethod::Get, &url, &github_headers(&auth), None) + let lookup = lookup_installation(client, jwt, base_url, owner, repo) .await .context("Failed to check GitHub App installation")?; - match resp.status { - 200 => Ok(true), - 404 => Ok(false), - 401 => bail!( + match lookup { + InstallationLookup::Found(_) => Ok(true), + InstallationLookup::NotFound => Ok(false), + InstallationLookup::Failed(401) => bail!( "GitHub App authentication failed. \ Check that app_id and GITHUB_APP_PRIVATE_KEY are correct." ), - 403 => bail!( + InstallationLookup::Failed(403) => bail!( "GitHub App installation is suspended. \ Re-enable it in your organization's GitHub App settings." ), - status => bail!("Unexpected status {status} checking GitHub App installation"), + InstallationLookup::Failed(status) => { + bail!("Unexpected status {status} checking GitHub App installation") + } } } @@ -1715,7 +1828,7 @@ mod tests { // ----------------------------------------------------------------------- fn test_rsa_key() -> &'static str { - include_str!("testdata/rsa_private.pem") + tests_mock::test_rsa_key() } #[test] @@ -1769,89 +1882,7 @@ mod tests { // MockHttpClient // ----------------------------------------------------------------------- - struct MockRoute { - method: HttpMethod, - path: String, - status: u16, - response_body: String, - assert_header: Option<(String, MockHeaderCheck)>, - assert_body_json: Option, - } - - enum MockHeaderCheck { - Equals(String), - } - - struct MockHttpClient { - routes: Vec, - } - - impl MockHttpClient { - fn new() -> Self { - Self { routes: vec![] } - } - - fn on(mut self, method: HttpMethod, path: &str, status: u16, body: &str) -> Self { - self.routes.push(MockRoute { - method, - path: path.to_string(), - status, - response_body: body.to_string(), - assert_header: None, - assert_body_json: None, - }); - self - } - - fn with_req_header(mut self, name: &str, value: &str) -> Self { - self.routes.last_mut().unwrap().assert_header = - Some((name.to_string(), MockHeaderCheck::Equals(value.to_string()))); - self - } - - fn with_req_body(mut self, json_str: &str) -> Self { - self.routes.last_mut().unwrap().assert_body_json = - Some(serde_json::from_str(json_str).unwrap()); - self - } - } - - impl HttpClient for MockHttpClient { - async fn request( - &self, - method: HttpMethod, - url: &str, - headers: &[(&str, &str)], - body: Option<&serde_json::Value>, - ) -> anyhow::Result { - for route in &self.routes { - if method == route.method && url.ends_with(&route.path) { - if let Some((name, MockHeaderCheck::Equals(expected))) = &route.assert_header { - let (_, v) = headers - .iter() - .find(|(k, _)| *k == name.as_str()) - .unwrap_or_else(|| { - panic!("Expected header '{name}' not found in request to {url}") - }); - assert_eq!(*v, expected.as_str(), "Header '{name}' mismatch for {url}"); - } - if let Some(expected_body) = &route.assert_body_json { - let actual = body.expect("Expected request body"); - assert_eq!(actual, expected_body, "Request body mismatch for {url}"); - } - return Ok(HttpResponse::new(route.status, route.response_body.clone())); - } - } - panic!( - "No mock route for {:?} {url}\nRegistered routes: {:?}", - method, - self.routes - .iter() - .map(|r| format!("{:?} {}", r.method, r.path)) - .collect::>() - ); - } - } + use crate::tests_mock::{self, MockHttpClient}; // ----------------------------------------------------------------------- // create_installation_access_token — success @@ -1901,6 +1932,62 @@ mod tests { ); } + /// The multi-repository mint sends one request listing every projected + /// repository name exactly once, primary first, with the shared + /// permissions; the installation lookup uses the primary repository. + #[tokio::test] + async fn multi_repository_mint_lists_every_repository_name_once() { + let access = GitHubRepositoryAccess::new( + Some("git@github.com:owner/repo.git"), + &[ + "owner/keystone".parse().unwrap(), + "owner/arc".parse().unwrap(), + ] + .into_iter() + .collect(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap() + .expect("origin should produce an access value"); + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/owner/repo/installation", + 200, + r#"{"id": 123}"#, + ) + .on( + HttpMethod::Post, + "/app/installations/123/access_tokens", + 201, + r#"{"token": "ghs_multi", "expires_at": "2026-01-01T12:00:00Z"}"#, + ) + .with_req_body( + r#"{"permissions":{"contents":"read"},"repositories":["repo","arc","keystone"]}"#, + ); + + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + + let token = creds + .mint_installation_token_for_repositories( + &mock, + access.owner(), + &access.repository_names(), + "", + access.permissions_json().unwrap(), + None, + ) + .await + .unwrap(); + + assert_eq!(token.token, "ghs_multi"); + } + #[tokio::test] async fn create_iat_requests_only_contents_write() { let mock = MockHttpClient::new() diff --git a/lib/components/fabro-github/src/tests_mock.rs b/lib/components/fabro-github/src/tests_mock.rs new file mode 100644 index 000000000..d0f344de3 --- /dev/null +++ b/lib/components/fabro-github/src/tests_mock.rs @@ -0,0 +1,104 @@ +//! Crate-internal test doubles shared by the `lib.rs` and `access` test +//! modules: a scripted [`HttpClient`] and a throwaway RSA key for JWT +//! signing. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crate::{HttpClient, HttpMethod, HttpResponse}; + +pub(crate) fn test_rsa_key() -> &'static str { + include_str!("testdata/rsa_private.pem") +} + +pub(crate) struct MockRoute { + method: HttpMethod, + path: String, + status: u16, + response_body: String, + assert_header: Option<(String, MockHeaderCheck)>, + assert_body_json: Option, +} + +pub(crate) enum MockHeaderCheck { + Equals(String), +} + +pub(crate) struct MockHttpClient { + routes: Vec, + request_count: AtomicUsize, +} + +impl MockHttpClient { + pub(crate) fn new() -> Self { + Self { + routes: vec![], + request_count: AtomicUsize::new(0), + } + } + + pub(crate) fn on(mut self, method: HttpMethod, path: &str, status: u16, body: &str) -> Self { + self.routes.push(MockRoute { + method, + path: path.to_string(), + status, + response_body: body.to_string(), + assert_header: None, + assert_body_json: None, + }); + self + } + + pub(crate) fn with_req_header(mut self, name: &str, value: &str) -> Self { + self.routes.last_mut().unwrap().assert_header = + Some((name.to_string(), MockHeaderCheck::Equals(value.to_string()))); + self + } + + pub(crate) fn with_req_body(mut self, json_str: &str) -> Self { + self.routes.last_mut().unwrap().assert_body_json = + Some(serde_json::from_str(json_str).unwrap()); + self + } + + pub(crate) fn request_count(&self) -> usize { + self.request_count.load(Ordering::SeqCst) + } +} + +impl HttpClient for MockHttpClient { + async fn request( + &self, + method: HttpMethod, + url: &str, + headers: &[(&str, &str)], + body: Option<&serde_json::Value>, + ) -> anyhow::Result { + self.request_count.fetch_add(1, Ordering::SeqCst); + for route in &self.routes { + if method == route.method && url.ends_with(&route.path) { + if let Some((name, MockHeaderCheck::Equals(expected))) = &route.assert_header { + let (_, v) = headers + .iter() + .find(|(k, _)| *k == name.as_str()) + .unwrap_or_else(|| { + panic!("Expected header '{name}' not found in request to {url}") + }); + assert_eq!(*v, expected.as_str(), "Header '{name}' mismatch for {url}"); + } + if let Some(expected_body) = &route.assert_body_json { + let actual = body.expect("Expected request body"); + assert_eq!(actual, expected_body, "Request body mismatch for {url}"); + } + return Ok(HttpResponse::new(route.status, route.response_body.clone())); + } + } + panic!( + "No mock route for {:?} {url}\nRegistered routes: {:?}", + method, + self.routes + .iter() + .map(|r| format!("{:?} {}", r.method, r.path)) + .collect::>() + ); + } +} diff --git a/lib/components/fabro-github/src/token_source.rs b/lib/components/fabro-github/src/token_source.rs index 276c5f9b8..6e67f3423 100644 --- a/lib/components/fabro-github/src/token_source.rs +++ b/lib/components/fabro-github/src/token_source.rs @@ -15,7 +15,7 @@ use std::fmt; use std::sync::Arc; use std::time::Duration; -use anyhow::Context as _; +use anyhow::{Context as _, bail}; use chrono::{DateTime, Utc}; use tokio::sync::Mutex; @@ -146,29 +146,54 @@ pub(crate) trait InstallationTokenMinter: Send + Sync { async fn mint(&self) -> anyhow::Result; } +/// Repository scope an App-backed source owns. +enum AppTokenScope { + /// A single-repository source that resolves its installation during each + /// mint. + Repository { + owner: String, + repo: String, + permissions: serde_json::Value, + }, + /// A validated declared set. Each mint resolves every target to one App + /// installation before creating the token. + Access(crate::GitHubRepositoryAccess), +} + /// Real minter backed by GitHub App credentials. struct AppTokenMinter { - creds: GitHubAppCredentials, - http: fabro_http::HttpClient, - owner: String, - repo: String, - base_url: String, - permissions: serde_json::Value, + creds: GitHubAppCredentials, + http: fabro_http::HttpClient, + base_url: String, + scope: AppTokenScope, } #[async_trait::async_trait] impl InstallationTokenMinter for AppTokenMinter { async fn mint(&self) -> anyhow::Result { - self.creds - .mint_installation_token( - &self.http, - &self.owner, - &self.repo, - &self.base_url, - self.permissions.clone(), - None, - ) - .await + match &self.scope { + AppTokenScope::Repository { + owner, + repo, + permissions, + } => { + self.creds + .mint_installation_token( + &self.http, + owner, + repo, + &self.base_url, + permissions.clone(), + None, + ) + .await + } + AppTokenScope::Access(access) => { + access + .mint_installation_token(&self.creds, &self.http, &self.base_url) + .await + } + } } } @@ -220,6 +245,17 @@ pub struct InstallationTokenSource { state: SourceState, } +fn repository_set_display(owner: &str, repos: &[String]) -> anyhow::Result { + match repos { + [primary] => Ok(format!("{owner}/{primary}")), + [primary, additional @ ..] => Ok(format!( + "{owner}/{primary} (+{} additional)", + additional.len() + )), + [] => bail!("token source requires at least one repository"), + } +} + impl InstallationTokenSource { /// Build a source for `creds` against the repository in `origin_url`. /// @@ -244,6 +280,32 @@ impl InstallationTokenSource { permissions: serde_json::Value, ) -> anyhow::Result> { let repo_display = format!("{owner}/{repo}"); + Self::with_app_scope(creds, repo_display, AppTokenScope::Repository { + owner, + repo, + permissions, + }) + } + + /// Build a source for a validated effective repository set. Minted + /// tokens are scoped to every repository in the set with the shared + /// permissions. App-backed sources also resolve every repository to one + /// shared installation before each mint. Caching, refresh margin, and + /// single-flight behavior are identical to the single-repository source. + pub fn for_access( + creds: &GitHubCredentials, + access: &crate::GitHubRepositoryAccess, + ) -> anyhow::Result> { + let repository_names = access.repository_names(); + let repo_display = repository_set_display(access.owner(), &repository_names)?; + Self::with_app_scope(creds, repo_display, AppTokenScope::Access(access.clone())) + } + + fn with_app_scope( + creds: &GitHubCredentials, + repo_display: String, + scope: AppTokenScope, + ) -> anyhow::Result> { let state = match creds { GitHubCredentials::Pat(token) => SourceState::Pat(SecretString::new(token.clone())), GitHubCredentials::Installation(token) => SourceState::Installation(token.clone()), @@ -255,10 +317,8 @@ impl InstallationTokenSource { minter: Box::new(AppTokenMinter { creds: app.clone(), http, - owner, - repo, base_url: crate::github_api_base_url(), - permissions, + scope, }), cache: Mutex::new(None), } @@ -502,6 +562,31 @@ mod tests { assert!(!source.mints_installation_tokens()); } + /// A source built from a validated multi-repository access value uses the + /// same state machine as the single-repository constructor: static + /// credentials pass through, and App credentials share the cache + /// machinery exercised by the `with_minter` tests below. + #[tokio::test] + async fn for_access_source_resolves_like_the_single_repository_source() { + let access = crate::GitHubRepositoryAccess::new( + Some("https://github.com/owner/repo.git"), + &["owner/keystone".parse().unwrap()].into_iter().collect(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap() + .expect("origin should produce an access value"); + + let source = InstallationTokenSource::for_access( + &GitHubCredentials::Pat("ghp_pat".to_string()), + &access, + ) + .unwrap(); + + let resolved = source.resolve().await.unwrap(); + assert_eq!(resolved.token.expose(), "ghp_pat"); + assert!(resolved.snapshot.is_static()); + } + #[tokio::test] async fn static_installation_token_resolves_until_expiry() { let valid = InstallationTokenSource::for_origin( diff --git a/lib/components/fabro-github/tests/live_access.rs b/lib/components/fabro-github/tests/live_access.rs new file mode 100644 index 000000000..0b13a582c --- /dev/null +++ b/lib/components/fabro-github/tests/live_access.rs @@ -0,0 +1,122 @@ +//! Opt-in live GitHub App test for additional-repository access. +//! +//! Verifies against the real GitHub API that one installation token scoped +//! to the primary repository plus one declared additional repository grants +//! Git read access to both. Runs only in live mode with these variables set +//! (it skips clearly otherwise): +//! +//! - `FABRO_TEST_GITHUB_APP_ID` — GitHub App id +//! - `GITHUB_APP_PRIVATE_KEY` — App private key (PEM, or base64-encoded PEM) +//! - `FABRO_TEST_GITHUB_ORIGIN` — HTTPS origin URL of the primary repository +//! - `FABRO_TEST_GITHUB_ADDITIONAL_REPO` — an `owner/repository` slug the +//! installation can see, ideally private, sharing the origin's owner +//! +//! The repositories come from the environment so no private slug is baked +//! into durable test output, and the minted token is only ever passed to +//! `git` through the child process environment. + +use std::collections::BTreeSet; +use std::process::Stdio; +use std::time::Duration; + +use fabro_github::token_source::InstallationTokenSource; +use fabro_github::{GitHubAppCredentials, GitHubCredentials, GitHubRepositoryAccess}; +use fabro_types::GitHubRepositorySlug; +use tokio::process::Command; +use tokio::time::sleep; + +fn env_var(name: &str) -> String { + #[expect( + clippy::disallowed_methods, + reason = "live e2e configuration comes from the process environment by design" + )] + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for this live test")) +} + +async fn ls_remote_with_token(slug: &GitHubRepositorySlug, token: &str) -> bool { + let url = slug.https_url(); + let mut command = Command::new("git"); + fabro_github::apply_probe_git_env(&mut command, token); + let output = command + .args(["ls-remote", &url, "HEAD"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .output() + .await + .expect("git should run"); + output.status.success() +} + +#[fabro_macros::e2e_test( + live("FABRO_TEST_GITHUB_APP_ID"), + live("GITHUB_APP_PRIVATE_KEY"), + live("FABRO_TEST_GITHUB_ORIGIN"), + live("FABRO_TEST_GITHUB_ADDITIONAL_REPO") +)] +async fn scoped_token_reaches_the_declared_additional_repository() { + let app_id = env_var("FABRO_TEST_GITHUB_APP_ID"); + let app = GitHubAppCredentials::from_env(Some(&app_id)) + .expect("GITHUB_APP_PRIVATE_KEY should decode as PEM or base64 PEM") + .expect("GITHUB_APP_PRIVATE_KEY must be set for this live test"); + let origin = env_var("FABRO_TEST_GITHUB_ORIGIN"); + let additional: GitHubRepositorySlug = env_var("FABRO_TEST_GITHUB_ADDITIONAL_REPO") + .parse() + .expect("FABRO_TEST_GITHUB_ADDITIONAL_REPO must be an owner/repository slug"); + let additional_set: BTreeSet = [additional.clone()].into_iter().collect(); + + let access = GitHubRepositoryAccess::new( + Some(&origin), + &additional_set, + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .expect("access request should validate") + .expect("origin should produce an access value"); + + // The production choreography: every target resolves to one shared + // installation, then one mint scoped to the whole effective set. + let creds = GitHubCredentials::App(app.clone()); + let source = + InstallationTokenSource::for_access(&creds, &access).expect("token source should build"); + let resolved = source.resolve().await.expect("scoped mint should succeed"); + let token = resolved.token.expose(); + + // The one token reads both the primary and the additional repository. + // A freshly minted token can hit GitHub's replication lag, so retry a + // few times with the same token before failing. + for slug in access.targets() { + let mut reachable = false; + for _ in 0..3 { + if ls_remote_with_token(slug, token).await { + reachable = true; + break; + } + sleep(Duration::from_secs(2)).await; + } + assert!( + reachable, + "scoped token should read every declared repository" + ); + } + + // Negative scope check: a token minted for the primary alone must not + // read the additional repository (proves server-side scoping, not just + // possession of a token). + let primary_only = GitHubRepositoryAccess::new( + Some(&origin), + &BTreeSet::new(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .expect("primary-only access should validate") + .expect("origin should produce an access value"); + let narrow_source = + InstallationTokenSource::for_access(&GitHubCredentials::App(app), &primary_only) + .expect("primary-only token source should build"); + let narrow = narrow_source + .resolve() + .await + .expect("primary-only mint should succeed"); + assert!( + !ls_remote_with_token(&additional, narrow.token.expose()).await, + "a primary-only token must not read the additional repository" + ); +} diff --git a/lib/components/fabro-llm/src/adapter_registry.rs b/lib/components/fabro-llm/src/adapter_registry.rs index a306c846b..60ad8ec42 100644 --- a/lib/components/fabro-llm/src/adapter_registry.rs +++ b/lib/components/fabro-llm/src/adapter_registry.rs @@ -269,91 +269,19 @@ mod tests { .unwrap_or_else(|error| panic!("built-in model '{selector}' should resolve: {error}")) } - /// One row of the route-equivalence table: model id plus the - /// `(deployment_id, transport, codec, billing_policy, agent_profile)` - /// tuple it must resolve to. - type RouteRow = ( - &'static str, - &'static str, - AdapterKind, - CodecKind, - BillingPolicy, - AgentProfileKind, - ); - - /// The compat mapping as an executable table: every built-in catalog - /// model resolves to exactly this tuple. Adding or rerouting a built-in - /// model means updating this table deliberately. #[test] - fn builtin_catalog_route_equivalence_table() { - use AdapterKind as T; - use AgentProfileKind as P; - use BillingPolicy as B; - use CodecKind as C; - - #[rustfmt::skip] - let expected: &[RouteRow] = &[ - // model id deployment_id transport codec billing profile - ("claude-fable-5", "claude-fable-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("claude-haiku-4-5", "claude-haiku-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-6", "claude-opus-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-7", "claude-opus-4-7", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-8", "claude-opus-4-8", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-5", "claude-opus-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("claude-sonnet-4-5", "claude-sonnet-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-sonnet-4-6", "claude-sonnet-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-sonnet-5", "claude-sonnet-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("deepseek-v4-flash", "deepseek-v4-flash", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("deepseek-v4-pro", "deepseek-v4-pro", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("gemini-3-flash-preview", "gemini-3-flash-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-flash-lite", "gemini-3.1-flash-lite", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-pro-preview", "gemini-3.1-pro-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-pro-preview-customtools", "gemini-3.1-pro-preview-customtools", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.5-flash", "gemini-3.5-flash", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("glm-4.7", "glm-4.7", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("glm-5.2", "glm-5.2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("gpt-5.4", "gpt-5.4", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.4-mini", "gpt-5.4-mini", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.4-pro", "gpt-5.4-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.5", "gpt-5.5", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.5-pro", "gpt-5.5-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.6-luna", "gpt-5.6-luna", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("gpt-5.6-sol", "gpt-5.6-sol", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("gpt-5.6-terra", "gpt-5.6-terra", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("kimi-k2.5", "kimi-k2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi), - ("kimi-k3", "kimi-k3", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi), - ("laguna-s-2.1", "poolside/laguna-s-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("laguna-xs-2.1", "poolside/laguna-xs-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("mercury-2", "mercury-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("minimax-m2.5", "minimax-m2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("venice-uncensored-1-2", "venice-uncensored-1-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("venice-uncensored-role-play", "venice-uncensored-role-play", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ]; - + fn every_builtin_catalog_offering_resolves() { let catalog = Catalog::builtin(); - let mut model_ids: Vec<&str> = catalog - .list(None) - .iter() - .map(|model| model.id.as_str()) - .collect(); - model_ids.sort_unstable(); - let mut expected_ids: Vec<&str> = expected.iter().map(|row| row.0).collect(); - expected_ids.sort_unstable(); - assert_eq!( - model_ids, expected_ids, - "route-equivalence table must cover every built-in model row" - ); - - for (model_id, deployment_id, transport, codec, billing_policy, agent_profile) in expected { - let model = select_from_all(catalog, model_id); - let route = resolve_route(catalog, model) - .unwrap_or_else(|| panic!("built-in model '{model_id}' should resolve")); - assert_eq!(route.deployment_id, *deployment_id, "{model_id}"); - assert_eq!(route.transport, *transport, "{model_id}"); - assert_eq!(route.codec, *codec, "{model_id}"); - assert_eq!(route.billing_policy, *billing_policy, "{model_id}"); - assert_eq!(route.agent_profile, *agent_profile, "{model_id}"); + for model in catalog.list(None) { + let route = resolve_route(catalog, model).unwrap_or_else(|| { + panic!( + "built-in offering '{}/{}' should resolve", + model.provider, model.id + ) + }); + assert_eq!(route.provider, model.provider); + assert!(!route.deployment_id.is_empty()); } } diff --git a/lib/components/fabro-llm/src/client.rs b/lib/components/fabro-llm/src/client.rs index fc2802be1..59cfa421d 100644 --- a/lib/components/fabro-llm/src/client.rs +++ b/lib/components/fabro-llm/src/client.rs @@ -2037,17 +2037,20 @@ reasoning = false client } - /// Live-dispatch counterpart of the adapter_registry route-equivalence - /// table: for every built-in model, `resolve_provider` lands on the same - /// provider the resolved route names. + /// For every built-in model selector, live dispatch and catalog selection + /// choose the same provider from the same ready-provider set. #[tokio::test] async fn dispatch_agrees_with_resolve_route_for_every_builtin_model() { let catalog = catalog_with(""); let client = client_with_all_catalog_providers(&catalog).await; + let ready_providers = catalog.all_provider_ids(); for model in catalog.list(None) { - let route = adapter_registry::resolve_route(&catalog, model) - .expect("built-in model should resolve to a route"); + let selected = catalog + .select(model.id.as_str(), None, &ready_providers) + .expect("built-in model should be selectable"); + let route = adapter_registry::resolve_route(&catalog, selected) + .expect("selected built-in model should resolve to a route"); let mut request = test_request(); request.model = model.id.to_string(); diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/response.rs b/lib/components/fabro-llm/src/codec/openai_compatible/response.rs index f7df78729..dc2779bf1 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/response.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/response.rs @@ -63,7 +63,9 @@ pub(super) fn decode_response( let wire_usage = api_resp.usage.as_ref(); let usage = wire_usage.map_or_else(TokenCounts::default, ApiUsage::token_counts); - let cost_usd = wire_usage.and_then(|u| u.cost); + let cost_usd = wire_usage + .and_then(|usage| usage.cost) + .or_else(|| api_resp.cost.as_ref().and_then(|cost| cost.usd)); let cost_source = translate::authoritative_cost_source(cost_usd); Ok(Response { diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs b/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs index f389ee086..bb2633b91 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs @@ -29,8 +29,8 @@ pub(super) struct StreamState { /// True after `finish_events()` has run (guards against duplicates). finished: bool, rate_limit: Option, - /// In-band USD cost from the usage chunk (OpenRouter), surfaced as - /// authoritative on the final response. + /// In-band USD cost from the response, surfaced as authoritative on the + /// final response. cost_usd: Option, } @@ -72,9 +72,13 @@ impl StreamState { // Capture usage if present (often in a dedicated chunk). if let Some(usage) = &chunk.usage { self.usage = usage.token_counts(); - // Keep a previously seen cost when a later usage chunk omits it. - self.cost_usd = usage.cost.or(self.cost_usd); } + let cost_usd = chunk + .usage + .as_ref() + .and_then(|usage| usage.cost) + .or_else(|| chunk.cost.as_ref().and_then(|cost| cost.usd)); + self.cost_usd = cost_usd.or(self.cost_usd); let choices = chunk.choices.as_mut()?; let choice = choices.first_mut()?; diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs b/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs index 62c6ede6b..83823d5f7 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs @@ -125,6 +125,12 @@ pub(super) struct ApiResponse { pub model: String, pub choices: Vec, pub usage: Option, + pub cost: Option, +} + +#[derive(serde::Deserialize)] +pub(super) struct ApiCost { + pub usd: Option, } #[derive(serde::Deserialize)] @@ -377,6 +383,7 @@ pub(super) struct StreamChunk { pub model: Option, pub choices: Option>, pub usage: Option, + pub cost: Option, } #[derive(serde::Deserialize)] diff --git a/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs b/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs index 0ff337cc5..22615dcd6 100644 --- a/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +++ b/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs @@ -795,6 +795,31 @@ async fn decode_usage_openrouter_cost_and_cache_write() { fabro_test::fabro_json_snapshot!(response); } +/// Venice reports authoritative USD cost in a top-level object rather than +/// the OpenRouter `usage.cost` field. +#[tokio::test] +async fn decode_usage_venice_top_level_cost() { + let response = decode_response(serde_json::json!({ + "id": "chatcmpl_venice_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "cost": {"usd": 0.00042, "diem": 0.0}, + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14 + } + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + // --------------------------------------------------------------------------- // Stream // --------------------------------------------------------------------------- @@ -841,6 +866,20 @@ async fn stream_usage_openrouter_cost() { fabro_test::fabro_json_snapshot!(events); } +/// Venice streams authoritative USD cost in a top-level object on the usage +/// chunk. +#[tokio::test] +async fn stream_usage_venice_top_level_cost() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"cost":{"usd":0.00031,"diem":0.0},"usage":{"prompt_tokens":12,"completion_tokens":2,"total_tokens":14}}"#, + "[DONE]", + ]); + let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + #[tokio::test] async fn stream_tool_call_deltas() { let sse = support::sse_data_transcript(&[ diff --git a/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap new file mode 100644 index 000000000..7236fc7c8 --- /dev/null +++ b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap @@ -0,0 +1,55 @@ +--- +source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_venice_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "ok" + } + ] + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_venice_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "ok" + }, + "finish_reason": "stop" + } + ], + "cost": { + "usd": 0.00042, + "diem": 0.0 + }, + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14 + } + }, + "warnings": [], + "rate_limit": null, + "cost_usd": 0.00042, + "cost_source": "authoritative" +} diff --git a/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap new file mode 100644 index 000000000..b2ac8b212 --- /dev/null +++ b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap @@ -0,0 +1,60 @@ +--- +source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Hi", + "text_id": null + }, + { + "type": "text_end", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_venice_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hi" + } + ] + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null, + "cost_usd": 0.00031, + "cost_source": "authoritative" + } + } +] diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index 1a90c6b21..9f03a93e3 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -8,6 +8,7 @@ pub(crate) enum CloneDecision { GitHub { origin_url: String, branch: Option, + commit_sha: Option, }, } @@ -71,6 +72,98 @@ pub(crate) fn repo_symlink_command(layout: &GitHubRepoLayout) -> String { ) } +pub(crate) fn exact_repository_init_command(clone_url: &str, checkout_path: &str) -> String { + format!( + "{git} init -- {path} && git -C {path} remote add origin {origin}", + git = sandbox::GIT, + path = sandbox::shell_quote(checkout_path), + origin = sandbox::shell_quote(clone_url), + ) +} + +/// Fetch a single admitted commit with the same history depth a branch clone +/// gets, so both paths can reach the same number of parent commits. +/// +/// The fetch names the commit directly rather than the branch: reachability of +/// the commit from the admitted branch is an admission-time invariant, not +/// something this layer re-verifies. +pub(crate) fn exact_fetch_command( + checkout_path: &str, + fetch_source: &str, + commit_sha: &str, + depth: Option, +) -> String { + let depth_arg = depth_argument(depth); + format!( + "{git} -C {} fetch{depth_arg} --no-tags {} -- {}", + sandbox::shell_quote(checkout_path), + sandbox::shell_quote(fetch_source), + sandbox::shell_quote(commit_sha), + git = sandbox::GIT, + ) +} + +/// Leading-space ` --depth N` fragment for a Git command, or empty when +/// `depth` is `None` to fetch full history. +pub(crate) fn depth_argument(depth: Option) -> String { + depth.map_or_else(String::new, |depth| format!(" --depth {depth}")) +} + +/// Point the admitted branch at `revision` and attach HEAD to it. +/// +/// The checkout attaches to a real branch instead of detaching so callers that +/// read the current branch back out of the workspace still see the admitted +/// branch name. +pub(crate) fn exact_branch_checkout_command( + checkout_path: &str, + branch: &str, + revision: &str, +) -> String { + format!( + "{git} -C {path} checkout -B {branch} {revision}", + path = sandbox::shell_quote(checkout_path), + branch = sandbox::shell_quote(branch), + revision = sandbox::shell_quote(revision), + git = sandbox::GIT, + ) +} + +/// Print the current HEAD commit and nothing else, for [`verify_exact_head`]. +pub(crate) fn exact_head_revision_command(checkout_path: &str) -> String { + format!( + "{git} -C {path} rev-parse HEAD", + path = sandbox::shell_quote(checkout_path), + git = sandbox::GIT, + ) +} + +/// Check out the admitted branch and print the resulting HEAD in one shell +/// command; stdout is the `rev-parse HEAD` output for [`verify_exact_head`]. +pub(crate) fn exact_checkout_verify_command( + checkout_path: &str, + branch: &str, + revision: &str, +) -> String { + format!( + "{} && {}", + exact_branch_checkout_command(checkout_path, branch, revision), + exact_head_revision_command(checkout_path), + ) +} + +pub(crate) fn verify_exact_head(output: &str, expected_sha: &str) -> crate::Result<()> { + let actual_sha = output.trim(); + let actual_sha = normalize_exact_commit_sha(actual_sha).map_err(|err| { + crate::Error::context("Exact checkout produced an invalid HEAD commit ID", err) + })?; + if actual_sha != expected_sha { + return Err(crate::Error::message( + "Exact checkout HEAD did not match the requested commit", + )); + } + Ok(()) +} + fn trim_root(root: &str) -> &str { let trimmed = root.trim_end_matches('/'); if trimmed.is_empty() { "/" } else { trimmed } @@ -97,7 +190,33 @@ pub(crate) fn decide_clone( skip_clone: bool, clone_origin_url: Option<&str>, clone_branch: Option<&str>, + clone_commit_sha: Option<&str>, ) -> crate::Result { + let commit_sha = clone_commit_sha + .map(normalize_exact_commit_sha) + .transpose()?; + + if commit_sha.is_some() { + if skip_clone { + return Err(crate::Error::message( + "Exact commit checkout requires cloning to be enabled", + )); + } + if clone_origin_url.is_none_or(|url| url.trim().is_empty()) { + return Err(crate::Error::message( + "Exact commit checkout requires a repository origin", + )); + } + // The branch names the checkout the run works on; it is not used to + // constrain which commits may be fetched. Admission is responsible for + // proving the commit belongs to the branch before it reaches here. + if clone_branch.is_none_or(|branch| branch.trim().is_empty()) { + return Err(crate::Error::message( + "Exact commit checkout requires a repository branch", + )); + } + } + if skip_clone { return Ok(CloneDecision::EmptyWorkspace { reason: EmptyWorkspaceReason::SkipClone, @@ -122,9 +241,19 @@ pub(crate) fn decide_clone( branch: clone_branch .filter(|branch| !branch.trim().is_empty()) .map(str::to_string), + commit_sha, }) } +fn normalize_exact_commit_sha(commit_sha: &str) -> crate::Result { + if commit_sha.len() != 40 || !commit_sha.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(crate::Error::message( + "Exact commit SHA must be exactly 40 ASCII hexadecimal characters", + )); + } + Ok(commit_sha.to_ascii_lowercase()) +} + pub(crate) fn clean_clone_origin_for_record(clone_origin_url: Option<&str>) -> Option { clone_origin_url .filter(|url| !url.trim().is_empty()) @@ -136,22 +265,73 @@ pub(crate) fn repo_cloned_for_record( clone_origin_url: Option<&str>, ) -> Option { Some(matches!( - decide_clone(skip_clone, clone_origin_url, None).ok()?, + decide_clone(skip_clone, clone_origin_url, None, None).ok()?, CloneDecision::GitHub { .. } )) } #[cfg(test)] mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + use super::*; + fn isolated_command(command: &mut Command) -> Output { + command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_AUTHOR_NAME", "Fabro Test") + .env("GIT_AUTHOR_EMAIL", "fabro-test@example.com") + .env("GIT_COMMITTER_NAME", "Fabro Test") + .env("GIT_COMMITTER_EMAIL", "fabro-test@example.com") + .output() + .expect("test command should start") + } + + #[expect( + clippy::disallowed_methods, + reason = "hermetic Git proof intentionally runs the local git executable synchronously" + )] + fn run_git(cwd: &Path, args: &[&str]) -> String { + let output = isolated_command(Command::new("git").current_dir(cwd).args(args)); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("git output should be UTF-8") + } + + #[expect( + clippy::disallowed_methods, + reason = "hermetic command-builder proof intentionally runs local Bash synchronously" + )] + fn run_shell(cwd: &Path, command: &str) -> String { + let output = isolated_command(Command::new("/bin/bash").current_dir(cwd).args([ + "--noprofile", + "--norc", + "-c", + command, + ])); + assert!( + output.status.success(), + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("command output should be UTF-8") + } + #[test] fn skip_clone_overrides_present_origin() { assert_eq!( decide_clone( true, Some("https://gitlab.com/acme/widgets.git"), - Some("main") + Some("main"), + None, ) .unwrap(), CloneDecision::EmptyWorkspace { @@ -163,7 +343,7 @@ mod tests { #[test] fn missing_origin_creates_empty_workspace() { assert_eq!( - decide_clone(false, None, None).unwrap(), + decide_clone(false, None, None, None).unwrap(), CloneDecision::EmptyWorkspace { reason: EmptyWorkspaceReason::MissingOrigin, } @@ -176,23 +356,257 @@ mod tests { decide_clone( false, Some("git@github.com:acme/widgets.git"), - Some("feature/work") + Some("feature/work"), + None, ) .unwrap(), CloneDecision::GitHub { origin_url: "https://github.com/acme/widgets".to_string(), branch: Some("feature/work".to_string()), + commit_sha: None, } ); } #[test] fn non_github_origin_fails_without_skip_clone() { - let error = decide_clone(false, Some("https://gitlab.com/acme/widgets.git"), None) - .expect_err("non-GitHub origins should fail"); + let error = decide_clone( + false, + Some("https://gitlab.com/acme/widgets.git"), + None, + None, + ) + .expect_err("non-GitHub origins should fail"); assert!(error.to_string().contains("GitHub repository origins only")); } + #[test] + fn exact_commit_sha_is_validated_and_normalized() { + let lowercase = "0123456789abcdef0123456789abcdef01234567"; + let uppercase = "ABCDEF0123456789ABCDEF0123456789ABCDEF01"; + + assert_eq!( + decide_clone( + false, + Some("https://github.com/acme/widgets"), + Some("moving-branch"), + Some(lowercase), + ) + .unwrap(), + CloneDecision::GitHub { + origin_url: "https://github.com/acme/widgets".to_string(), + branch: Some("moving-branch".to_string()), + commit_sha: Some(lowercase.to_string()), + } + ); + assert_eq!( + decide_clone( + false, + Some("https://github.com/acme/widgets"), + Some("main"), + Some(uppercase), + ) + .unwrap(), + CloneDecision::GitHub { + origin_url: "https://github.com/acme/widgets".to_string(), + branch: Some("main".to_string()), + commit_sha: Some(uppercase.to_ascii_lowercase()), + } + ); + } + + #[test] + fn exact_commit_sha_rejects_noncanonical_inputs() { + for sha in [ + "", + "0123456789abcdef0123456789abcdef0123456", + "0123456789abcdef0123456789abcdef012345678", + "0123456789abcdef0123456789abcdef0123456g", + " 0123456789abcdef0123456789abcdef01234567", + "0123456789abcdef0123456789abcdef01234567 ", + "0123456789abcdef0123456789abcdef012345é", + ] { + let error = decide_clone( + false, + Some("https://github.com/acme/widgets"), + None, + Some(sha), + ) + .expect_err("invalid exact commit SHA should fail"); + assert!( + error.to_string().contains("40 ASCII hexadecimal"), + "unexpected error for {sha:?}: {error}" + ); + } + } + + #[test] + fn exact_checkout_requires_clone_origin_and_branch() { + let sha = "0123456789abcdef0123456789abcdef01234567"; + let skip_error = decide_clone( + true, + Some("https://github.com/acme/widgets"), + Some("main"), + Some(sha), + ) + .expect_err("exact checkout with skip-clone should fail"); + assert!(skip_error.to_string().contains("requires cloning")); + + for origin in [None, Some(""), Some(" ")] { + let error = decide_clone(false, origin, Some("main"), Some(sha)) + .expect_err("exact checkout without an origin should fail"); + assert!(error.to_string().contains("requires a repository origin")); + } + + for branch in [None, Some(""), Some(" ")] { + let error = decide_clone( + false, + Some("https://github.com/acme/widgets"), + branch, + Some(sha), + ) + .expect_err("exact checkout without a branch should fail"); + assert!(error.to_string().contains("requires a repository branch")); + } + } + + #[test] + fn docker_exact_checkout_commands_quote_inputs() { + let sha = "0123456789abcdef0123456789abcdef01234567"; + let init = exact_repository_init_command( + "https://token@example.com/acme/widgets.git?x=a b", + "/repos/acme's widgets", + ); + let fetch = exact_fetch_command( + "/repos/acme's widgets", + "https://token@example.com/acme/widgets.git?x=a b", + sha, + Some(10), + ); + let checkout = + exact_checkout_verify_command("/repos/acme's widgets", "feature/a b", "FETCH_HEAD"); + + assert_eq!( + init, + "git -c maintenance.auto=0 -c gc.auto=0 init -- \"/repos/acme's widgets\" && git -C \"/repos/acme's widgets\" remote add origin 'https://token@example.com/acme/widgets.git?x=a b'" + ); + assert_eq!( + fetch, + "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" fetch --depth 10 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!( + checkout, + "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" checkout -B 'feature/a b' FETCH_HEAD && git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" rev-parse HEAD" + ); + } + + #[test] + fn exact_fetch_omits_depth_for_full_history() { + assert_eq!( + exact_fetch_command( + "/repos/acme/widgets", + "origin", + "0123456789abcdef0123456789abcdef01234567", + None, + ), + "git -c maintenance.auto=0 -c gc.auto=0 -C /repos/acme/widgets fetch --no-tags origin -- 0123456789abcdef0123456789abcdef01234567" + ); + } + + #[test] + fn exact_checkout_verification_rejects_invalid_or_mismatched_head() { + let expected = "0123456789abcdef0123456789abcdef01234567"; + verify_exact_head("0123456789ABCDEF0123456789ABCDEF01234567\n", expected) + .expect("uppercase command output should normalize"); + + let invalid = verify_exact_head("fatal: not a revision", expected) + .expect_err("non-SHA output should fail verification"); + assert!(invalid.to_string().contains("invalid HEAD commit ID")); + assert!(!invalid.to_string().contains("fatal: not a revision")); + + let mismatched = verify_exact_head("1123456789abcdef0123456789abcdef01234567", expected) + .expect_err("mismatched SHA should fail verification"); + assert!(mismatched.to_string().contains("did not match")); + } + + #[test] + #[expect( + clippy::disallowed_methods, + reason = "hermetic Git proof uses isolated synchronous temp-repository I/O" + )] + fn exact_checkout_fetches_admitted_commit_after_branch_advances() { + let temp = tempfile::tempdir().expect("tempdir"); + let remote = temp.path().join("remote.git"); + let source = temp.path().join("source"); + let checkout = temp.path().join("exact checkout"); + fs::create_dir(&source).expect("source directory"); + + run_git(temp.path(), &[ + "init", + "--bare", + remote.to_str().expect("UTF-8 remote path"), + ]); + run_git(&source, &["init"]); + fs::write(source.join("revision.txt"), "A\n").expect("write commit A"); + run_git(&source, &["add", "revision.txt"]); + run_git(&source, &["commit", "-m", "commit A"]); + run_git(&source, &["branch", "-M", "main"]); + run_git(&source, &[ + "remote", + "add", + "origin", + remote.to_str().expect("UTF-8 remote path"), + ]); + run_git(&source, &["push", "-u", "origin", "main"]); + let admitted_sha = run_git(&source, &["rev-parse", "HEAD"]).trim().to_string(); + + fs::write(source.join("revision.txt"), "B\n").expect("write commit B"); + run_git(&source, &["commit", "-am", "commit B"]); + run_git(&source, &["push", "origin", "main"]); + let advanced_sha = run_git(&source, &["rev-parse", "HEAD"]).trim().to_string(); + assert_ne!(admitted_sha, advanced_sha); + + let remote_path = remote.to_str().expect("UTF-8 remote path"); + let checkout_path = checkout.to_str().expect("UTF-8 checkout path"); + run_shell( + temp.path(), + &exact_repository_init_command(remote_path, checkout_path), + ); + run_shell( + temp.path(), + &exact_fetch_command(checkout_path, remote_path, &admitted_sha, Some(10)), + ); + let checked_out_sha = run_shell( + temp.path(), + &exact_checkout_verify_command(checkout_path, "main", "FETCH_HEAD"), + ); + + assert_eq!(checked_out_sha.trim(), admitted_sha); + assert_eq!( + fs::read_to_string(checkout.join("revision.txt")).expect("checked-out contents"), + "A\n" + ); + assert_eq!( + run_git(&checkout, &["symbolic-ref", "HEAD"]).trim(), + "refs/heads/main", + "HEAD should stay attached to the admitted branch" + ); + assert_eq!( + run_git(&checkout, &["rev-parse", "--abbrev-ref", "HEAD"]).trim(), + "main" + ); + assert_eq!( + run_git(temp.path(), &[ + "--git-dir", + remote_path, + "rev-parse", + "refs/heads/main", + ],) + .trim(), + advanced_sha + ); + } + #[test] fn github_layout_maps_ssh_origin_to_repos_checkout_and_workspace_link() { let layout = github_repo_layout( diff --git a/lib/components/fabro-sandbox/src/config.rs b/lib/components/fabro-sandbox/src/config.rs index c1304db5a..208278d18 100644 --- a/lib/components/fabro-sandbox/src/config.rs +++ b/lib/components/fabro-sandbox/src/config.rs @@ -18,6 +18,9 @@ pub struct DaytonaSettings { pub labels: Option>, pub snapshot: Option, pub network: Option, + /// Git history depth for the repository clone; `None` clones full + /// history. + pub clone_depth: Option, #[serde(default)] pub skip_clone: bool, } diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 1313f4ba3..a58bdf6ec 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -14,7 +14,7 @@ use daytona_api_client::models::SandboxState; use daytona_api_client::models::api_key_list::Permissions; use daytona_sdk::api_types::SignedPortPreviewUrl; use daytona_sdk::toolbox_types::Command as SessionCommandResult; -use daytona_sdk::{DaytonaError, SessionCommandLogsResult}; +use daytona_sdk::{DaytonaError, GitCloneOptions, SessionCommandLogsResult}; use fabro_github::GitHubCredentials; use fabro_github::token_source::InstallationTokenSource; use fabro_static::EnvVars; @@ -67,6 +67,23 @@ pub(crate) const DAYTONA_DASHBOARD_SANDBOXES_URL: &str = const FABRO_SANDBOX_USER_AGENT: &str = concat!("fabro-sandbox/", env!("CARGO_PKG_VERSION")); pub const DAYTONA_CREDENTIAL_PROBE_TIMEOUT: Duration = Duration::from_secs(20); const DAYTONA_BASH_SESSION_PROBE_TIMEOUT: Duration = Duration::from_secs(20); +/// Shared budget for required setup after Daytona's native clone returns. +/// +/// The native clone has its own provider lifecycle. Starting this deadline +/// afterward prevents a slow successful clone from consuming the budget for +/// required branch attachment and workspace linking. +const DAYTONA_POST_CLONE_SETUP_TIMEOUT: Duration = Duration::from_mins(5); +/// Best-effort push-credential setup should not consume or extend the required +/// post-clone setup budget. +const DAYTONA_CREDENTIAL_SETUP_TIMEOUT: Duration = Duration::from_secs(10); +/// Grace for the toolbox to return the server-side command timeout response. +/// The SDK truncates the server timeout to whole seconds, so the server can +/// fire up to one second before the shared deadline; this additional grace +/// lets that response reach the client afterward. +const DAYTONA_CLIENT_TIMEOUT_GRACE: Duration = Duration::from_secs(1); +/// The Daytona SDK serializes command timeouts as whole seconds. Do not send a +/// zero-second timeout when the shared deadline is nearly exhausted. +const DAYTONA_MIN_SERVER_TIMEOUT: Duration = Duration::from_secs(1); /// Upper bound on explicit and Drop-triggered Daytona cleanup calls (session /// deletion, temporary stdin files) so a stalled REST call cannot block /// cancellation/timeout paths indefinitely. @@ -84,12 +101,16 @@ const DAYTONA_STATE_CHANGE_POLL_INTERVAL: Duration = Duration::from_secs(1); /// leaked by a dead worker. An explicit `0` disables auto-stop entirely. const DEFAULT_AUTO_STOP_INTERVAL_MINUTES: i32 = 120; +pub(crate) fn daytona_not_found(err: &DaytonaError) -> bool { + matches!(err, DaytonaError::NotFound { .. }) || err.status_code() == Some(404) +} + /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ - Permissions::WriteColonSnapshots, - Permissions::DeleteColonSnapshots, - Permissions::WriteColonSandboxes, - Permissions::DeleteColonSandboxes, + Permissions::WRITE_SNAPSHOTS, + Permissions::DELETE_SNAPSHOTS, + Permissions::WRITE_SANDBOXES, + Permissions::DELETE_SANDBOXES, ]; pub use crate::config::{ @@ -223,10 +244,10 @@ fn join_perms(perms: &[Permissions]) -> String { fn perm_wire_str(permission: Permissions) -> &'static str { match permission { - Permissions::WriteColonSnapshots => "write:snapshots", - Permissions::DeleteColonSnapshots => "delete:snapshots", - Permissions::WriteColonSandboxes => "write:sandboxes", - Permissions::DeleteColonSandboxes => "delete:sandboxes", + Permissions::WRITE_SNAPSHOTS => "write:snapshots", + Permissions::DELETE_SNAPSHOTS => "delete:snapshots", + Permissions::WRITE_SANDBOXES => "write:sandboxes", + Permissions::DELETE_SANDBOXES => "delete:sandboxes", _ => "unknown", } } @@ -435,6 +456,7 @@ pub struct DaytonaSandbox { /// Explicit branch to clone. When set, overrides the branch detected by /// the submitted run spec. clone_branch: Option, + clone_commit_sha: Option, } impl DaytonaSandbox { @@ -448,8 +470,17 @@ impl DaytonaSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, api_key: Option, ) -> crate::Result { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + )?; + } let api_key = resolve_daytona_api_key(api_key); let client = build_daytona_client(api_key.clone()) .await @@ -473,6 +504,7 @@ impl DaytonaSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, }) } @@ -525,6 +557,7 @@ impl DaytonaSandbox { run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, }) } @@ -586,6 +619,125 @@ impl DaytonaSandbox { } } + fn report_clone_failure(&self, origin_url: &str, err: crate::Error) -> crate::Error { + self.emit(SandboxEvent::GitCloneFailed { + url: origin_url.to_string(), + error: err.to_string(), + causes: err.causes(), + }); + err + } + + /// Report a clone/setup failure, clean up the created sandbox, and emit + /// the matching initialization failure. + async fn fail_clone_initialization( + &self, + sandbox: daytona_sdk::Sandbox, + origin_url: &str, + init_start: Instant, + err: crate::Error, + ) -> crate::Error { + let err = self.report_clone_failure(origin_url, err); + let err = self.finish_failed_initialization(sandbox, err).await; + self.fail_init(init_start, err) + } + + /// Point the admitted branch at the exact commit and verify the resulting + /// HEAD. + /// + /// Daytona's native clone honors `commit_id`, but leaves the workspace on + /// whatever ref its own checkout produced. Re-pointing the branch keeps the + /// admitted branch name readable back out of the workspace, matching what + /// the Docker provider produces for the same inputs. + async fn attach_exact_commit_branch( + process_svc: &daytona_sdk::ProcessService, + checkout_path: &str, + branch: &str, + expected_sha: &str, + deadline: time::Instant, + ) -> crate::Result<()> { + Self::run_required_post_clone_command( + process_svc, + &clone_source::exact_branch_checkout_command(checkout_path, branch, expected_sha), + "/", + "git checkout exact commit", + deadline, + ) + .await?; + let head = Self::run_required_post_clone_command( + process_svc, + &clone_source::exact_head_revision_command(checkout_path), + "/", + "git rev-parse HEAD after exact checkout", + deadline, + ) + .await?; + clone_source::verify_exact_head(&head, expected_sha) + } + + /// Execute one post-clone command under the shared setup deadline. + /// + /// The SDK timeout asks Daytona to terminate the remote process. The outer + /// timeout is a transport backstop in case the toolbox never returns that + /// result. Callers must not retry after a timeout because the remote + /// process may still be winding down. + async fn execute_post_clone_command( + process_svc: &daytona_sdk::ProcessService, + command: &str, + cwd: &str, + label: &'static str, + deadline: time::Instant, + ) -> crate::Result { + let remaining = deadline.saturating_duration_since(time::Instant::now()); + if remaining < DAYTONA_MIN_SERVER_TIMEOUT { + return Err(crate::Error::message(format!( + "Daytona post-clone setup deadline expired before {label}" + ))); + } + + let wrapped = wrap_bash_command(command); + let options = daytona_sdk::ExecuteCommandOptions { + cwd: Some(cwd.to_string()), + timeout: Some(remaining), + ..Default::default() + }; + let execution = process_svc.execute_command(&wrapped, options); + time::timeout( + remaining.saturating_add(DAYTONA_CLIENT_TIMEOUT_GRACE), + execution, + ) + .await + .map_err(|_| { + crate::Error::message(format!( + "Daytona post-clone setup timed out while running {label}" + )) + })? + .map_err(|e| crate::Error::context(format!("Failed to run {label}"), e)) + } + + /// Run one required local step after the native clone and return stdout. + async fn run_required_post_clone_command( + process_svc: &daytona_sdk::ProcessService, + command: &str, + cwd: &str, + label: &'static str, + deadline: time::Instant, + ) -> crate::Result { + let start = Instant::now(); + let result = + Self::execute_post_clone_command(process_svc, command, cwd, label, deadline).await?; + if result.exit_code != 0 { + return Err(crate::Error::exec(label, ExecResult { + stdout: result.result, + stderr: String::new(), + exit_code: Some(result.exit_code), + termination: CommandTermination::Exited, + duration_ms: elapsed_ms(start), + })); + } + Ok(result.result) + } + fn fail_init(&self, init_start: Instant, err: crate::Error) -> crate::Error { let duration_ms = u64::try_from(init_start.elapsed().as_millis()).unwrap_or(u64::MAX); self.emit(SandboxEvent::InitializeFailed { @@ -765,22 +917,60 @@ impl DaytonaSandbox { }) } - /// Discard a sandbox that failed its Bash probe. + /// Discard a sandbox whose initialization failed after creation. /// - /// A failed cleanup is logged rather than returned: the Bash failure is - /// what the operator needs to act on. - async fn delete_unusable_sandbox( - sandbox: &daytona_sdk::Sandbox, - bash_error: crate::Error, - ) -> crate::Error { - if let Err(cleanup_error) = sandbox.delete().await { - tracing::warn!( - error = %cleanup_error, - sandbox = %sandbox.name, - "Failed to delete Daytona sandbox after its Bash check failed" - ); + /// A failed cleanup is logged rather than returned: the initialization + /// failure is what the operator needs to act on. The SDK handle is + /// returned only when the caller should retain it for a lifecycle-level + /// cleanup retry. + async fn cleanup_failed_initialization_sandbox( + sandbox: daytona_sdk::Sandbox, + initialization_error: crate::Error, + ) -> (crate::Error, Option) { + match Self::delete_daytona_sandbox(&sandbox).await { + Ok(()) => (initialization_error, None), + Err(cleanup_error) => { + tracing::warn!( + error = %crate::display_for_log(&cleanup_error), + sandbox = %sandbox.name, + "Failed to delete Daytona sandbox after initialization failed" + ); + (initialization_error, Some(sandbox)) + } } - bash_error + } + + async fn delete_daytona_sandbox(sandbox: &daytona_sdk::Sandbox) -> crate::Result<()> { + match time::timeout(DAYTONA_CLEANUP_TIMEOUT, sandbox.delete()).await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) if daytona_not_found(&err) => Ok(()), + Ok(Err(err)) => Err(crate::Error::context( + "Failed to delete Daytona sandbox", + err, + )), + Err(_) => Err(crate::Error::message(format!( + "Timed out deleting Daytona sandbox after {}s", + DAYTONA_CLEANUP_TIMEOUT.as_secs() + ))), + } + } + + async fn finish_failed_initialization( + &self, + sandbox: daytona_sdk::Sandbox, + initialization_error: crate::Error, + ) -> crate::Error { + let (initialization_error, retry_sandbox) = + Self::cleanup_failed_initialization_sandbox(sandbox, initialization_error).await; + if let Some(sandbox) = retry_sandbox { + if self.sandbox.set(sandbox).is_err() { + tracing::warn!( + "Failed to retain Daytona sandbox handle after initialization cleanup \ + failed" + ); + } + } + initialization_error } /// Get the sandbox, returning an error if not yet initialized. @@ -1273,7 +1463,7 @@ impl Sandbox for DaytonaSandbox { })?; if let Err(bash_error) = Self::probe_bash(&sandbox).await { - let err = Self::delete_unusable_sandbox(&sandbox, bash_error).await; + let err = self.finish_failed_initialization(sandbox, bash_error).await; return Err(self.fail_init(init_start, err)); } @@ -1281,6 +1471,7 @@ impl Sandbox for DaytonaSandbox { self.config.skip_clone, self.clone_origin_url.as_deref(), self.clone_branch.as_deref(), + self.clone_commit_sha.as_deref(), ) .map_err(|e| self.fail_init(init_start, e))?; @@ -1305,7 +1496,11 @@ impl Sandbox for DaytonaSandbox { self.set_working_directory(WORKING_DIRECTORY) .map_err(|err| self.fail_init(init_start, err))?; } - CloneDecision::GitHub { origin_url, branch } => { + CloneDecision::GitHub { + origin_url, + branch, + commit_sha, + } => { let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT) .map_err(|err| self.fail_init(init_start, err))?; @@ -1398,16 +1593,12 @@ impl Sandbox for DaytonaSandbox { let git_svc = sandbox.git().await.map_err(|e| { let err = crate::Error::context("Failed to get Daytona git service", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); + let err = self.report_clone_failure(&origin_url, err); self.fail_init(init_start, err) })?; let clone_plan = git_retry::RetryPlan::clone_default(None); - let clone_result = git_retry::retry_clone( + let clone_result = git_retry::retry_git_operation( SandboxProviderKind::Daytona, "clone", &clone_plan, @@ -1415,11 +1606,13 @@ impl Sandbox for DaytonaSandbox { let git_svc = &git_svc; let origin = origin_url.as_str(); let target = layout.primary_repo_path.as_str(); - let options = daytona_sdk::GitCloneOptions { + let options = GitCloneOptions { branch: branch.clone(), + commit_id: commit_sha.clone(), username: username.clone(), password: password.clone(), - ..Default::default() + depth: self.config.clone_depth, + ..GitCloneOptions::default() }; async move { git_svc.clone(origin, target, options).await } }, @@ -1428,156 +1621,150 @@ impl Sandbox for DaytonaSandbox { .await; match clone_result { - Ok(()) => { - let process_svc = sandbox.process().await.map_err(|e| { - let err = - crate::Error::context("Failed to get Daytona process service", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); - self.fail_init(init_start, err) - })?; - let symlink_cmd = clone_source::repo_symlink_command(&layout); - let symlink_result = process_svc - .execute_command( - &wrap_bash_command(&symlink_cmd), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|e| { - let err = crate::Error::context( - "Failed to create Daytona workspace repo symlink", - e, - ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); - self.fail_init(init_start, err) - })?; - if symlink_result.exit_code != 0 { - let err = crate::Error::exec( - "create Daytona workspace repo symlink", - ExecResult { - stdout: symlink_result.result.clone(), - stderr: String::new(), - exit_code: Some(symlink_result.exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, - }, - ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); - return Err(self.fail_init(init_start, err)); - } - - let clone_duration = - u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); - self.emit(SandboxEvent::GitCloneCompleted { - url: origin_url.clone(), - duration_ms: clone_duration, - }); - - let _ = self.repo_cloned.set(true); - let _ = self.origin_url.set(origin_url.clone()); - self.set_working_directory(layout.execution_directory.clone()) - .map_err(|err| self.fail_init(init_start, err))?; - if let Some(resolved) = resolved_token { - match fabro_github::embed_token_in_url( - &origin_url, - resolved.token.expose(), - ) { - Ok(auth_url) => { - let cmd = format!( - "git -c maintenance.auto=0 remote set-url origin {}", - shell_quote(auth_url.as_raw_url().as_str()), - ); - let opts = daytona_sdk::ExecuteCommandOptions { - cwd: Some(layout.execution_directory.clone()), - ..Default::default() - }; - let wrapped = wrap_bash_command(&cmd); - match process_svc.execute_command(&wrapped, opts).await { - Ok(r) if r.exit_code != 0 => { - let err = crate::Error::exec( - "git remote set-url origin (Daytona post-clone)", - ExecResult { - stdout: String::new(), - stderr: redact_auth_url( - &r.result, - Some(&auth_url), - ), - exit_code: Some(r.exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, - }, - ); - tracing::warn!( - error = %crate::display_for_log(&err), - "Failed to set Daytona sandbox push credentials \ - on origin — subsequent git push from this \ - sandbox will fail" - ); - } - Ok(_) => { - // Origin now carries this token; - // record it so refreshes compare - // against the clone generation. - self.push_credentials.record_embedded(resolved).await; - } - Err(_) => { - tracing::warn!( - error_class = "daytona_set_url_exec_failed", - "Daytona exec failed while setting push credentials \ - on origin — subsequent git push from this \ - sandbox will fail" - ); - } - } - } - Err(e) => { - tracing::warn!( - origin = %fabro_redact::redacted_url_for_log(&origin_url), - error = %e, - "Failed to build authenticated origin URL — \ - subsequent git push from this sandbox will fail" - ); - } - } - } - } + Ok(()) => {} Err(e) if self.push_credentials.source().is_none() => { let err = crate::Error::context( - "Git clone failed. If this is a private repository, \ - configure a GitHub App with `fabro install` and install it \ - for your organization.", + "Git clone failed. If this is a private repository, configure a \ + GitHub App with `fabro install` and install it for your organization.", e, ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url, - error: err.to_string(), - causes: err.causes(), - }); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); } Err(e) => { let err = crate::Error::context("Failed to clone repo into Daytona sandbox", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url, - error: err.to_string(), - causes: err.causes(), - }); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + } + } + + let post_clone_deadline = time::Instant::now() + DAYTONA_POST_CLONE_SETUP_TIMEOUT; + let process_svc = match sandbox.process().await { + Ok(process_svc) => process_svc, + Err(e) => { + let err = crate::Error::context("Failed to get Daytona process service", e); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + } + }; + + if let Some(expected_sha) = commit_sha.as_deref() { + let Some(branch) = branch.as_deref().filter(|branch| !branch.trim().is_empty()) + else { + let err = crate::Error::message( + "Exact commit checkout requires a repository branch", + ); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + }; + if let Err(err) = Self::attach_exact_commit_branch( + &process_svc, + &layout.primary_repo_path, + branch, + expected_sha, + post_clone_deadline, + ) + .await + { + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + } + } + + let symlink_cmd = clone_source::repo_symlink_command(&layout); + if let Err(err) = Self::run_required_post_clone_command( + &process_svc, + &symlink_cmd, + "/", + "create Daytona workspace repo symlink", + post_clone_deadline, + ) + .await + { + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + } + + let clone_duration = + u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); + self.emit(SandboxEvent::GitCloneCompleted { + url: origin_url.clone(), + duration_ms: clone_duration, + }); + + let _ = self.repo_cloned.set(true); + let _ = self.origin_url.set(origin_url.clone()); + self.set_working_directory(layout.execution_directory.clone()) + .map_err(|err| self.fail_init(init_start, err))?; + if let Some(resolved) = resolved_token { + match fabro_github::embed_token_in_url(&origin_url, resolved.token.expose()) { + Ok(auth_url) => { + let credential_deadline = + time::Instant::now() + DAYTONA_CREDENTIAL_SETUP_TIMEOUT; + let cmd = format!( + "git -c maintenance.auto=0 remote set-url origin {}", + shell_quote(auth_url.as_raw_url().as_str()), + ); + match Self::execute_post_clone_command( + &process_svc, + &cmd, + &layout.execution_directory, + "git remote set-url origin (Daytona post-clone)", + credential_deadline, + ) + .await + { + Ok(r) if r.exit_code != 0 => { + let err = crate::Error::exec( + "git remote set-url origin (Daytona post-clone)", + ExecResult { + stdout: String::new(), + stderr: redact_auth_url( + &r.result, + Some(&auth_url), + ), + exit_code: Some(r.exit_code), + termination: CommandTermination::Exited, + duration_ms: 0, + }, + ); + tracing::warn!( + error = %crate::display_for_log(&err), + "Failed to set Daytona sandbox push credentials \ + on origin — subsequent git push from this \ + sandbox will fail" + ); + } + Ok(_) => { + // Origin now carries this token; record it so refreshes compare + // against the clone generation. + self.push_credentials.record_embedded(resolved).await; + } + Err(_) => { + tracing::warn!( + error_class = "daytona_set_url_exec_failed", + "Daytona exec failed while setting push credentials \ + on origin — subsequent git push from this \ + sandbox will fail" + ); + } + } + } + Err(e) => { + tracing::warn!( + origin = %fabro_redact::redacted_url_for_log(&origin_url), + error = %e, + "Failed to build authenticated origin URL — \ + subsequent git push from this sandbox will fail" + ); + } } } } @@ -1655,8 +1842,7 @@ impl Sandbox for DaytonaSandbox { let start = Instant::now(); if let Some(sandbox) = self.sandbox.get() { tracing::info!("Deleting Daytona sandbox"); - if let Err(e) = sandbox.delete().await { - let err = crate::Error::context("Failed to delete Daytona sandbox", e); + if let Err(err) = Self::delete_daytona_sandbox(sandbox).await { self.emit(SandboxEvent::DeleteFailed { provider: "daytona".into(), error: err.to_string(), @@ -2902,6 +3088,284 @@ mod tests { use super::*; use crate::sandbox::BASH_PROBE_MARKER; + #[tokio::test] + async fn invalid_exact_sha_fails_before_daytona_client_construction() { + let error = DaytonaSandbox::new( + DaytonaConfig::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + Some("main".to_string()), + Some("not-a-sha".to_string()), + Some("dtn_not_used".to_string()), + ) + .await + .err() + .expect("validation should run before building a Daytona client"); + + assert!(error.to_string().contains("40 ASCII hexadecimal")); + assert!(!error.to_string().contains("Daytona client")); + } + + #[tokio::test] + async fn exact_sha_without_branch_fails_before_daytona_client_construction() { + let error = DaytonaSandbox::new( + DaytonaConfig::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + None, + Some("0123456789abcdef0123456789abcdef01234567".to_string()), + Some("dtn_not_used".to_string()), + ) + .await + .err() + .expect("branch validation should run before building a Daytona client"); + + assert!(error.to_string().contains("requires a repository branch")); + assert!(!error.to_string().contains("Daytona client")); + } + + fn mock_sandbox_body(sandbox_id: &str) -> serde_json::Value { + serde_json::json!({ + "id": sandbox_id, + "organizationId": "org-1", + "name": sandbox_id, + "user": "daytona", + "env": {}, + "labels": {}, + "public": false, + "networkBlockAll": false, + "target": "us", + "cpu": 2.0, + "gpu": 0.0, + "memory": 4.0, + "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", + "state": "started" + }) + } + + async fn mock_sandbox_handle(server: &MockServer, sandbox_id: &str) -> daytona_sdk::Sandbox { + let sandbox_response = server + .mock_async(|when, then| { + when.method(GET).path(format!("/sandbox/{sandbox_id}")); + then.status(200) + .header("content-type", "application/json") + .json_body(mock_sandbox_body(sandbox_id)); + }) + .await; + let client = build_daytona_client_with( + Some("dtn_test".to_string()), + Some(server.base_url()), + None, + Some(fabro_test::test_http_client()), + ) + .await + .expect("create Daytona client"); + let sandbox = client.get(sandbox_id).await.expect("get mock sandbox"); + sandbox_response.assert_async().await; + sandbox + } + + async fn mock_process_service( + server: &MockServer, + sandbox_id: &str, + ) -> daytona_sdk::ProcessService { + let server_url = server.base_url(); + let toolbox_response = server + .mock_async(|when, then| { + when.method(GET) + .path(format!("/sandbox/{sandbox_id}/toolbox-proxy-url")); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "url": server_url })); + }) + .await; + let sandbox = mock_sandbox_handle(server, sandbox_id).await; + let process_svc = sandbox.process().await.expect("get process service"); + toolbox_response.assert_async().await; + process_svc + } + + #[tokio::test] + async fn post_clone_command_sends_server_timeout_and_returns_output() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-success"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")) + .body_includes(r#""cwd":"/work""#) + .body_matches(r#""timeout":[1-9][0-9]*"#); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "expected output" + })); + }) + .await; + + let output = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now() + Duration::from_secs(30), + ) + .await + .expect("post-clone command should succeed"); + + assert_eq!(output, "expected output"); + execute.assert_async().await; + } + + #[tokio::test] + async fn expired_post_clone_deadline_does_not_dispatch_command() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-expired"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "unexpected" + })); + }) + .await; + + let error = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now(), + ) + .await + .expect_err("expired deadline should fail before dispatch"); + + assert!(error.to_string().contains("deadline expired")); + execute.assert_calls_async(0).await; + } + + #[tokio::test] + async fn post_clone_command_has_client_side_timeout_backstop() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-stalled"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")) + .body_matches(r#""timeout":[1-9][0-9]*"#); + then.status(200) + .delay(Duration::from_secs(10)) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "too late" + })); + }) + .await; + + let error = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now() + Duration::from_millis(1_200), + ) + .await + .expect_err("stalled toolbox response should hit the client backstop"); + + assert!(error.to_string().contains("timed out")); + execute.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_deletes_created_daytona_sandbox() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-failed-initialization"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(200) + .header("content-type", "application/json") + .json_body(mock_sandbox_body(sandbox_id)); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert!(retry_sandbox.is_none()); + delete.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_retains_sandbox_when_delete_fails() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-failed-cleanup"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(500) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "message": "try again" })); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert_eq!( + retry_sandbox.as_ref().map(|sandbox| sandbox.id.as_str()), + Some(sandbox_id) + ); + delete.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_treats_missing_sandbox_as_deleted() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-already-deleted"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(404) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "message": "not found" })); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert!(retry_sandbox.is_none()); + delete.assert_async().await; + } + fn api_key_body(permissions: &[&str]) -> serde_json::Value { serde_json::json!({ "name": "delete-only", @@ -2978,6 +3442,7 @@ mod tests { run_id: None, clone_origin_url: None, clone_branch: None, + clone_commit_sha: None, } } @@ -2994,6 +3459,7 @@ mod tests { "size": null, "entrypoint": null, "errorReason": null, + "sourceSandboxId": null, "lastUsedAt": null, "createdAt": "2026-05-01T00:00:00Z", "updatedAt": "2026-05-01T00:00:00Z" @@ -3015,6 +3481,7 @@ mod tests { "gpu": 0.0, "memory": 4.0, "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", "state": state.to_string() }) } @@ -3025,6 +3492,7 @@ mod tests { assert!(config.snapshot.is_none()); assert!(config.auto_stop_interval.is_none()); assert!(config.labels.is_none()); + assert!(config.clone_depth.is_none()); } #[test] @@ -3152,6 +3620,7 @@ mod tests { None, None, None, + None, Some("dtn_test".to_string()), ) .await @@ -3190,6 +3659,7 @@ mod tests { None, None, None, + None, Some("dtn_test".to_string()), ) .await @@ -3517,6 +3987,7 @@ mod tests { Some(run_id), None, None, + None, Some("dtn_test".to_string()), ) .await @@ -3698,10 +4169,7 @@ mod tests { fn missing_display_uses_daytona_wire_scope_names() { let check = DaytonaKeyCheck { key_name: "delete-only".to_string(), - missing: vec![ - Permissions::WriteColonSnapshots, - Permissions::WriteColonSandboxes, - ], + missing: vec![Permissions::WRITE_SNAPSHOTS, Permissions::WRITE_SANDBOXES], }; assert_eq!(check.missing_display(), "write:snapshots, write:sandboxes"); @@ -3840,6 +4308,7 @@ mod tests { "gpu": 0.0, "memory": 4.0, "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", "state": "started" })); }) diff --git a/lib/components/fabro-sandbox/src/details.rs b/lib/components/fabro-sandbox/src/details.rs index 2ada66652..0d7fe6390 100644 --- a/lib/components/fabro-sandbox/src/details.rs +++ b/lib/components/fabro-sandbox/src/details.rs @@ -672,18 +672,22 @@ pub(crate) mod daytona { DaytonaState::Creating | DaytonaState::PendingBuild | DaytonaState::BuildingSnapshot - | DaytonaState::PullingSnapshot => SandboxState::Provisioning, - DaytonaState::Starting => SandboxState::Starting, - DaytonaState::Started => SandboxState::Running, - DaytonaState::Stopping | DaytonaState::Archiving => SandboxState::Stopping, + | DaytonaState::PullingSnapshot + | DaytonaState::Forking => SandboxState::Provisioning, + DaytonaState::Starting | DaytonaState::Resuming => SandboxState::Starting, + DaytonaState::Started | DaytonaState::Snapshotting => SandboxState::Running, + DaytonaState::Stopping | DaytonaState::Archiving | DaytonaState::Pausing => { + SandboxState::Stopping + } DaytonaState::Stopped => SandboxState::Stopped, + DaytonaState::Paused => SandboxState::Paused, DaytonaState::Restoring => SandboxState::Restoring, DaytonaState::Resizing => SandboxState::Resizing, DaytonaState::Archived => SandboxState::Archived, DaytonaState::Destroying => SandboxState::Deleting, DaytonaState::Destroyed => SandboxState::Deleted, DaytonaState::Error | DaytonaState::BuildFailed => SandboxState::Error, - DaytonaState::Unknown => SandboxState::Unknown, + DaytonaState::Unknown | DaytonaState::UnknownDefaultOpenApi => SandboxState::Unknown, } } @@ -755,6 +759,22 @@ pub(crate) mod daytona { ); } + #[test] + fn pause_states_normalize_to_fabro_states() { + assert_eq!( + normalize_daytona_state(DaytonaState::Pausing), + SandboxState::Stopping + ); + assert_eq!( + normalize_daytona_state(DaytonaState::Paused), + SandboxState::Paused + ); + assert_eq!( + normalize_daytona_state(DaytonaState::Resuming), + SandboxState::Starting + ); + } + #[test] fn gibibytes_to_bytes_converts_positive_values() { assert_eq!(gibibytes_to_bytes(2.0), Some(2 * 1024 * 1024 * 1024)); diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index 7758813bd..83b0dc362 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -18,6 +18,7 @@ use bollard::image::CreateImageOptions; use bollard::models::{ContainerInspectResponse, HostConfig}; use fabro_github::GitHubCredentials; use fabro_github::token_source::InstallationTokenSource; +use fabro_types::settings::run::RunCloneSettings; use fabro_types::{CommandOutputStream, CommandTermination, RunId, SandboxProviderKind}; use fabro_util::time::elapsed_ms; use futures::StreamExt; @@ -48,7 +49,7 @@ const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for ev pub(crate) const WORKING_DIRECTORY: &str = "/workspace"; pub(crate) const REPOS_ROOT: &str = "/repos"; -const GIT_CLONE_DEPTH: usize = 10; +const DEFAULT_GIT_CLONE_DEPTH: usize = RunCloneSettings::DEFAULT_DEPTH.unsigned_abs() as usize; const GIT_CLONE_TIMEOUT: Duration = Duration::from_mins(5); #[cfg(test)] const EXEC_STOP_POLL_SLEEP_SECONDS: &str = "0.005"; @@ -59,6 +60,14 @@ const EXEC_TERM_GRACE_SECONDS: &str = "0.02"; #[cfg(not(test))] const EXEC_TERM_GRACE_SECONDS: &str = "0.2"; +/// Whether a failing git step talked to the remote. Local steps cannot fail on +/// credentials, so they must not suggest reconfiguring the GitHub App. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CloneStep { + Network, + Local, +} + struct DockerCloneFailure { error: crate::Error, retry_reason: Option, @@ -113,6 +122,9 @@ pub struct DockerSandboxOptions { pub auto_pull: bool, /// Additional `KEY=VALUE` environment variables for the container. pub env_vars: Vec, + /// Maximum Git history depth fetched during clone; `None` fetches full + /// history. + pub clone_depth: Option, /// Create an empty workspace instead of cloning even when an origin exists. pub skip_clone: bool, } @@ -126,6 +138,7 @@ impl Default for DockerSandboxOptions { cpu_quota: None, auto_pull: true, env_vars: Vec::new(), + clone_depth: Some(DEFAULT_GIT_CLONE_DEPTH), skip_clone: false, } } @@ -138,6 +151,7 @@ pub struct DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, container_id: OnceCell, repo_cloned: OnceCell, working_directory: OnceCell, @@ -168,7 +182,16 @@ impl DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, ) -> crate::Result { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + )?; + } let docker = Docker::connect_with_local_defaults().map_err(crate::Error::docker_connect)?; Self::with_docker_client( docker, @@ -177,6 +200,7 @@ impl DockerSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, ) } @@ -187,6 +211,7 @@ impl DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, ) -> crate::Result { let push_credentials = PushCredentialState::new(push_credentials::build_token_source( github_app, @@ -199,6 +224,7 @@ impl DockerSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, container_id: OnceCell::new(), repo_cloned: OnceCell::new(), working_directory: OnceCell::new(), @@ -224,6 +250,7 @@ impl DockerSandbox { run_id, clone_origin_url.clone(), clone_branch, + None, )?; sandbox.validate_managed_container(container_id).await?; sandbox @@ -715,19 +742,23 @@ impl DockerSandbox { Ok(()) } - /// Preserve a failed `git clone` result while masking the auth URL. + /// Preserve a failed git step result while masking the auth URL. fn clone_failure_error( &self, result: ExecResult, + label: &'static str, auth_url: Option<&fabro_redact::DisplaySafeUrl>, + step: CloneStep, ) -> crate::Error { - let error = result - .into_exec_error_with_redactor("git clone", |output| redact_auth_url(output, auth_url)); - let message = if self.push_credentials.source().is_none() { - "Git clone failed. If this is a private repository, configure a GitHub App with \ - `fabro install` and install it for your organization." - } else { - "Failed to clone repository into Docker sandbox" + let error = + result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)); + let message = match step { + CloneStep::Network if self.push_credentials.source().is_none() => { + "Git clone failed. If this is a private repository, configure a GitHub App with \ + `fabro install` and install it for your organization." + } + CloneStep::Network => "Failed to clone repository into Docker sandbox", + CloneStep::Local => "Failed to prepare the cloned repository in the Docker sandbox", }; crate::Error::context(message, error) } @@ -741,10 +772,104 @@ impl DockerSandbox { err } + /// Run a local (non-network) step of the exact checkout under the shared + /// clone deadline. + /// + /// Materializing a large working tree takes far longer than the short fixed + /// timeout used for trivial container commands, so these steps get the same + /// budget the branch clone path gives its fetch and checkout. + async fn run_exact_local_git_command( + &self, + command: &str, + label: &'static str, + clone_deadline: time::Instant, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> crate::Result { + let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); + let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); + if timeout_ms == 0 { + return Err(crate::Error::message(format!( + "{label} deadline expired before the step could run" + ))); + } + let result = self + .docker_exec_shell(command, timeout_ms, Some("/"), None, None) + .await + .map_err(|error| crate::Error::context(format!("{label} transport failed"), error))?; + if result.is_success() { + Ok(result) + } else { + Err(self.clone_failure_error(result, label, auth_url, CloneStep::Local)) + } + } + + /// Run a network git command inside the container with clone retry + /// semantics under the shared clone deadline. + async fn retry_git_transfer( + &self, + command: &str, + op: &'static str, + label: &'static str, + exec_label: &'static str, + clone_deadline: time::Instant, + credential_context: CredentialContext, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> Result<(), DockerCloneFailure> { + let plan = git_retry::RetryPlan::clone_default(Some(clone_deadline)); + git_retry::retry_git_operation( + SandboxProviderKind::Docker, + op, + &plan, + |_attempt| async move { + let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); + let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); + if timeout_ms == 0 { + return Err(DockerCloneFailure { + error: crate::Error::message(format!( + "{label} deadline expired before retry" + )), + retry_reason: None, + }); + } + let result = self + .docker_exec_shell_streaming(ExecStreamingRequest { + timeout_ms: Some(timeout_ms), + working_dir: Some("/"), + ..ExecStreamingRequest::new(command) + }) + .await + .map_err(|error| DockerCloneFailure { + error: crate::Error::context( + format!("{label} transport failed"), + error, + ), + retry_reason: None, + })? + .result; + if result.is_success() { + return Ok(()); + } + let retry_reason = classify_docker_clone_result(&result, credential_context); + Err(DockerCloneFailure { + error: self.clone_failure_error( + result, + exec_label, + auth_url, + CloneStep::Network, + ), + retry_reason, + }) + }, + |failure: &DockerCloneFailure| failure.retry_reason, + ) + .await + } + async fn clone_github_repo( &self, origin_url: String, branch: Option, + commit_sha: Option, ) -> crate::Result<()> { self.verify_git_available().await?; let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT)?; @@ -805,60 +930,93 @@ impl DockerSandbox { } } - let command = git_clone_command(clone_url, branch.as_deref(), &layout.primary_repo_path); let clone_deadline = time::Instant::now() + GIT_CLONE_TIMEOUT; - let clone_plan = git_retry::RetryPlan::clone_default(Some(clone_deadline)); - let clone_result = git_retry::retry_clone( - SandboxProviderKind::Docker, - "clone", - &clone_plan, - |_attempt| { - let command = command.as_str(); - let auth_url = auth_url.as_ref(); - async move { - let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); - let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); - if timeout_ms == 0 { - return Err(DockerCloneFailure { - error: crate::Error::message( - "Docker git clone deadline expired before retry", - ), - retry_reason: None, - }); - } - let result = self - .docker_exec_shell_streaming(ExecStreamingRequest { - timeout_ms: Some(timeout_ms), - working_dir: Some("/"), - ..ExecStreamingRequest::new(command) - }) - .await - .map_err(|error| DockerCloneFailure { - error: crate::Error::context( - "Docker git clone transport failed", - error, - ), - retry_reason: None, - })? - .result; - if result.is_success() { - return Ok(()); - } - let retry_reason = - classify_docker_clone_result(&result, clone_credential_context); - Err(DockerCloneFailure { - error: self.clone_failure_error(result, auth_url), - retry_reason, - }) - } - }, - |failure: &DockerCloneFailure| failure.retry_reason, - ) - .await; + if let Some(expected_sha) = commit_sha.as_deref() { + // `decide_clone` already rejects an exact commit without a branch; + // re-check here so the checkout can never silently drop the branch + // name callers read back out of the workspace. + let Some(branch) = branch.as_deref().filter(|branch| !branch.trim().is_empty()) else { + let error = + crate::Error::message("Exact commit checkout requires a repository branch"); + return Err(self.report_clone_failure(&origin_url, error)); + }; - if let Err(failure) = clone_result { - let err = failure.error; - return Err(self.report_clone_failure(&origin_url, err)); + let init_command = + clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); + if let Err(error) = self + .run_exact_local_git_command( + &init_command, + "initialize Docker exact repository checkout", + clone_deadline, + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let fetch_command = clone_source::exact_fetch_command( + &layout.primary_repo_path, + "origin", + expected_sha, + self.config.clone_depth, + ); + if let Err(failure) = self + .retry_git_transfer( + &fetch_command, + "fetch", + "Docker exact fetch", + "git fetch exact commit", + clone_deadline, + clone_credential_context, + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, failure.error)); + } + + let checkout_command = clone_source::exact_checkout_verify_command( + &layout.primary_repo_path, + branch, + "FETCH_HEAD", + ); + let head = match self + .run_exact_local_git_command( + &checkout_command, + "git checkout exact commit", + clone_deadline, + auth_url.as_ref(), + ) + .await + { + Ok(result) => result, + Err(error) => return Err(self.report_clone_failure(&origin_url, error)), + }; + if let Err(error) = clone_source::verify_exact_head(&head.stdout, expected_sha) { + return Err(self.report_clone_failure(&origin_url, error)); + } + } else { + let command = git_clone_command( + clone_url, + branch.as_deref(), + &layout.primary_repo_path, + self.config.clone_depth, + ); + if let Err(failure) = self + .retry_git_transfer( + &command, + "clone", + "Docker git clone", + "git clone", + clone_deadline, + clone_credential_context, + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, failure.error)); + } } let symlink_command = clone_source::repo_symlink_command(&layout); @@ -1380,15 +1538,19 @@ async fn cache_docker_stdio_completion( } } -fn git_clone_command(clone_url: &str, branch: Option<&str>, checkout_path: &str) -> String { - let mut command = "git -c maintenance.auto=0 -c gc.auto=0 clone".to_string(); +fn git_clone_command( + clone_url: &str, + branch: Option<&str>, + checkout_path: &str, + depth: Option, +) -> String { + let mut command = format!("{} clone", sandbox::GIT); if let Some(branch) = branch { command.push_str(" --branch "); command.push_str(&shell_quote(branch)); command.push_str(" --single-branch"); } - command.push_str(" --depth "); - command.push_str(&GIT_CLONE_DEPTH.to_string()); + command.push_str(&clone_source::depth_argument(depth)); command.push_str(" --no-tags"); command.push_str(" -- "); command.push_str(&shell_quote(clone_url)); @@ -1623,6 +1785,7 @@ impl Sandbox for DockerSandbox { self.config.skip_clone, self.clone_origin_url.as_deref(), self.clone_branch.as_deref(), + self.clone_commit_sha.as_deref(), ) .map_err(|e| self.fail_init(init_start, e))?; @@ -1640,8 +1803,12 @@ impl Sandbox for DockerSandbox { } let _ = self.repo_cloned.set(false); } - CloneDecision::GitHub { origin_url, branch } => { - if let Err(e) = self.clone_github_repo(origin_url, branch).await { + CloneDecision::GitHub { + origin_url, + branch, + commit_sha, + } => { + if let Err(e) = self.clone_github_repo(origin_url, branch, commit_sha).await { return Err(self.fail_init(init_start, e)); } } @@ -2402,22 +2569,154 @@ mod tests { let options = DockerSandboxOptions::default(); assert_eq!(options.image, "buildpack-deps:noble"); assert_eq!(options.network_mode.as_deref(), Some("bridge")); + assert_eq!(options.clone_depth, Some(DEFAULT_GIT_CLONE_DEPTH)); assert!(!options.skip_clone); } #[test] - fn clone_command_uses_depth_ten_without_tags_for_branch_clone() { + fn clone_command_uses_configured_depth_without_tags_for_branch_clone() { let command = git_clone_command( "https://github.com/fabro-sh/fabro", Some("main"), "/repos/fabro-sh/fabro", + Some(1), ); assert_eq!( command, - "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --depth 10 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --depth 1 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" ); } + #[test] + fn clone_command_without_branch_retains_legacy_shape() { + let command = git_clone_command( + "https://github.com/fabro-sh/fabro", + None, + "/repos/fabro-sh/fabro", + Some(DEFAULT_GIT_CLONE_DEPTH), + ); + assert_eq!( + command, + "git -c maintenance.auto=0 -c gc.auto=0 clone --depth 100 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + ); + } + + #[test] + fn clone_command_omits_depth_for_full_clone() { + let command = git_clone_command( + "https://github.com/fabro-sh/fabro", + Some("main"), + "/repos/fabro-sh/fabro", + None, + ); + assert_eq!( + command, + "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + ); + } + + #[test] + fn invalid_exact_sha_fails_before_docker_connection() { + let error = DockerSandbox::new( + DockerSandboxOptions::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + Some("main".to_string()), + Some("not-a-sha".to_string()), + ) + .err() + .expect("validation should run before connecting to Docker"); + + assert!(error.to_string().contains("40 ASCII hexadecimal")); + assert!(!error.to_string().contains("Docker daemon")); + } + + #[test] + fn exact_sha_without_branch_fails_before_docker_connection() { + let error = DockerSandbox::new( + DockerSandboxOptions::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + None, + Some("0123456789abcdef0123456789abcdef01234567".to_string()), + ) + .err() + .expect("branch validation should run before connecting to Docker"); + + assert!(error.to_string().contains("requires a repository branch")); + assert!(!error.to_string().contains("Docker daemon")); + } + + #[test] + fn exact_checkout_failure_preserves_safe_source_chain() { + let docker = Docker::connect_with_http("http://127.0.0.1:2375", 5, API_DEFAULT_VERSION) + .expect("mock Docker client should connect"); + let sandbox = test_docker_sandbox(docker, "test-container"); + let token = "ghs_exact_checkout_secret"; + let auth_url = fabro_github::embed_token_in_url("https://github.com/acme/widgets", token) + .expect("authenticated URL"); + let error = sandbox.clone_failure_error( + ExecResult { + stdout: String::new(), + stderr: format!( + "fatal: unable to access {}: synthetic low-level failure", + auth_url.as_raw_url() + ), + exit_code: Some(128), + termination: CommandTermination::Exited, + duration_ms: 1, + }, + "git fetch exact commit", + Some(&auth_url), + CloneStep::Network, + ); + + let causes = error.causes(); + assert!( + causes + .iter() + .any(|cause| cause.contains("git fetch exact commit failed")), + "source chain should retain the exec failure: {causes:?}" + ); + let rendered = crate::display_for_log(&error); + assert!(!rendered.contains(token)); + assert!(!rendered.contains(auth_url.as_raw_url().as_str())); + assert!(rendered.contains("synthetic low-level failure")); + } + + #[test] + fn local_checkout_failure_does_not_blame_github_credentials() { + let docker = Docker::connect_with_http("http://127.0.0.1:2375", 5, API_DEFAULT_VERSION) + .expect("mock Docker client should connect"); + let sandbox = test_docker_sandbox(docker, "test-container"); + let failure = ExecResult { + stdout: String::new(), + stderr: "error: pathspec 'FETCH_HEAD' did not match".to_string(), + exit_code: Some(1), + termination: CommandTermination::Exited, + duration_ms: 1, + }; + + let local = crate::display_for_log(&sandbox.clone_failure_error( + failure.clone(), + "git checkout exact commit", + None, + CloneStep::Local, + )); + assert!(!local.contains("fabro install"), "{local}"); + assert!(local.contains("prepare the cloned repository"), "{local}"); + + let network = crate::display_for_log(&sandbox.clone_failure_error( + failure, + "git fetch exact commit", + None, + CloneStep::Network, + )); + assert!(network.contains("fabro install"), "{network}"); + } + #[test] fn clone_result_uses_stderr_before_stdout() { let result = ExecResult { @@ -2717,6 +3016,7 @@ mod tests { None, None, None, + None, ) .expect("test sandbox should build"); sandbox diff --git a/lib/components/fabro-sandbox/src/from_environment.rs b/lib/components/fabro-sandbox/src/from_environment.rs index 5399927ea..657b4c179 100644 --- a/lib/components/fabro-sandbox/src/from_environment.rs +++ b/lib/components/fabro-sandbox/src/from_environment.rs @@ -8,7 +8,9 @@ use std::path::{Path, PathBuf}; use fabro_types::settings::ResolveError; #[cfg(feature = "daytona")] use fabro_types::settings::run::DockerfileSource as ResolvedDockerfileSource; -use fabro_types::settings::run::{EnvironmentNetworkMode, RunEnvironmentSettings}; +use fabro_types::settings::run::{ + EnvironmentNetworkMode, RunCloneSettings, RunEnvironmentSettings, +}; #[cfg(feature = "daytona")] use crate::config::{ @@ -23,19 +25,16 @@ use crate::docker::DockerSandboxOptions; #[must_use] pub fn daytona_config_from_environment( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, ) -> DaytonaConfig { DaytonaConfig { auto_stop_interval: settings .lifecycle .auto_stop .map(|duration| duration_to_minutes_i32(duration.as_std())), - labels: (!settings.labels.is_empty()).then(|| settings.labels.clone()), - snapshot: settings - .image - .dockerfile - .as_ref() - .map(|dockerfile| DaytonaSnapshotSettings { + labels: (!settings.labels.is_empty()).then(|| settings.labels.clone()), + snapshot: settings.image.dockerfile.as_ref().map(|dockerfile| { + DaytonaSnapshotSettings { cpu: settings.resources.cpu, memory: settings .resources @@ -53,15 +52,17 @@ pub fn daytona_config_from_environment( SandboxDockerfileSource::Path { path: path.clone() } } }), - }), - network: Some(match settings.network.mode { + } + }), + network: Some(match settings.network.mode { EnvironmentNetworkMode::Block => DaytonaNetwork::Block, EnvironmentNetworkMode::AllowAll => DaytonaNetwork::AllowAll, EnvironmentNetworkMode::CidrAllowList => { DaytonaNetwork::AllowList(settings.network.allow.clone()) } }), - skip_clone, + clone_depth: clone.depth_limit(), + skip_clone: !clone.enabled, } } @@ -69,7 +70,7 @@ pub fn daytona_config_from_environment( #[must_use] pub fn docker_config_from_environment( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, ) -> DockerSandboxOptions { // No vault is available on this path (server preflight / manifest), so a // `{{ secrets.* }}` value keeps its source form. Nothing else is left to @@ -84,25 +85,23 @@ pub fn docker_config_from_environment( .iter() .map(|(key, value)| (key.clone(), value.as_source())) .collect(); - docker_config_from_environment_env(settings, skip_clone, env) + docker_config_from_environment_env(settings, clone, env) } #[cfg(feature = "docker")] pub fn docker_config_from_environment_with_secrets( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, secrets_lookup: impl FnMut(&str) -> Option, ) -> Result { let env = settings.resolve_env(secrets_lookup)?; - Ok(docker_config_from_environment_env( - settings, skip_clone, env, - )) + Ok(docker_config_from_environment_env(settings, clone, env)) } #[cfg(feature = "docker")] fn docker_config_from_environment_env( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, env: std::collections::HashMap, ) -> DockerSandboxOptions { let mut env_vars = env @@ -133,7 +132,10 @@ fn docker_config_from_environment_env( .cpu .map(|cpu| i64::from(cpu).saturating_mul(100_000)), env_vars, - skip_clone, + clone_depth: clone + .depth_limit() + .and_then(|depth| usize::try_from(depth).ok()), + skip_clone: !clone.enabled, ..DockerSandboxOptions::default() } } diff --git a/lib/components/fabro-sandbox/src/git_retry.rs b/lib/components/fabro-sandbox/src/git_retry.rs index 7b16cc256..93e64973f 100644 --- a/lib/components/fabro-sandbox/src/git_retry.rs +++ b/lib/components/fabro-sandbox/src/git_retry.rs @@ -199,7 +199,7 @@ pub fn classify_failure(message: &str, cred: CredentialContext) -> Option BackoffPolicy { +fn replication_backoff() -> BackoffPolicy { BackoffPolicy { initial_delay: Duration::from_secs(3), factor: 3.0, @@ -229,6 +229,13 @@ pub struct RetryPlan { } impl RetryPlan { + /// Host-side repository probes use the same attempt count and pacing as + /// clone operations against a freshly minted token. + #[must_use] + pub fn repository_probe() -> Self { + Self::clone_default(None) + } + /// The clone policy both providers already trust: 3 attempts, 3s/9s /// backoff, no plan-level bounds. Docker supplies its existing absolute /// five-minute deadline through `outer_deadline`; Daytona supplies none. @@ -236,7 +243,7 @@ impl RetryPlan { pub fn clone_default(outer_deadline: Option) -> Self { Self { max_attempts: 3, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: None, per_attempt_timeout: None, outer_deadline, @@ -249,7 +256,7 @@ impl RetryPlan { pub fn checkpoint_push() -> Self { Self { max_attempts: 3, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: Some(Duration::from_secs(90)), per_attempt_timeout: Some(Duration::from_mins(1)), outer_deadline: None, @@ -316,13 +323,13 @@ impl RetryPlan { } } -/// Run a clone operation, repeating it while the failure looks transient. +/// Run a git operation, repeating it while the failure looks transient. /// /// `attempt` receives the 1-based attempt number. `classify` decides whether /// an error is worth repeating; `None` returns it to the caller untouched. /// A retry starts only when its backoff fits before the plan's effective /// deadline. The final error is returned as-is. -pub(crate) async fn retry_clone( +pub async fn retry_git_operation( provider: SandboxProviderKind, op: &str, plan: &RetryPlan, @@ -595,7 +602,7 @@ mod tests { async fn first_success_runs_one_attempt() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -615,7 +622,7 @@ mod tests { async fn retries_until_a_later_attempt_succeeds() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -641,7 +648,7 @@ mod tests { async fn exhausted_attempts_return_the_final_error() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -665,7 +672,7 @@ mod tests { async fn unretryable_failure_stops_immediately() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -692,7 +699,7 @@ mod tests { let attempts = Attempts::default(); let deadline = time::Instant::now() + Duration::from_secs(2); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(Some(deadline)), @@ -715,7 +722,7 @@ mod tests { async fn unbounded_plan_runs_all_attempts() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Daytona, "clone", &RetryPlan::clone_default(None), @@ -736,13 +743,13 @@ mod tests { let attempts = Attempts::default(); let plan = RetryPlan { max_attempts: 5, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: Some(Duration::from_secs(4)), per_attempt_timeout: None, outer_deadline: None, }; - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "push", &plan, diff --git a/lib/components/fabro-sandbox/src/lib.rs b/lib/components/fabro-sandbox/src/lib.rs index fb442e9fb..3594aa0c0 100644 --- a/lib/components/fabro-sandbox/src/lib.rs +++ b/lib/components/fabro-sandbox/src/lib.rs @@ -43,7 +43,9 @@ pub use fabro_github::token_source::{ InstallationTokenSource, ResolvedToken, TokenProvenance, TokenSnapshot, }; pub use fabro_types::{RunSandboxInstance, SandboxProviderKind}; -pub use git_retry::{CredentialContext, GitRetryReason, RetryPlan, classify_failure}; +pub use git_retry::{ + CredentialContext, GitRetryReason, RetryPlan, classify_failure, retry_git_operation, +}; pub use local::LocalSandbox; #[cfg(feature = "daytona")] pub use provider::daytona::DaytonaSandboxProvider; diff --git a/lib/components/fabro-sandbox/src/provider/daytona.rs b/lib/components/fabro-sandbox/src/provider/daytona.rs index 7295afd97..45b212181 100644 --- a/lib/components/fabro-sandbox/src/provider/daytona.rs +++ b/lib/components/fabro-sandbox/src/provider/daytona.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use async_trait::async_trait; -use daytona_sdk::DaytonaError; use fabro_static::EnvVars; use fabro_types::{SandboxInfo, SandboxProviderKind}; @@ -89,7 +88,7 @@ impl SandboxProvider for DaytonaSandboxProvider { let client = self.client().await?; let sandbox = match client.get(id).await { Ok(sandbox) => sandbox, - Err(err) if daytona_not_found(&err) => return Ok(None), + Err(err) if daytona::daytona_not_found(&err) => return Ok(None), Err(err) => { return Err(crate::Error::context( format!("Failed to get Daytona sandbox '{id}'"), @@ -130,6 +129,7 @@ impl SandboxProvider for DaytonaSandboxProvider { run_id, clone_origin_url, clone_branch, + None, Some(api_key), ) .await?; @@ -144,7 +144,7 @@ impl SandboxProvider for DaytonaSandboxProvider { let client = self.client().await?; let sandbox = match client.get(id).await { Ok(sandbox) => sandbox, - Err(err) if daytona_not_found(&err) => return Ok(()), + Err(err) if daytona::daytona_not_found(&err) => return Ok(()), Err(err) => { return Err(crate::Error::context( format!("Failed to get Daytona sandbox '{id}' before delete"), @@ -166,7 +166,3 @@ impl SandboxProvider for DaytonaSandboxProvider { fn managed_from_sdk_sandbox(sandbox: &daytona_sdk::Sandbox) -> bool { managed_labels::is_managed(&sandbox.labels) } - -fn daytona_not_found(err: &DaytonaError) -> bool { - matches!(err, DaytonaError::NotFound { .. }) || err.status_code() == Some(404) -} diff --git a/lib/components/fabro-sandbox/src/provider/docker.rs b/lib/components/fabro-sandbox/src/provider/docker.rs index 865e361be..37337eaed 100644 --- a/lib/components/fabro-sandbox/src/provider/docker.rs +++ b/lib/components/fabro-sandbox/src/provider/docker.rs @@ -104,6 +104,7 @@ impl SandboxProvider for DockerSandboxProvider { run_id, clone_origin_url, clone_branch, + None, )?; sandbox.initialize().await?; let container_id = sandbox.container_identifier()?.to_string(); diff --git a/lib/components/fabro-sandbox/src/sandbox.rs b/lib/components/fabro-sandbox/src/sandbox.rs index 822c59a23..f9d153b2e 100644 --- a/lib/components/fabro-sandbox/src/sandbox.rs +++ b/lib/components/fabro-sandbox/src/sandbox.rs @@ -23,7 +23,7 @@ use crate::git_retry::{self, CredentialContext, GitRetryReason, RetryPlan}; use crate::push_credentials::{CredentialLease, PushCredentialState, RefreshErrorKind}; /// Git command prefix that disables background maintenance. -const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; +pub(crate) const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; pub const DEFAULT_EXEC_OUTPUT_TAIL_BYTES: usize = 8 * 1024; diff --git a/lib/components/fabro-sandbox/src/sandbox_spec.rs b/lib/components/fabro-sandbox/src/sandbox_spec.rs index 7fcc7232a..06565eb95 100644 --- a/lib/components/fabro-sandbox/src/sandbox_spec.rs +++ b/lib/components/fabro-sandbox/src/sandbox_spec.rs @@ -32,6 +32,7 @@ pub enum SandboxSpec { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, }, #[cfg(feature = "daytona")] Daytona { @@ -40,6 +41,7 @@ pub enum SandboxSpec { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, api_key: Option, }, } @@ -202,6 +204,7 @@ impl SandboxSpec { run_id, clone_origin_url, clone_branch, + clone_commit_sha, } => { let mut sandbox = DockerSandbox::new( config.clone(), @@ -209,6 +212,7 @@ impl SandboxSpec { *run_id, clone_origin_url.clone(), clone_branch.clone(), + clone_commit_sha.clone(), ) .context("Failed to create Docker sandbox")?; if let Some(callback) = event_callback { @@ -223,6 +227,7 @@ impl SandboxSpec { run_id, clone_origin_url, clone_branch, + clone_commit_sha, api_key, } => { let mut sandbox = DaytonaSandbox::new( @@ -231,6 +236,7 @@ impl SandboxSpec { *run_id, clone_origin_url.clone(), clone_branch.clone(), + clone_commit_sha.clone(), api_key.clone(), ) .await @@ -276,6 +282,7 @@ mod tests { run_id: None, clone_origin_url: Some("git@github.com:brynary/rack-test.git".to_string()), clone_branch: Some("main".to_string()), + clone_commit_sha: None, }; let mut sandbox = MockSandbox::linux(); sandbox.working_dir = "/workspace/rack-test"; @@ -300,6 +307,34 @@ mod tests { runtime.primary_repo_link.as_deref(), Some("/workspace/rack-test") ); + let runtime_json = serde_json::to_value(&runtime).expect("runtime should serialize"); + assert!(runtime_json.get("clone_commit_sha").is_none()); + } + + #[cfg(feature = "docker")] + #[tokio::test] + async fn invalid_exact_checkout_spec_fails_before_provider_connection() { + let spec = SandboxSpec::Docker { + config: DockerSandboxOptions::default(), + github_app: None, + run_id: None, + clone_origin_url: Some("https://github.com/acme/widgets".to_string()), + clone_branch: Some("main".to_string()), + clone_commit_sha: Some("not-a-sha".to_string()), + }; + + let error = spec + .build(None) + .await + .err() + .expect("spec validation should run before Docker connection"); + assert!( + error + .to_string() + .contains("Failed to create Docker sandbox") + ); + assert!(format!("{error:#}").contains("40 ASCII hexadecimal")); + assert!(!format!("{error:#}").contains("Docker daemon")); } #[cfg(feature = "docker")] @@ -314,6 +349,7 @@ mod tests { run_id: None, clone_origin_url: Some("https://gitlab.com/acme/widgets".to_string()), clone_branch: None, + clone_commit_sha: None, }; let mut sandbox = MockSandbox::linux(); sandbox.working_dir = "/workspace"; diff --git a/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs b/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs index 85e146041..9fdaf8fb8 100644 --- a/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs +++ b/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs @@ -37,6 +37,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?, ); @@ -74,6 +75,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; sandbox.initialize().await?; @@ -179,6 +181,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; @@ -228,6 +231,7 @@ mod daytona_streaming_live { Some("https://github.com/brynary/rack-test".to_string()), None, None, + None, ) .await?; @@ -295,6 +299,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; diff --git a/lib/components/fabro-sandbox/tests/docker_streaming.rs b/lib/components/fabro-sandbox/tests/docker_streaming.rs index 6274c638c..c04ccf1a0 100644 --- a/lib/components/fabro-sandbox/tests/docker_streaming.rs +++ b/lib/components/fabro-sandbox/tests/docker_streaming.rs @@ -40,6 +40,7 @@ async fn streaming_timeout_terminates_docker_exec_before_returning() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -112,6 +113,7 @@ async fn streaming_command_receives_exact_stdin_and_eof() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -173,6 +175,7 @@ async fn cloned_docker_sandbox_uses_repos_checkout_and_workspace_symlink() { None, Some("https://github.com/brynary/rack-test".to_string()), None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -239,6 +242,7 @@ async fn docker_runs_clean_bash_through_both_command_paths() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -332,6 +336,7 @@ async fn docker_glob_matches_patterns_containing_a_path_separator() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox diff --git a/lib/components/fabro-store/src/blob_store.rs b/lib/components/fabro-store/src/blob_store.rs index 6d19ed978..54b8124d3 100644 --- a/lib/components/fabro-store/src/blob_store.rs +++ b/lib/components/fabro-store/src/blob_store.rs @@ -77,6 +77,13 @@ impl BlobStore { } } + pub(crate) fn sqlite_pool_for_legacy_import(&self) -> Option<&SqlitePool> { + match &self.backend { + BlobBackend::Slate(_) => None, + BlobBackend::Sqlite(pool) => Some(pool), + } + } + pub async fn write(&self, bytes: &[u8]) -> Result { match &self.backend { BlobBackend::Slate(repo) => { diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs new file mode 100644 index 000000000..d9ea4ba93 --- /dev/null +++ b/lib/components/fabro-store/src/legacy_blob_import.rs @@ -0,0 +1,1568 @@ +//! Temporary compatibility importer for the legacy SlateDB blob keyspace. +//! +//! Remove this module with the Slate blob backend after the approved legacy +//! support window ends. + +use std::error::Error as StdError; +use std::fmt; + +use bytes::Bytes; +use fabro_types::BlobHash; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire as _, Sqlite}; +#[cfg(test)] +use tokio::sync::Barrier; +use tracing::debug; + +use crate::keys::SlateKey; +use crate::{BlobStore, Database}; + +const MAX_BATCH_ROWS: usize = 100; +const MAX_BATCH_BYTES: u64 = 1024 * 1024; +const PASSIVE_CHECKPOINT_BYTES: u64 = 8 * 1024 * 1024; + +/// Aggregate progress from one legacy blob import attempt. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct LegacyBlobImportReport { + /// Source rows observed under the exact legacy blob prefix. + pub scanned_rows: u64, + /// Raw source value bytes observed under the exact legacy blob prefix. + pub scanned_bytes: u64, + /// Destination rows newly inserted by committed transactions. + pub imported_rows: u64, + /// Destination bytes newly inserted by committed transactions. + pub imported_bytes: u64, + /// Byte-equal destination rows accepted by committed transactions. + pub existing_rows: u64, + /// Bytes belonging to byte-equal destination rows. + pub existing_bytes: u64, + /// Source rows rejected for malformed keys or digest mismatches. + pub invalid_rows: u64, + /// Destination rows rejected because their bytes differed. + pub conflicting_rows: u64, + /// Successfully committed import transactions. + pub committed_batches: u64, + /// Successful passive WAL checkpoints during the import. + pub passive_checkpoints: u64, +} + +/// A failed legacy blob import and the durable progress completed before it. +pub struct LegacyBlobImportError { + report: LegacyBlobImportReport, + failure: LegacyBlobImportFailure, +} + +impl LegacyBlobImportError { + /// Returns the aggregate durable progress completed before the failure. + #[must_use] + pub fn report(&self) -> &LegacyBlobImportReport { + &self.report + } + + /// Returns secondary errors encountered while cleaning up the failed + /// import. + /// + /// The standard error source chain preserves the failure that interrupted + /// the import. Because that chain is linear, rollback, connection-setting + /// restoration, and connection-retirement errors are exposed separately. + pub fn cleanup_errors(&self) -> impl Iterator { + let mut errors = Vec::new(); + self.failure.collect_cleanup_errors(&mut errors); + errors.into_iter() + } +} + +impl fmt::Debug for LegacyBlobImportError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("LegacyBlobImportError") + .field("report", &self.report) + .field("failure", &self.failure.kind()) + .finish() + } +} + +impl fmt::Display for LegacyBlobImportError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "legacy blob import failed after scanning {} rows and committing {} batches: {}", + self.report.scanned_rows, self.report.committed_batches, self.failure + ) + } +} + +impl StdError for LegacyBlobImportError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(self.failure.primary_failure()) + } +} + +#[derive(strum::IntoStaticStr, thiserror::Error)] +#[strum(serialize_all = "snake_case")] +enum LegacyBlobImportFailure { + #[error("the legacy blob import target is not backed by SQLite")] + WrongTargetBackend, + + #[error("opening the legacy blob source")] + OpenSource(#[source] crate::Error), + + #[error("opening the legacy blob scan")] + OpenSourceScan(#[source] slatedb::Error), + + #[error("reading the legacy blob scan")] + ReadSourceScan(#[source] slatedb::Error), + + #[error("a legacy blob key is not canonical")] + InvalidSourceKey, + + #[error("legacy blob bytes do not match their key digest")] + SourceDigestMismatch, + + #[error("a legacy blob import counter overflowed")] + CounterOverflow, + + #[error("acquiring the SQLite import connection")] + AcquireConnection(#[source] sqlx::Error), + + #[error("reading the SQLite automatic checkpoint setting")] + ReadAutomaticCheckpoint(#[source] sqlx::Error), + + #[error("disabling SQLite automatic checkpointing")] + DisableAutomaticCheckpoint(#[source] sqlx::Error), + + #[error("starting a SQLite blob import transaction")] + BeginTransaction(#[source] sqlx::Error), + + #[error("inserting a SQLite blob row")] + InsertDestination(#[source] sqlx::Error), + + #[error("reading an existing SQLite blob row")] + ReadDestination(#[source] sqlx::Error), + + #[error("SQLite contains different bytes for a legacy blob hash")] + DestinationConflict, + + #[error("committing a SQLite blob import transaction")] + CommitTransaction(#[source] sqlx::Error), + + #[error("rolling back a failed SQLite blob import transaction")] + RollbackTransaction { + #[source] + source: sqlx::Error, + prior: Box, + }, + + #[error("running a passive SQLite WAL checkpoint")] + PassiveCheckpoint(#[source] sqlx::Error), + + #[error("the passive SQLite WAL checkpoint could not complete")] + PassiveCheckpointBusy, + + #[error("running the final SQLite WAL checkpoint")] + FinalCheckpoint(#[source] sqlx::Error), + + #[error("the final SQLite WAL checkpoint could not complete")] + FinalCheckpointBusy, + + #[error("restoring the SQLite automatic checkpoint setting")] + RestoreAutomaticCheckpoint { + #[source] + source: sqlx::Error, + prior: Option>, + retirement_error: Option, + }, +} + +impl LegacyBlobImportFailure { + fn kind(&self) -> &'static str { + self.into() + } + + fn primary_failure(&self) -> &Self { + match self { + Self::RollbackTransaction { prior, .. } + | Self::RestoreAutomaticCheckpoint { + prior: Some(prior), .. + } => prior.primary_failure(), + _ => self, + } + } + + fn collect_cleanup_errors<'a>(&'a self, errors: &mut Vec<&'a (dyn StdError + 'static)>) { + match self { + Self::RollbackTransaction { source, prior } => { + errors.push(source); + prior.collect_cleanup_errors(errors); + } + Self::RestoreAutomaticCheckpoint { + source, + prior, + retirement_error, + } => { + errors.push(source); + if let Some(retirement_error) = retirement_error { + errors.push(retirement_error); + } + if let Some(prior) = prior { + prior.collect_cleanup_errors(errors); + } + } + _ => {} + } + } +} + +impl fmt::Debug for LegacyBlobImportFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut debug = formatter.debug_struct("LegacyBlobImportFailure"); + debug.field("kind", &self.kind()); + match self { + Self::RollbackTransaction { prior, .. } => { + debug.field("prior_failure", &prior.kind()); + } + Self::RestoreAutomaticCheckpoint { + prior, + retirement_error, + .. + } => { + debug + .field("prior_failure", &prior.as_deref().map(Self::kind)) + .field("retirement_failed", &retirement_error.is_some()); + } + _ => {} + } + debug.finish() + } +} + +#[derive(Default)] +struct ImportControls { + #[cfg(test)] + after_automatic_checkpoint_disabled: Option>, + #[cfg(test)] + source_after_rows: Option, + #[cfg(test)] + passive_checkpoint: bool, + #[cfg(test)] + final_checkpoint: bool, + #[cfg(test)] + restore_automatic_checkpoint: bool, +} + +impl ImportControls { + #[cfg(test)] + async fn after_automatic_checkpoint_disabled(&self) { + if let Some(barrier) = &self.after_automatic_checkpoint_disabled { + barrier.wait().await; + barrier.wait().await; + } + } + + fn source_scan_error(&self, scanned_rows: u64) -> Option { + #[cfg(test)] + if self.source_after_rows == Some(scanned_rows) { + return Some(slatedb::Error::unavailable( + "injected legacy source transport failure".to_owned(), + )); + } + let _ = (self, scanned_rows); + None + } + + fn passive_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.passive_checkpoint { + return Some(sqlx::Error::Protocol( + "injected passive checkpoint failure".to_owned(), + )); + } + let _ = self; + None + } + + fn final_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.final_checkpoint { + return Some(sqlx::Error::Protocol( + "injected final checkpoint failure".to_owned(), + )); + } + let _ = self; + None + } + + fn restore_automatic_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.restore_automatic_checkpoint { + return Some(sqlx::Error::Protocol( + "injected automatic checkpoint restoration failure".to_owned(), + )); + } + let _ = self; + None + } +} + +struct ImportConnection { + connection: Option>, + retire_on_drop: bool, +} + +impl ImportConnection { + fn new(connection: PoolConnection) -> Self { + Self { + connection: Some(connection), + retire_on_drop: false, + } + } + + fn get_mut(&mut self) -> &mut PoolConnection { + self.connection + .as_mut() + .expect("import connection exists until explicit retirement") + } + + fn retire_if_dropped(&mut self) { + self.retire_on_drop = true; + } + + fn checkpoint_setting_restored(&mut self) { + self.retire_on_drop = false; + } + + async fn retire(mut self) -> Result<(), sqlx::Error> { + let Some(mut connection) = self.connection.take() else { + return Ok(()); + }; + connection.close_on_drop(); + connection.close().await + } +} + +impl Drop for ImportConnection { + fn drop(&mut self) { + if self.retire_on_drop { + if let Some(connection) = &mut self.connection { + connection.close_on_drop(); + } + } + } +} + +struct PendingBlob { + hash: BlobHash, + bytes: Bytes, +} + +#[derive(Clone, Copy, Default)] +struct BatchReport { + imported_rows: u64, + imported_bytes: u64, + existing_rows: u64, + existing_bytes: u64, +} + +impl Database { + /// Strictly imports the legacy SlateDB blob keyspace into a SQLite blob + /// store. + /// + /// # Errors + /// + /// Returns a typed error with partial durable progress if source + /// validation, destination persistence, checkpointing, or connection + /// cleanup fails. + pub async fn import_legacy_blobs_into( + &self, + target: &BlobStore, + ) -> std::result::Result { + self.import_legacy_blobs_with_controls(target, &ImportControls::default()) + .await + } + + async fn import_legacy_blobs_with_controls( + &self, + target: &BlobStore, + controls: &ImportControls, + ) -> std::result::Result { + let mut report = LegacyBlobImportReport::default(); + let result = self + .run_legacy_blob_import(target, controls, &mut report) + .await; + + match result { + Ok(()) => { + debug_import_outcome("complete", &report, None); + Ok(report) + } + Err(failure) => { + debug_import_outcome("failed", &report, Some(failure.kind())); + Err(LegacyBlobImportError { report, failure }) + } + } + } + + async fn run_legacy_blob_import( + &self, + target: &BlobStore, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, + ) -> Result<(), LegacyBlobImportFailure> { + let pool = target + .sqlite_pool_for_legacy_import() + .ok_or(LegacyBlobImportFailure::WrongTargetBackend)?; + let mut connection = pool + .acquire() + .await + .map_err(LegacyBlobImportFailure::AcquireConnection)?; + let previous_automatic_checkpoint = sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await + .map_err(LegacyBlobImportFailure::ReadAutomaticCheckpoint)?; + let mut connection = ImportConnection::new(connection); + // A cancelled PRAGMA future may already have changed connection-local + // state. Arm retirement before the first mutating await so an altered + // connection can never return to the pool without restoration. + connection.retire_if_dropped(); + + let import_result = match set_automatic_checkpoint(connection.get_mut(), 0).await { + Ok(()) => { + #[cfg(test)] + controls.after_automatic_checkpoint_disabled().await; + self.copy_legacy_blobs(connection.get_mut(), controls, report) + .await + } + Err(source) => Err(LegacyBlobImportFailure::DisableAutomaticCheckpoint(source)), + }; + + let restore_result = if let Some(error) = controls.restore_automatic_checkpoint_error() { + Err(error) + } else { + set_automatic_checkpoint(connection.get_mut(), previous_automatic_checkpoint).await + }; + + if let Err(source) = restore_result { + let retirement_error = connection.retire().await.err(); + return Err(LegacyBlobImportFailure::RestoreAutomaticCheckpoint { + source, + prior: import_result.err().map(Box::new), + retirement_error, + }); + } + + connection.checkpoint_setting_restored(); + import_result + } + + async fn copy_legacy_blobs( + &self, + connection: &mut PoolConnection, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, + ) -> Result<(), LegacyBlobImportFailure> { + let source = self + .open_db() + .await + .map_err(LegacyBlobImportFailure::OpenSource)?; + let prefix = SlateKey::new("blobs").with("sha256").into_prefix(); + let prefix_bytes = prefix.as_ref().to_vec(); + let mut entries = source + .scan_prefix(&prefix_bytes) + .await + .map_err(LegacyBlobImportFailure::OpenSourceScan)?; + let mut pending = Vec::with_capacity(MAX_BATCH_ROWS); + let mut pending_bytes = 0_u64; + let mut bytes_since_checkpoint = 0_u64; + + loop { + if let Some(source) = controls.source_scan_error(report.scanned_rows) { + return Err(LegacyBlobImportFailure::ReadSourceScan(source)); + } + let Some(entry) = entries + .next() + .await + .map_err(LegacyBlobImportFailure::ReadSourceScan)? + else { + break; + }; + checked_add(&mut report.scanned_rows, 1)?; + let value_bytes = usize_to_u64(entry.value.len())?; + checked_add(&mut report.scanned_bytes, value_bytes)?; + + let hash = validate_source_entry(&entry.key, &entry.value, &prefix_bytes, report)?; + + let would_exceed_rows = pending.len() == MAX_BATCH_ROWS; + let would_exceed_bytes = pending_bytes + .checked_add(value_bytes) + .ok_or(LegacyBlobImportFailure::CounterOverflow)? + > MAX_BATCH_BYTES; + if !pending.is_empty() && (would_exceed_rows || would_exceed_bytes) { + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + } + + pending.push(PendingBlob { + hash, + bytes: entry.value, + }); + checked_add(&mut pending_bytes, value_bytes)?; + + if value_bytes > MAX_BATCH_BYTES { + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + } + } + + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + run_checkpoint(connection, CheckpointKind::Final, controls).await + } +} + +fn validate_source_entry( + key: &[u8], + value: &[u8], + prefix: &[u8], + report: &mut LegacyBlobImportReport, +) -> Result { + let Some(suffix) = key.strip_prefix(prefix) else { + return invalid_source_row(report, LegacyBlobImportFailure::InvalidSourceKey); + }; + let canonical = suffix.len() == 64 + && suffix + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)); + if !canonical { + return invalid_source_row(report, LegacyBlobImportFailure::InvalidSourceKey); + } + + let hash = std::str::from_utf8(suffix) + .ok() + .and_then(|value| value.parse().ok()) + .ok_or(LegacyBlobImportFailure::InvalidSourceKey); + let hash = match hash { + Ok(hash) => hash, + Err(failure) => return invalid_source_row(report, failure), + }; + if BlobHash::new(value) != hash { + return invalid_source_row(report, LegacyBlobImportFailure::SourceDigestMismatch); + } + Ok(hash) +} + +fn invalid_source_row( + report: &mut LegacyBlobImportReport, + failure: LegacyBlobImportFailure, +) -> Result { + checked_add(&mut report.invalid_rows, 1)?; + Err(failure) +} + +async fn commit_pending_batch( + connection: &mut PoolConnection, + pending: &mut Vec, + pending_bytes: &mut u64, + bytes_since_checkpoint: &mut u64, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, +) -> Result<(), LegacyBlobImportFailure> { + if pending.is_empty() { + return Ok(()); + } + + let batch = std::mem::take(pending); + *pending_bytes = 0; + let batch_report = commit_batch(connection, batch, report).await?; + + let mut updated = *report; + checked_add(&mut updated.imported_rows, batch_report.imported_rows)?; + checked_add(&mut updated.imported_bytes, batch_report.imported_bytes)?; + checked_add(&mut updated.existing_rows, batch_report.existing_rows)?; + checked_add(&mut updated.existing_bytes, batch_report.existing_bytes)?; + checked_add(&mut updated.committed_batches, 1)?; + *report = updated; + + checked_add(bytes_since_checkpoint, batch_report.imported_bytes)?; + if *bytes_since_checkpoint >= PASSIVE_CHECKPOINT_BYTES { + run_checkpoint(connection, CheckpointKind::Passive, controls).await?; + checked_add(&mut report.passive_checkpoints, 1)?; + *bytes_since_checkpoint = 0; + } + + Ok(()) +} + +async fn commit_batch( + connection: &mut PoolConnection, + batch: Vec, + report: &mut LegacyBlobImportReport, +) -> Result { + let mut transaction = connection + .begin() + .await + .map_err(LegacyBlobImportFailure::BeginTransaction)?; + let batch_result = async { + let mut batch_report = BatchReport::default(); + for blob in batch { + let value_bytes = usize_to_u64(blob.bytes.len())?; + let result = sqlx::query( + "INSERT INTO blobs (hash, data) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", + ) + .bind(blob.hash.to_string()) + .bind(blob.bytes.as_ref()) + .execute(&mut *transaction) + .await + .map_err(LegacyBlobImportFailure::InsertDestination)?; + + if result.rows_affected() == 1 { + checked_add(&mut batch_report.imported_rows, 1)?; + checked_add(&mut batch_report.imported_bytes, value_bytes)?; + continue; + } + + let stored: Vec = sqlx::query_scalar("SELECT data FROM blobs WHERE hash = ?") + .bind(blob.hash.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(LegacyBlobImportFailure::ReadDestination)?; + if stored != blob.bytes { + checked_add(&mut report.conflicting_rows, 1)?; + return Err(LegacyBlobImportFailure::DestinationConflict); + } + checked_add(&mut batch_report.existing_rows, 1)?; + checked_add(&mut batch_report.existing_bytes, value_bytes)?; + } + Ok(batch_report) + } + .await; + + match batch_result { + Ok(batch_report) => { + transaction + .commit() + .await + .map_err(LegacyBlobImportFailure::CommitTransaction)?; + Ok(batch_report) + } + Err(prior) => match transaction.rollback().await { + Ok(()) => Err(prior), + Err(source) => Err(LegacyBlobImportFailure::RollbackTransaction { + source, + prior: Box::new(prior), + }), + }, + } +} + +#[derive(Clone, Copy)] +enum CheckpointKind { + Passive, + Final, +} + +async fn run_checkpoint( + connection: &mut PoolConnection, + kind: CheckpointKind, + controls: &ImportControls, +) -> Result<(), LegacyBlobImportFailure> { + let (statement, injected_error) = match kind { + CheckpointKind::Passive => ( + "PRAGMA wal_checkpoint(PASSIVE)", + controls.passive_checkpoint_error(), + ), + CheckpointKind::Final => ( + "PRAGMA wal_checkpoint(TRUNCATE)", + controls.final_checkpoint_error(), + ), + }; + if let Some(source) = injected_error { + return Err(match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpoint(source), + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpoint(source), + }); + } + + let result = sqlx::query_as::<_, (i64, i64, i64)>(statement) + .fetch_one(&mut **connection) + .await; + let (busy, _, _) = result.map_err(|source| match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpoint(source), + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpoint(source), + })?; + if busy != 0 { + return Err(match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpointBusy, + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpointBusy, + }); + } + Ok(()) +} + +async fn set_automatic_checkpoint( + connection: &mut PoolConnection, + pages: i64, +) -> Result<(), sqlx::Error> { + // `pages` comes directly from SQLite as an integer, so the dynamic PRAGMA + // contains no caller-controlled text and cannot change the SQL shape. + let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); + sqlx::query(statement).execute(&mut **connection).await?; + Ok(()) +} + +fn usize_to_u64(value: usize) -> Result { + u64::try_from(value).map_err(|_| LegacyBlobImportFailure::CounterOverflow) +} + +fn checked_add(value: &mut u64, amount: u64) -> Result<(), LegacyBlobImportFailure> { + *value = value + .checked_add(amount) + .ok_or(LegacyBlobImportFailure::CounterOverflow)?; + Ok(()) +} + +fn debug_import_outcome( + outcome: &'static str, + report: &LegacyBlobImportReport, + failure_kind: Option<&'static str>, +) { + debug!( + outcome, + failure_kind, + scanned_rows = report.scanned_rows, + scanned_bytes = report.scanned_bytes, + imported_rows = report.imported_rows, + imported_bytes = report.imported_bytes, + existing_rows = report.existing_rows, + existing_bytes = report.existing_bytes, + invalid_rows = report.invalid_rows, + conflicting_rows = report.conflicting_rows, + committed_batches = report.committed_batches, + passive_checkpoints = report.passive_checkpoints, + "Legacy blob import finished" + ); +} + +#[cfg(test)] +mod tests { + use std::fmt::{self, Write as _}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use bytes::Bytes; + use fabro_types::BlobHash; + use object_store::memory::InMemory; + use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; + use tokio::sync::Barrier; + use tracing::field::{Field, Visit}; + use tracing::instrument::WithSubscriber as _; + use tracing::span::{Attributes, Id, Record}; + use tracing::{Event, Metadata, Subscriber}; + + use super::{ + ImportControls, LegacyBlobImportFailure, LegacyBlobImportReport, MAX_BATCH_BYTES, + PASSIVE_CHECKPOINT_BYTES, set_automatic_checkpoint, + }; + use crate::keys::SlateKey; + use crate::{BlobStore, Database}; + + type TestResult = std::result::Result>; + + struct TestContext { + _dir: tempfile::TempDir, + source: Database, + source_db: slatedb::Db, + sqlite: sqlx::SqlitePool, + target: BlobStore, + } + + impl TestContext { + async fn new() -> TestResult { + let source = Database::new( + Arc::new(InMemory::new()), + "legacy-blob-import-tests", + Duration::from_millis(1), + None, + ); + let source_db = source.open_db().await?; + let dir = tempfile::tempdir()?; + let sqlite = fabro_db::Database::connect(dir.path().join("fabro.sqlite3")).await?; + sqlite.migrate().await?; + let sqlite = sqlite.clone_pool(); + let target = BlobStore::new(sqlite.clone()); + Ok(Self { + _dir: dir, + source, + source_db, + sqlite, + target, + }) + } + + async fn new_with_single_sqlite_connection() -> TestResult { + let mut context = Self::new().await?; + let options = context.sqlite.connect_options().as_ref().clone(); + context.sqlite.close().await; + let sqlite = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + context.target = BlobStore::new(sqlite.clone()); + context.sqlite = sqlite; + Ok(context) + } + + async fn put_blob(&self, bytes: &[u8]) -> TestResult { + let hash = BlobHash::new(bytes); + let key = SlateKey::new("blobs").with("sha256").with(hash); + self.source_db.put(key, bytes).await?; + Ok(hash) + } + + async fn put_raw(&self, key: Vec, bytes: &[u8]) -> TestResult<()> { + self.source_db.put(key, bytes).await?; + Ok(()) + } + + async fn source_entries(&self) -> TestResult, Vec)>> { + let mut entries = self.source_db.scan_prefix(Vec::::new()).await?; + let mut snapshot = Vec::new(); + while let Some(entry) = entries.next().await? { + snapshot.push((entry.key.to_vec(), entry.value.to_vec())); + } + Ok(snapshot) + } + + async fn destination_rows(&self) -> TestResult { + Ok(sqlx::query_scalar("SELECT COUNT(*) FROM blobs") + .fetch_one(&self.sqlite) + .await?) + } + + async fn insert_destination(&self, hash: BlobHash, bytes: &[u8]) -> TestResult<()> { + sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") + .bind(hash.to_string()) + .bind(bytes) + .execute(&self.sqlite) + .await?; + Ok(()) + } + + async fn delete_destination(&self, hash: BlobHash) -> TestResult<()> { + sqlx::query("DELETE FROM blobs WHERE hash = ?") + .bind(hash.to_string()) + .execute(&self.sqlite) + .await?; + Ok(()) + } + + async fn set_automatic_checkpoint(&self, pages: i64) -> TestResult<()> { + let mut connection = self.sqlite.acquire().await?; + let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); + sqlx::query(statement).execute(&mut *connection).await?; + Ok(()) + } + + async fn automatic_checkpoint(&self) -> TestResult { + let mut connection = self.sqlite.acquire().await?; + Ok(sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await?) + } + + async fn import(&self) -> TestResult { + Ok(self.source.import_legacy_blobs_into(&self.target).await?) + } + } + + fn legacy_prefix() -> Vec { + SlateKey::new("blobs") + .with("sha256") + .into_prefix() + .as_ref() + .to_vec() + } + + async fn seed_blobs( + context: &TestContext, + count: usize, + ) -> TestResult)>> { + let mut blobs = Vec::with_capacity(count); + for index in 0..count { + let bytes = format!("legacy-blob-{index:04}").into_bytes(); + let hash = context.put_blob(&bytes).await?; + blobs.push((hash, bytes)); + } + blobs.sort_by_key(|(hash, _)| *hash); + Ok(blobs) + } + + #[tokio::test] + async fn imports_valid_raw_blobs_and_reports_aggregate_progress() -> TestResult<()> { + let context = TestContext::new().await?; + let binary = [0_u8, 0xff, 0x80, b'a']; + let binary_hash = context.put_blob(&binary).await?; + let empty_hash = context.put_blob(b"").await?; + let source_before = context.source_entries().await?; + + let report = context.import().await?; + + assert_eq!(report, LegacyBlobImportReport { + scanned_rows: 2, + scanned_bytes: 4, + imported_rows: 2, + imported_bytes: 4, + committed_batches: 1, + ..LegacyBlobImportReport::default() + }); + assert_eq!( + context.target.read(&binary_hash).await?, + Some(Bytes::copy_from_slice(&binary)) + ); + assert_eq!(context.target.read(&empty_hash).await?, Some(Bytes::new())); + assert_eq!(context.source_entries().await?, source_before); + Ok(()) + } + + #[tokio::test] + async fn empty_source_succeeds_without_committing_a_batch() -> TestResult<()> { + let context = TestContext::new().await?; + + let report = context.import().await?; + + assert_eq!(report, LegacyBlobImportReport::default()); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn scan_is_limited_to_the_exact_legacy_prefix() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"included").await?; + context + .source_db + .put( + SlateKey::new("blobs").with("other").with("ignored"), + b"nearby", + ) + .await?; + + let report = context.import().await?; + + assert_eq!(report.scanned_rows, 1); + assert_eq!(report.imported_rows, 1); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn rejects_every_noncanonical_legacy_key_shape() -> TestResult<()> { + let valid_hash = BlobHash::new(b"valid-shape").to_string(); + let cases = [ + ("empty", Vec::new()), + ("short", vec![b'0'; 63]), + ("long", vec![b'0'; 65]), + ("uppercase", vec![b'A'; 64]), + ("non_hex", vec![b'g'; 64]), + ("non_utf8", vec![0xff; 64]), + ( + "extra_segment", + [valid_hash.as_bytes(), b"\0extra"].concat(), + ), + ]; + + for (case, suffix) in cases { + let context = TestContext::new().await?; + let mut key = legacy_prefix(); + key.extend_from_slice(&suffix); + context.put_raw(key, b"source-bytes").await?; + let source_before = context.source_entries().await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("noncanonical key should fail import"); + + assert_eq!(error.report().scanned_rows, 1, "case {case}"); + assert_eq!(error.report().invalid_rows, 1, "case {case}"); + assert_eq!(error.report().committed_batches, 0, "case {case}"); + assert!( + matches!(&error.failure, LegacyBlobImportFailure::InvalidSourceKey), + "case {case}: {error:?}" + ); + assert_eq!(context.destination_rows().await?, 0, "case {case}"); + assert_eq!( + context.source_entries().await?, + source_before, + "case {case}" + ); + } + Ok(()) + } + + #[tokio::test] + async fn rejects_source_bytes_that_do_not_match_the_key_digest() -> TestResult<()> { + let context = TestContext::new().await?; + let mut key = legacy_prefix(); + key.extend_from_slice(BlobHash::new(b"expected").to_string().as_bytes()); + context.put_raw(key, b"different").await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("digest mismatch should fail import"); + + assert_eq!(error.report().scanned_rows, 1); + assert_eq!(error.report().scanned_bytes, 9); + assert_eq!(error.report().invalid_rows, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::SourceDigestMismatch + )); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn invalid_source_row_discards_the_uncommitted_pending_batch() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"valid-but-not-yet-committed").await?; + let mut invalid_key = legacy_prefix(); + invalid_key.extend_from_slice(&[b'z'; 64]); + context.put_raw(invalid_key, b"invalid").await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("invalid row should stop the import"); + + assert_eq!(error.report().scanned_rows, 2); + assert_eq!(error.report().invalid_rows, 1); + assert_eq!(error.report().imported_rows, 0); + assert_eq!(error.report().committed_batches, 0); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn equal_destination_rows_are_retry_progress() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"first").await?; + context.put_blob(b"second").await?; + + let first = context.import().await?; + let second = context.import().await?; + + assert_eq!(first.imported_rows, 2); + assert_eq!(second.scanned_rows, 2); + assert_eq!(second.imported_rows, 0); + assert_eq!(second.existing_rows, 2); + assert_eq!(second.existing_bytes, 11); + assert_eq!(second.committed_batches, 1); + assert_eq!(context.destination_rows().await?, 2); + Ok(()) + } + + #[tokio::test] + async fn destination_conflict_rolls_back_the_current_batch() -> TestResult<()> { + let context = TestContext::new().await?; + let blobs = seed_blobs(&context, 2).await?; + context + .insert_destination(blobs[1].0, b"conflicting-destination-bytes") + .await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("differing destination row should fail import"); + + assert_eq!(error.report().scanned_rows, 2); + assert_eq!(error.report().conflicting_rows, 1); + assert_eq!(error.report().imported_rows, 0); + assert_eq!(error.report().committed_batches, 0); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::DestinationConflict + )); + assert_eq!(context.target.read(&blobs[0].0).await?, None); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn interrupted_import_preserves_commits_and_retry_converges() -> TestResult<()> { + let context = TestContext::new().await?; + let blobs = seed_blobs(&context, 101).await?; + let conflict = blobs[100].0; + context + .insert_destination(conflict, b"conflicting-destination-bytes") + .await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("last-row conflict should interrupt import"); + + assert_eq!(error.report().scanned_rows, 101); + assert_eq!(error.report().imported_rows, 100); + assert_eq!(error.report().conflicting_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert_eq!(context.destination_rows().await?, 101); + + context.delete_destination(conflict).await?; + let retry = context.import().await?; + assert_eq!(retry.scanned_rows, 101); + assert_eq!(retry.existing_rows, 100); + assert_eq!(retry.imported_rows, 1); + assert_eq!(retry.committed_batches, 2); + assert_eq!(context.destination_rows().await?, 101); + Ok(()) + } + + #[tokio::test] + async fn row_limit_splits_one_hundred_and_one_values() -> TestResult<()> { + let context = TestContext::new().await?; + seed_blobs(&context, 101).await?; + + let report = context.import().await?; + + assert_eq!(report.imported_rows, 101); + assert_eq!(report.committed_batches, 2); + Ok(()) + } + + #[tokio::test] + async fn byte_limit_splits_batches_and_allows_exact_limit() -> TestResult<()> { + let split_context = TestContext::new().await?; + split_context.put_blob(&vec![b'a'; 600 * 1024]).await?; + split_context.put_blob(&vec![b'b'; 600 * 1024]).await?; + let split = split_context.import().await?; + assert_eq!(split.committed_batches, 2); + + let exact_context = TestContext::new().await?; + exact_context + .put_blob(&vec![b'x'; usize::try_from(MAX_BATCH_BYTES)?]) + .await?; + exact_context.put_blob(b"").await?; + let exact = exact_context.import().await?; + assert_eq!(exact.committed_batches, 1); + Ok(()) + } + + #[tokio::test] + async fn oversized_value_is_committed_alone_between_surrounding_batches() -> TestResult<()> { + let context = TestContext::new().await?; + let oversized = vec![b'o'; usize::try_from(MAX_BATCH_BYTES + 1)?]; + let oversized_hash = BlobHash::new(&oversized); + let mut lower = None; + let mut upper = None; + for index in 0_u32..10_000 { + let bytes = format!("surrounding-{index}").into_bytes(); + let hash = BlobHash::new(&bytes); + if hash < oversized_hash && lower.is_none() { + lower = Some(bytes); + } else if hash > oversized_hash && upper.is_none() { + upper = Some(bytes); + } + if lower.is_some() && upper.is_some() { + break; + } + } + let lower = lower.expect("search should find a hash below the oversized value"); + let upper = upper.expect("search should find a hash above the oversized value"); + context.put_blob(&lower).await?; + context.put_blob(&oversized).await?; + context.put_blob(&upper).await?; + + let report = context.import().await?; + + assert_eq!(report.imported_rows, 3); + assert_eq!(report.committed_batches, 3); + Ok(()) + } + + #[tokio::test] + async fn passive_checkpoint_runs_once_after_crossing_each_batch_threshold() -> TestResult<()> { + let crossing_context = TestContext::new().await?; + crossing_context + .put_blob(&vec![b'c'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) + .await?; + let crossing = crossing_context.import().await?; + assert_eq!(crossing.passive_checkpoints, 1); + + let multi_context = TestContext::new().await?; + multi_context + .put_blob(&vec![ + b'm'; + usize::try_from(PASSIVE_CHECKPOINT_BYTES * 2 + 1)? + ]) + .await?; + let multiple = multi_context.import().await?; + assert_eq!(multiple.passive_checkpoints, 1); + Ok(()) + } + + #[tokio::test] + async fn passive_checkpoint_failure_returns_durable_partial_report() -> TestResult<()> { + let context = TestContext::new().await?; + context + .put_blob(&vec![b'p'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) + .await?; + let controls = ImportControls { + passive_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected passive checkpoint should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert_eq!(error.report().passive_checkpoints, 0); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::PassiveCheckpoint(sqlx::Error::Protocol(_)) + )); + let failure_source = std::error::Error::source(&error) + .and_then(std::error::Error::source) + .expect("checkpoint failure should preserve its SQL source"); + assert!(failure_source.downcast_ref::().is_some()); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn final_checkpoint_failure_returns_committed_progress() -> TestResult<()> { + let context = TestContext::new().await?; + context + .put_blob(b"committed-before-final-checkpoint") + .await?; + let controls = ImportControls { + final_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected final checkpoint should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::FinalCheckpoint(sqlx::Error::Protocol(_)) + )); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn source_transport_failure_preserves_commits_and_typed_source() -> TestResult<()> { + let context = TestContext::new().await?; + seed_blobs(&context, 101).await?; + let controls = ImportControls { + source_after_rows: Some(101), + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected source transport failure should stop import"); + + assert_eq!(error.report().scanned_rows, 101); + assert_eq!(error.report().imported_rows, 100); + assert_eq!(error.report().committed_batches, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::ReadSourceScan(_) + )); + let failure_source = std::error::Error::source(&error) + .and_then(std::error::Error::source) + .expect("transport failure should preserve its SlateDB source"); + assert!(failure_source.downcast_ref::().is_some()); + + let retry = context.import().await?; + assert_eq!(retry.existing_rows, 100); + assert_eq!(retry.imported_rows, 1); + assert_eq!(context.destination_rows().await?, 101); + Ok(()) + } + + #[tokio::test] + async fn restoration_failure_preserves_the_prior_source_chain() -> TestResult<()> { + let context = TestContext::new().await?; + let controls = ImportControls { + source_after_rows: Some(0), + restore_automatic_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("both injected failures should fail import"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut saw_slate_source = false; + while let Some(source) = current { + saw_slate_source |= source.downcast_ref::().is_some(); + current = source.source(); + } + assert!(saw_slate_source, "prior source was absent from the chain"); + assert!( + error + .cleanup_errors() + .any(|source| source.downcast_ref::().is_some()), + "restoration source was absent from cleanup errors" + ); + Ok(()) + } + + #[tokio::test] + async fn automatic_checkpoint_setting_is_restored_after_success_and_failure() -> TestResult<()> + { + let success = TestContext::new_with_single_sqlite_connection().await?; + success.set_automatic_checkpoint(37).await?; + success.put_blob(b"success").await?; + success.import().await?; + assert_eq!(success.automatic_checkpoint().await?, 37); + + let failure = TestContext::new_with_single_sqlite_connection().await?; + failure.set_automatic_checkpoint(41).await?; + let mut invalid_key = legacy_prefix(); + invalid_key.extend_from_slice(&[b'z'; 64]); + failure.put_raw(invalid_key, b"invalid").await?; + failure + .source + .import_legacy_blobs_into(&failure.target) + .await + .expect_err("invalid source should fail import"); + assert_eq!(failure.automatic_checkpoint().await?, 41); + Ok(()) + } + + #[tokio::test] + async fn cancellation_retires_a_connection_with_disabled_checkpointing() -> TestResult<()> { + let source = Database::new( + Arc::new(InMemory::new()), + "legacy-blob-import-cancellation-test", + Duration::from_millis(1), + None, + ); + let dir = tempfile::tempdir()?; + let options = SqliteConnectOptions::new() + .filename(dir.path().join("fabro.sqlite3")) + .create_if_missing(true) + .journal_mode(SqliteJournalMode::Wal); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + sqlx::query("CREATE TABLE blobs (hash TEXT PRIMARY KEY NOT NULL, data BLOB NOT NULL)") + .execute(&pool) + .await?; + let target = Arc::new(BlobStore::new(pool.clone())); + + let mut connection = pool.acquire().await?; + set_automatic_checkpoint(&mut connection, 73).await?; + drop(connection); + + let barrier = Arc::new(Barrier::new(2)); + let controls = ImportControls { + after_automatic_checkpoint_disabled: Some(Arc::clone(&barrier)), + ..ImportControls::default() + }; + let task = tokio::spawn({ + let source = source.clone(); + let target = Arc::clone(&target); + async move { + let mut report = LegacyBlobImportReport::default(); + source + .run_legacy_blob_import(&target, &controls, &mut report) + .await + } + }); + + barrier.wait().await; + task.abort(); + let join_error = task.await.expect_err("aborted import should be cancelled"); + assert!(join_error.is_cancelled()); + + let mut connection = pool.acquire().await?; + let observed: i64 = sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await?; + assert_ne!(observed, 0); + Ok(()) + } + + #[tokio::test] + async fn failed_connection_restoration_retires_the_connection() -> TestResult<()> { + let context = TestContext::new().await?; + context.set_automatic_checkpoint(43).await?; + context.put_blob(b"committed-before-restore").await?; + let controls = ImportControls { + restore_automatic_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected restoration failure should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::RestoreAutomaticCheckpoint { + source: sqlx::Error::Protocol(_), + .. + } + )); + assert_ne!(context.automatic_checkpoint().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn slate_backed_target_is_rejected_before_scanning() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"never-scanned").await?; + let slate_target = context.source.blobs().await?; + + let error = context + .source + .import_legacy_blobs_into(&slate_target) + .await + .expect_err("Slate target should be rejected"); + + assert_eq!(*error.report(), LegacyBlobImportReport::default()); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::WrongTargetBackend + )); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn logs_and_error_rendering_expose_counts_but_not_row_data() -> TestResult<()> { + let context = TestContext::new().await?; + let sensitive_key_fragment = "sensitive-invalid-legacy-key"; + let sensitive_content = b"sensitive-blob-content"; + let mut key = legacy_prefix(); + key.extend_from_slice(sensitive_key_fragment.as_bytes()); + context.put_raw(key, sensitive_content).await?; + let capture = CapturedEvents::default(); + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .with_subscriber(capture.clone()) + .await + .expect_err("invalid key should fail import"); + + let rendered = format!("{error} {error:?}"); + let events = capture.events().join("\n"); + for output in [&rendered, &events] { + assert!(!output.contains(sensitive_key_fragment)); + assert!(!output.contains(std::str::from_utf8(sensitive_content)?)); + } + assert!(events.contains("scanned_rows=1"), "captured: {events}"); + assert!(events.contains("invalid_rows=1"), "captured: {events}"); + assert!( + events.contains("failure_kind=\"invalid_source_key\""), + "captured: {events}" + ); + assert_eq!(capture.events().len(), 1); + Ok(()) + } + + #[derive(Clone, Default)] + struct CapturedEvents { + events: Arc>>, + next_span_id: Arc, + } + + impl CapturedEvents { + fn events(&self) -> Vec { + self.events + .lock() + .expect("captured tracing events mutex should not be poisoned") + .clone() + } + } + + impl Subscriber for CapturedEvents { + fn enabled(&self, _metadata: &Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &Attributes<'_>) -> Id { + Id::from_u64(self.next_span_id.fetch_add(1, Ordering::Relaxed) + 1) + } + + fn record(&self, _span: &Id, _values: &Record<'_>) {} + + fn record_follows_from(&self, _span: &Id, _follows: &Id) {} + + fn event(&self, event: &Event<'_>) { + let mut visitor = CapturedFields::default(); + event.record(&mut visitor); + self.events + .lock() + .expect("captured tracing events mutex should not be poisoned") + .push(visitor.output); + } + + fn enter(&self, _span: &Id) {} + + fn exit(&self, _span: &Id) {} + } + + #[derive(Default)] + struct CapturedFields { + output: String, + } + + impl Visit for CapturedFields { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + write!(&mut self.output, "{}={value:?};", field.name()) + .expect("writing tracing fields to String cannot fail"); + } + } +} diff --git a/lib/components/fabro-store/src/lib.rs b/lib/components/fabro-store/src/lib.rs index 00a95d4e0..c06c3b852 100644 --- a/lib/components/fabro-store/src/lib.rs +++ b/lib/components/fabro-store/src/lib.rs @@ -5,6 +5,7 @@ mod blob_store; mod error; mod keyed_mutex; mod keys; +mod legacy_blob_import; mod record; mod run_sessions; mod run_state; @@ -25,6 +26,7 @@ pub use fabro_types::{ BlobHash, EventEnvelope, PendingInterviewRecord, Run, RunProjection, StageId, StageProjection, }; pub use keyed_mutex::{KeyedMutex, KeyedMutexGuard}; +pub use legacy_blob_import::{LegacyBlobImportError, LegacyBlobImportReport}; pub use run_sessions::{ ProjectedRunSession, project_run_session, project_run_session_with_context, project_run_sessions, diff --git a/lib/components/fabro-store/src/run_state.rs b/lib/components/fabro-store/src/run_state.rs index 0c59af57a..da54a2d29 100644 --- a/lib/components/fabro-store/src/run_state.rs +++ b/lib/components/fabro-store/src/run_state.rs @@ -1040,6 +1040,7 @@ fn projection_from_created(event: &EventEnvelope) -> Result { graph: props.graph.clone(), graph_source: props.workflow_source.clone(), workflow_slug: props.workflow_slug.clone(), + workflow_version_id: props.workflow_version_id, automation: props.automation.clone(), source_directory: props.source_directory.clone(), labels, @@ -2367,12 +2368,37 @@ mod tests { let projection = RunProjection::apply_events(&[event]).unwrap(); assert_eq!(projection.retried_from, None); + assert_eq!(projection.spec.workflow_version_id, None); assert_eq!( build_summary(&projection, &fixtures::RUN_1).retried_from, None ); } + #[test] + fn run_created_projects_workflow_version_id_into_spec() { + let workflow_version_id = test_support::test_workflow_version_id(); + let event = test_raw_event( + 1, + "run.created", + &json!({ + "settings": WorkflowSettings::default(), + "graph": Graph::new("test"), + "workflow_version_id": workflow_version_id, + "labels": {}, + "provenance": test_support::test_run_provenance() + }), + None, + ); + + let projection = RunProjection::apply_events(&[event]).unwrap(); + + assert_eq!( + projection.spec.workflow_version_id, + Some(workflow_version_id) + ); + } + #[test] fn run_created_replay_ignores_unknown_properties() { let provenance = test_support::test_run_provenance(); diff --git a/lib/components/fabro-store/src/run_summary_store.rs b/lib/components/fabro-store/src/run_summary_store.rs index dcbec847e..8017509f3 100644 --- a/lib/components/fabro-store/src/run_summary_store.rs +++ b/lib/components/fabro-store/src/run_summary_store.rs @@ -595,6 +595,7 @@ mod tests { graph: Graph::new("test"), graph_source: None, workflow_slug: Some("test-workflow".to_string()), + workflow_version_id: None, automation: None, source_directory: None, labels: HashMap::new(), diff --git a/lib/components/fabro-store/src/slate/mod.rs b/lib/components/fabro-store/src/slate/mod.rs index 6158e18a4..d17c75518 100644 --- a/lib/components/fabro-store/src/slate/mod.rs +++ b/lib/components/fabro-store/src/slate/mod.rs @@ -100,7 +100,7 @@ impl Database { self.base_prefix.clone() } - async fn open_db(&self) -> Result { + pub(crate) async fn open_db(&self) -> Result { let db = self .db .get_or_try_init(|| async { @@ -587,6 +587,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("night-sky".to_string()), + workflow_version_id: None, automation: None, source_directory: Some(format!("/tmp/{label}")), labels: std::collections::HashMap::from([("team".to_string(), "infra".to_string())]), diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index 290aacf91..7bedbda70 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -14,6 +14,7 @@ use fabro_validate::Diagnostic; use regex::Regex; use thiserror::Error as ThisError; +use crate::event::RunEventPersistenceError; use crate::outcome::{FailureDetail, Outcome, StageOutcome}; /// Classify an LLM error into a `FailureCategory` based on its structure. @@ -721,6 +722,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: RunEventPersistenceError) -> Self { + Self::engine_with_source("run event persistence failed", err) + } +} + impl From for Error { fn from(err: fabro_checkpoint::MetadataError) -> Self { match err { diff --git a/lib/components/fabro-workflow/src/event.rs b/lib/components/fabro-workflow/src/event.rs index a5c1f583e..656b69922 100644 --- a/lib/components/fabro-workflow/src/event.rs +++ b/lib/components/fabro-workflow/src/event.rs @@ -18,7 +18,7 @@ pub use self::redaction::{ build_redacted_event_payload, event_payload_from_redacted_json, redacted_event_json, }; pub use self::sink::{ - RunEventLogger, RunEventSink, StoreProgressLogger, append_event, append_event_if, - append_event_to_sink, + RunEventLogger, RunEventPersistenceError, RunEventSink, StoreProgressLogger, append_event, + append_event_if, append_event_to_sink, }; pub use crate::stage_scope::StageScope; diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index 85a02769a..7d9af8f79 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -71,6 +71,7 @@ fn event_body_from_event(event: &Event) -> EventBody { labels, source_directory, workflow_slug, + workflow_version_id, automation, provenance, manifest_blob, @@ -90,6 +91,7 @@ fn event_body_from_event(event: &Event) -> EventBody { labels: labels.clone(), source_directory: source_directory.clone(), workflow_slug: workflow_slug.clone(), + workflow_version_id: *workflow_version_id, automation: automation.clone(), provenance: provenance.clone(), manifest_blob: *manifest_blob, @@ -1445,7 +1447,7 @@ mod tests { use ::fabro_types::{ AutomationRef, EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel, RunProvenance, StageId, SystemActorKind, fixtures, - run_event as fabro_types, + run_event as fabro_types, test_support, }; use chrono::Utc; use fabro_agent::{ @@ -2828,6 +2830,7 @@ mod tests { name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }; + let workflow_version_id = test_support::test_workflow_version_id(); let stored = to_run_event(&fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, @@ -2838,6 +2841,7 @@ mod tests { labels: BTreeMap::default(), source_directory: Some("/tmp/run".to_string()), workflow_slug: None, + workflow_version_id: Some(workflow_version_id), automation: Some(automation.clone()), provenance, manifest_blob: None, @@ -2854,6 +2858,7 @@ mod tests { panic!("expected run.created body"); }; assert_eq!(props.automation, Some(automation)); + assert_eq!(props.workflow_version_id, Some(workflow_version_id)); } #[test] diff --git a/lib/components/fabro-workflow/src/event/events.rs b/lib/components/fabro-workflow/src/event/events.rs index 03ffe816a..f916f7985 100644 --- a/lib/components/fabro-workflow/src/event/events.rs +++ b/lib/components/fabro-workflow/src/event/events.rs @@ -6,7 +6,7 @@ use ::fabro_types::{ PairTarget, ParallelBranchId, ParallelBranchResult, PendingReason, PermissionLevel, Principal, PullRequestCreationId, PullRequestLink, ReviewTarget, RunFailure, RunId, RunNoticeLevel, RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource, RunTiming, - SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason, + SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason, WorkflowVersionId, run_event as fabro_types, }; use fabro_agent::{AgentEvent, SandboxEvent}; @@ -24,34 +24,36 @@ use crate::outcome::{BilledModelUsage, FailureDetail, Outcome}; )] pub enum Event { RunCreated { - run_id: RunId, - title: Option, - settings: serde_json::Value, - graph: serde_json::Value, + run_id: RunId, + title: Option, + settings: serde_json::Value, + graph: serde_json::Value, #[serde(default, skip_serializing_if = "Option::is_none")] - workflow_source: Option, - labels: BTreeMap, + workflow_source: Option, + labels: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] - source_directory: Option, + source_directory: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - workflow_slug: Option, + workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - automation: Option, - provenance: RunProvenance, + workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - manifest_blob: Option, + automation: Option, + provenance: RunProvenance, #[serde(default, skip_serializing_if = "Option::is_none")] - spec_blob: Option, + manifest_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - git: Option, + spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - fork_source_ref: Option, + git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - retried_from: Option, + fork_source_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - parent_id: Option, + retried_from: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - web_url: Option, + parent_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + web_url: Option, }, WorkflowRunStarted { name: String, diff --git a/lib/components/fabro-workflow/src/event/sink.rs b/lib/components/fabro-workflow/src/event/sink.rs index 150c69f72..a82f2e20f 100644 --- a/lib/components/fabro-workflow/src/event/sink.rs +++ b/lib/components/fabro-workflow/src/event/sink.rs @@ -5,8 +5,9 @@ use std::sync::Arc; use ::fabro_types::{RunEvent, RunId, RunProjection}; use anyhow::Result; use fabro_store::RunDatabase; +use fabro_util::error::{SharedError, collect_chain}; use tokio::io::{AsyncWrite, AsyncWriteExt}; -use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot}; +use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot, watch}; use super::emitter::Emitter; use super::redaction::{build_redacted_event_payload, redacted_event_json}; @@ -42,9 +43,15 @@ pub async fn append_event_to_sink( sink: &RunEventSink, run_id: &RunId, event: &Event, -) -> Result<()> { +) -> Result<(), RunEventPersistenceError> { let stored = to_run_event(run_id, event); - sink.write_run_event(&stored).await + sink.write_run_event(&stored) + .await + .map_err(|err| RunEventPersistenceError::Write { + run_id: *run_id, + event: stored.body.event_name().to_string(), + source: SharedError::new(err), + }) } #[derive(Clone)] @@ -149,86 +156,100 @@ impl RunEventSink { )] enum RunEventCommand { Event(RunEvent), - Flush(oneshot::Sender<()>), + Flush(oneshot::Sender>), +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum RunEventPersistenceError { + #[error("failed to persist run event {event} for run {run_id}")] + Write { + run_id: RunId, + event: String, + #[source] + source: SharedError, + }, + #[error("run event persistence task stopped")] + TaskStopped, } #[derive(Clone)] pub struct RunEventLogger { - tx: mpsc::UnboundedSender, + tx: mpsc::UnboundedSender, + failure_rx: watch::Receiver>, } impl RunEventLogger { #[must_use] pub fn new(sink: RunEventSink) -> Self { let (tx, mut rx) = mpsc::unbounded_channel(); + let (failure_tx, failure_rx) = watch::channel(None); tokio::spawn(async move { - // A dropped run event is unrecoverable history loss, so the first - // one is an ERROR worth investigating. A broken sink fails for - // every event that follows, so report the rest as a count at flush - // instead of one ERROR per event. Flush runs per stage and per - // agent turn, so only losses since the last summary are reported. - let mut write_failures: u64 = 0; - let mut summarized_failures: u64 = 0; + // The watch channel is the single record of the latched failure: + // the worker is its only writer, so borrowing it here cannot race. while let Some(command) = rx.recv().await { match command { RunEventCommand::Event(event) => { + if failure_tx.borrow().is_some() { + continue; + } if let Err(err) = sink.write_run_event(&event).await { - write_failures += 1; - if write_failures == 1 { - tracing::error!( - run_id = %event.run_id, - event = %event.body.event_name(), - error = %err, - "Failed to write run event", - ); - } else { - tracing::debug!( - run_id = %event.run_id, - event = %event.body.event_name(), - failures = write_failures, - error = %err, - "Failed to write run event", - ); - } + let rendered_error = collect_chain(err.as_ref()).join(": "); + tracing::error!( + run_id = %event.run_id, + event = %event.body.event_name(), + error = %rendered_error, + "Failed to persist run event; stopping workflow", + ); + failure_tx.send_replace(Some(RunEventPersistenceError::Write { + run_id: event.run_id, + event: event.body.event_name().to_string(), + source: SharedError::new(err), + })); } } RunEventCommand::Flush(tx) => { - if write_failures > summarized_failures { - tracing::error!( - lost = write_failures - summarized_failures, - total = write_failures, - "Run events were lost to write failures", - ); - summarized_failures = write_failures; - } - let _ = tx.send(()); + let result = failure_tx.borrow().clone().map_or(Ok(()), Err); + let _ = tx.send(result); } } } }); - Self { tx } + Self { tx, failure_rx } } pub fn register(&self, emitter: &Emitter) { let tx = self.tx.clone(); emitter.on_event(move |event| { if tx.send(RunEventCommand::Event(event.clone())).is_err() { - tracing::warn!("Run event logger channel closed while forwarding event"); + tracing::error!( + run_id = %event.run_id, + event = %event.body.event_name(), + "Run event persistence task stopped while forwarding event", + ); } }); } - pub async fn flush(&self) { + pub async fn wait_for_failure(&self) -> RunEventPersistenceError { + let mut failure_rx = self.failure_rx.clone(); + let failure = failure_rx.wait_for(Option::is_some).await; + match failure { + Ok(failure) => failure + .clone() + .expect("wait_for only returns values matching the predicate"), + Err(_) => RunEventPersistenceError::TaskStopped, + } + } + + pub async fn flush(&self) -> Result<(), RunEventPersistenceError> { let (tx, rx) = oneshot::channel(); if self.tx.send(RunEventCommand::Flush(tx)).is_err() { - tracing::warn!("Run event logger channel closed before flush"); - return; - } - if rx.await.is_err() { - tracing::warn!("Run event logger flush dropped before completion"); + return Err(RunEventPersistenceError::TaskStopped); } + rx.await + .unwrap_or(Err(RunEventPersistenceError::TaskStopped)) } } @@ -249,14 +270,15 @@ impl StoreProgressLogger { self.inner.register(emitter); } - pub async fn flush(&self) { - self.inner.flush().await; + pub async fn flush(&self) -> Result<(), RunEventPersistenceError> { + self.inner.flush().await } } #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use ::fabro_types::{Graph, RunNoticeLevel, WorkflowSettings, fixtures}; use fabro_types::test_support; @@ -279,23 +301,24 @@ mod tests { ); let run_store = store.create_run(&fixtures::RUN_7).await.unwrap(); append_event(&run_store, &fixtures::RUN_7, &Event::RunCreated { - run_id: fixtures::RUN_7, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_7, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -438,7 +461,7 @@ mod tests { logger.register(&emitter); emitter.emit(&Event::RunPaused); - logger.flush().await; + logger.flush().await.unwrap(); let mut reader = BufReader::new(reader); let mut line = String::new(); @@ -447,4 +470,38 @@ mod tests { let payload = event_payload_from_redacted_json(line.trim_end(), &fixtures::RUN_8).unwrap(); assert_eq!(payload.as_value()["event"], "run.paused"); } + + #[tokio::test] + async fn run_event_logger_latches_write_failure_and_preserves_cause_chain() { + let writes = Arc::new(AtomicUsize::new(0)); + let writes_for_sink = Arc::clone(&writes); + let sink = RunEventSink::callback(move |_| { + writes_for_sink.fetch_add(1, Ordering::SeqCst); + async { + Err( + anyhow::anyhow!("request failed with status 413 Payload Too Large") + .context("worker lost canonical run store during append run event"), + ) + } + }); + let logger = RunEventLogger::new(sink); + let emitter = Emitter::new(fixtures::RUN_8); + logger.register(&emitter); + + emitter.emit(&Event::RunPaused); + + let failure = logger.wait_for_failure().await; + let rendered = collect_chain(&failure).join(": "); + assert!(rendered.contains("run.paused"), "{rendered}"); + assert!( + rendered.contains("worker lost canonical run store"), + "{rendered}" + ); + assert!(rendered.contains("413 Payload Too Large"), "{rendered}"); + + emitter.emit(&Event::RunUnpaused); + let flush_failure = logger.flush().await.unwrap_err(); + assert_eq!(collect_chain(&flush_failure), collect_chain(&failure)); + assert_eq!(writes.load(Ordering::SeqCst), 1); + } } diff --git a/lib/components/fabro-workflow/src/git.rs b/lib/components/fabro-workflow/src/git.rs index e7ca4feda..5d72aa43d 100644 --- a/lib/components/fabro-workflow/src/git.rs +++ b/lib/components/fabro-workflow/src/git.rs @@ -352,24 +352,25 @@ mod tests { let store = test_store(); let run = store.create_run(&fixtures::RUN_1).await.unwrap(); append_event(&run, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/git_bridge.rs b/lib/components/fabro-workflow/src/git_bridge.rs new file mode 100644 index 000000000..4f822811a --- /dev/null +++ b/lib/components/fabro-workflow/src/git_bridge.rs @@ -0,0 +1,438 @@ +//! Secret-free Git bridging environment for additional-repository access. +//! +//! When a run declares additional GitHub repositories, every resolved +//! command/tool/ACP environment receives `GIT_CONFIG_COUNT` / +//! `GIT_CONFIG_KEY_n` / `GIT_CONFIG_VALUE_n` entries that make plain Git +//! commands work against the declared set through the managed +//! `GITHUB_TOKEN`: +//! +//! - a credential helper for `https://github.com` that reads `$GITHUB_TOKEN` +//! from the invoking Git process's environment at invocation time, so token +//! refresh flows through per-stage environment resolution with no bridging +//! update; +//! - per-repository `url..insteadOf` rewrites for the +//! `git@github.com:owner/repo[.git]` and +//! `ssh://git@github.com/owner/repo[.git]` SSH spellings of each effective +//! repository. +//! +//! None of the values contain a secret; the token lives only in +//! `GITHUB_TOKEN`. +//! +//! The credential helper is host-scoped to `https://github.com`, not +//! path-scoped. This is safe because the token is scoped server-side to the +//! declared repository set and is only ever offered to github.com. It does +//! change one failure mode for *undeclared* repositories: public HTTPS +//! clones are unaffected (Git tries unauthenticated first), while private +//! undeclared HTTPS repositories fail with a GitHub authorization error +//! instead of a missing-credential error. Both fail; only the diagnostic +//! differs. +//! +//! `insteadOf` matches by string prefix, not exactly: a rule for +//! `owner/repo` also matches `owner/repo-other`. An undeclared repository +//! that shares a declared prefix is therefore rewritten to HTTPS; the scoped +//! token is invalid for it at GitHub, so authority is unchanged, but its Git +//! transport changes from SSH to HTTPS. + +use std::collections::HashMap; + +use fabro_github::{GITHUB_CREDENTIAL_HELPER, GITHUB_CREDENTIAL_HELPER_KEY}; +use fabro_types::GitHubRepositorySlug; + +use crate::error::Error; + +/// Section base for the effective repositories' HTTPS routes. +const GITHUB_HTTPS_BASE: &str = "https://github.com/"; + +/// Merge the bridging entries into `env` for the effective repository set +/// (primary first). Appends after any valid user-provided `GIT_CONFIG_COUNT` +/// overlay without overwriting it, and fails with a configuration error when +/// the user overlay is malformed rather than silently replacing it. +pub(crate) fn merge_git_bridge_env( + env: &mut HashMap, + targets: &[&GitHubRepositorySlug], +) -> Result<(), Error> { + let start = user_git_config_count(env)?; + let entries = bridge_entries(targets, GITHUB_HTTPS_BASE); + let total = start + entries.len(); + for (offset, (key, value)) in entries.into_iter().enumerate() { + let index = start + offset; + env.insert(format!("GIT_CONFIG_KEY_{index}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{index}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), total.to_string()); + // Fail instead of hanging when access is missing or invalid; a user who + // explicitly configured prompting keeps their value. + env.entry("GIT_TERMINAL_PROMPT".to_string()) + .or_insert_with(|| "0".to_string()); + Ok(()) +} + +/// The bridge's Git config entries in order: the credential helper, then two +/// SSH-to-HTTPS rewrites per repository. `https_base` is +/// [`GITHUB_HTTPS_BASE`] in production; contract tests substitute a local +/// `file://` root to prove real Git applies the generated entries without +/// touching the network. +fn bridge_entries(targets: &[&GitHubRepositorySlug], https_base: &str) -> Vec<(String, String)> { + let mut entries = Vec::with_capacity(1 + targets.len() * 2); + entries.push(( + GITHUB_CREDENTIAL_HELPER_KEY.to_string(), + GITHUB_CREDENTIAL_HELPER.to_string(), + )); + for slug in targets { + let owner = slug.owner(); + let repo = slug.repo(); + let https = format!("{https_base}{owner}/{repo}"); + // One prefix rule per SSH spelling covers both the bare and `.git` + // suffixed forms. + entries.push(( + format!("url.{https}.insteadOf"), + format!("git@github.com:{owner}/{repo}"), + )); + entries.push(( + format!("url.{https}.insteadOf"), + format!("ssh://git@github.com/{owner}/{repo}"), + )); + } + entries +} + +/// Validate and measure a user-provided `GIT_CONFIG_COUNT` overlay so the +/// bridge appends after it. Orphaned `GIT_CONFIG_KEY_n` entries without a +/// count are inert to Git and are treated as absent. +fn user_git_config_count(env: &HashMap) -> Result { + let Some(raw) = env.get("GIT_CONFIG_COUNT") else { + return Ok(0); + }; + let count: usize = raw.trim().parse().map_err(|_| { + Error::Precondition(format!( + "environment variable GIT_CONFIG_COUNT must be a non-negative integer to combine \ + with Fabro's Git bridging entries, got `{raw}`" + )) + })?; + for index in 0..count { + let key = format!("GIT_CONFIG_KEY_{index}"); + let value = format!("GIT_CONFIG_VALUE_{index}"); + if !env.contains_key(&key) || !env.contains_key(&value) { + return Err(Error::Precondition(format!( + "GIT_CONFIG_COUNT is {count} but {key} or {value} is missing; fix the indexed \ + Git config overlay so Fabro can append its bridging entries after it" + ))); + } + } + Ok(count) +} + +#[cfg(test)] +#[expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "contract tests drive the installed git binary synchronously in non-async tests" +)] +mod tests { + use std::path::Path; + use std::process::Command; + + use super::*; + + fn slug(value: &str) -> GitHubRepositorySlug { + value.parse().expect("test slug should parse") + } + + fn bridged_env( + base_env: HashMap, + targets: &[&GitHubRepositorySlug], + ) -> HashMap { + let mut env = base_env; + merge_git_bridge_env(&mut env, targets).expect("bridge entries should merge"); + env + } + + /// Run `git` with ONLY the bridge-relevant environment: the inherited + /// user/system/global Git config is disabled so assertions observe just + /// the generated entries. + fn git(args: &[&str], env: &HashMap, cwd: &Path) -> std::process::Output { + let mut command = Command::new("git"); + command + .args(args) + .current_dir(cwd) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "true"); + for (key, value) in env { + command.env(key, value); + } + command.output().expect("git should run") + } + + /// Create a bare fixture answering both the bare and `.git`-suffixed + /// routes, the way GitHub serves both HTTPS spellings. + fn init_bare_fixture(root: &Path, owner_repo: &str) -> String { + let fixture = root.join(format!("{owner_repo}.git")); + std::fs::create_dir_all(&fixture).unwrap(); + let init = Command::new("git") + .args(["init", "--bare", "--initial-branch=main"]) + .arg(&fixture) + .output() + .expect("git init should run"); + assert!(init.status.success(), "{init:?}"); + #[cfg(unix)] + std::os::unix::fs::symlink(&fixture, root.join(owner_repo)).unwrap(); + format!("file://{}/", root.display()) + } + + #[test] + fn no_targets_means_no_bridge_call_and_empty_env_stays_empty() { + // The caller only bridges when the additional set is non-empty; the + // pure entry builder is still total for the primary-only case. + assert_eq!(bridge_entries(&[], GITHUB_HTTPS_BASE).len(), 1); + let env: HashMap = HashMap::new(); + assert!(!env.contains_key("GIT_CONFIG_COUNT")); + } + + #[test] + fn merges_helper_rewrites_count_and_terminal_prompt() { + let keystone = slug("fabro-sh/keystone"); + let fabro = slug("fabro-sh/fabro"); + let env = bridged_env(HashMap::new(), &[&fabro, &keystone]); + + assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("5")); + assert_eq!( + env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("credential.https://github.com.helper") + ); + assert_eq!( + env.get("GIT_CONFIG_KEY_1").map(String::as_str), + Some("url.https://github.com/fabro-sh/fabro.insteadOf") + ); + assert_eq!( + env.get("GIT_CONFIG_VALUE_1").map(String::as_str), + Some("git@github.com:fabro-sh/fabro") + ); + assert_eq!( + env.get("GIT_CONFIG_VALUE_2").map(String::as_str), + Some("ssh://git@github.com/fabro-sh/fabro") + ); + assert_eq!( + env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("0") + ); + + // No secrets anywhere in the generated values. + for (key, value) in &env { + assert!(!value.contains("ghs_"), "{key}={value}"); + } + } + + #[test] + fn respects_an_explicit_user_terminal_prompt() { + let keystone = slug("fabro-sh/keystone"); + let env = bridged_env( + HashMap::from([("GIT_TERMINAL_PROMPT".to_string(), "1".to_string())]), + &[&keystone], + ); + assert_eq!( + env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("1") + ); + } + + #[test] + fn appends_after_a_valid_user_git_config_overlay() { + let keystone = slug("fabro-sh/keystone"); + let env = bridged_env( + HashMap::from([ + ("GIT_CONFIG_COUNT".to_string(), "1".to_string()), + ("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()), + ("GIT_CONFIG_VALUE_0".to_string(), "Overlay User".to_string()), + ]), + &[&keystone], + ); + + assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("4")); + assert_eq!( + env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("user.name"), + "user entry must survive at its original index" + ); + assert_eq!( + env.get("GIT_CONFIG_KEY_1").map(String::as_str), + Some("credential.https://github.com.helper") + ); + + // Real Git sees both the user's entry and the appended bridge entry. + let dir = tempfile::tempdir().unwrap(); + let output = git(&["config", "--list"], &env, dir.path()); + assert!(output.status.success(), "{output:?}"); + let listed = String::from_utf8_lossy(&output.stdout); + assert!(listed.contains("user.name=Overlay User"), "{listed}"); + assert!( + listed.contains("credential.https://github.com.helper"), + "{listed}" + ); + } + + #[test] + fn rejects_a_malformed_user_git_config_overlay() { + let keystone = slug("fabro-sh/keystone"); + + let mut non_numeric = HashMap::from([("GIT_CONFIG_COUNT".to_string(), "two".to_string())]); + let err = merge_git_bridge_env(&mut non_numeric, &[&keystone]).unwrap_err(); + assert!(err.to_string().contains("GIT_CONFIG_COUNT"), "{err}"); + + let mut missing_index = HashMap::from([ + ("GIT_CONFIG_COUNT".to_string(), "2".to_string()), + ("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()), + ("GIT_CONFIG_VALUE_0".to_string(), "Overlay".to_string()), + ]); + let err = merge_git_bridge_env(&mut missing_index, &[&keystone]).unwrap_err(); + assert!(err.to_string().contains("GIT_CONFIG_KEY_1"), "{err}"); + } + + /// With the bridge active, `git credential fill` for github.com resolves + /// through the generated helper and reads `$GITHUB_TOKEN` from the + /// invoking process environment at invocation time. + #[test] + fn credential_helper_reads_github_token_at_invocation_time() { + use std::io::Write as _; + + let keystone = slug("fabro-sh/keystone"); + let mut env = bridged_env(HashMap::new(), &[&keystone]); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + let dir = tempfile::tempdir().unwrap(); + let mut command = Command::new("git"); + command + .args(["credential", "fill"]) + .current_dir(dir.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + for (key, value) in &env { + command.env(key, value); + } + let mut child = command.spawn().expect("git credential fill should spawn"); + child + .stdin + .as_mut() + .unwrap() + .write_all(b"protocol=https\nhost=github.com\npath=fabro-sh/keystone\n\n") + .unwrap(); + let output = child.wait_with_output().unwrap(); + + assert!(output.status.success(), "{output:?}"); + let filled = String::from_utf8_lossy(&output.stdout); + assert!(filled.contains("username=x-access-token"), "{filled}"); + assert!(filled.contains("password=test-token-value"), "{filled}"); + } + + /// Real Git applies the generated `insteadOf` rewrites: the exact SSH + /// spellings of a declared repository resolve to their HTTPS-analog + /// route (a local `file://` fixture here, so no network is involved), + /// while `GIT_SSH_COMMAND=false` proves SSH is never attempted. + #[test] + fn declared_ssh_urls_rewrite_to_the_https_route() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + + for url in [ + "ssh://git@github.com/fabro-sh/keystone.git", + "ssh://git@github.com/fabro-sh/keystone", + "git@github.com:fabro-sh/keystone.git", + "git@github.com:fabro-sh/keystone", + ] { + let output = git(&["ls-remote", url], &env, root.path()); + assert!( + output.status.success(), + "{url} should rewrite to the fixture route: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + + /// An undeclared SSH URL that shares no declared prefix is not + /// rewritten: Git still routes it to SSH, where the scripted + /// `GIT_SSH_COMMAND=false` fails immediately without network access. + #[test] + fn undeclared_ssh_urls_are_not_rewritten() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + let output = git( + &["ls-remote", "git@github.com:fabro-sh/undeclared"], + &env, + root.path(), + ); + assert!(!output.status.success(), "{output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + // Not rewritten: the failure never mentions the local HTTPS-analog + // fixture route, so Git still chose the SSH transport. + assert!( + !stderr.contains(&root.path().display().to_string()), + "undeclared URL must not be rewritten to the fixture route: {stderr}" + ); + assert!(!stderr.contains("test-token-value"), "{stderr}"); + } + + /// Prefix collision: with `fabro-sh/keystone` declared, both SSH + /// spellings of `fabro-sh/keystone-other` are rewritten to the HTTPS + /// route (prefix match), where access fails — at GitHub this is an + /// authorization error for the scoped token — and no token leaks into + /// the output. + #[test] + fn prefix_colliding_undeclared_repositories_rewrite_and_fail_without_token_leak() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + for url in [ + "git@github.com:fabro-sh/keystone-other", + "ssh://git@github.com/fabro-sh/keystone-other.git", + ] { + let output = git(&["ls-remote", url], &env, root.path()); + assert!(!output.status.success(), "{url}: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + // The failure names the (missing) HTTPS-analog fixture route, + // proving the prefix rule rewrote the URL away from SSH. + assert!( + stderr.contains("keystone-other"), + "{url} must be rewritten away from SSH, got: {stderr}" + ); + assert!( + stderr.contains(&root.path().display().to_string()), + "{url} must land on the rewritten route, got: {stderr}" + ); + assert!(!stderr.contains("test-token-value"), "{stderr}"); + } + } +} diff --git a/lib/components/fabro-workflow/src/handler/agent.rs b/lib/components/fabro-workflow/src/handler/agent.rs index 99d20431b..b262cb976 100644 --- a/lib/components/fabro-workflow/src/handler/agent.rs +++ b/lib/components/fabro-workflow/src/handler/agent.rs @@ -489,24 +489,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await @@ -629,7 +630,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("plan", 1)).unwrap(); @@ -656,7 +657,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("work", 1)).unwrap(); @@ -1150,7 +1151,7 @@ All checks passed. .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("step", 1)).unwrap(); @@ -1554,7 +1555,7 @@ Some text in between. .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("report", 1)).unwrap(); diff --git a/lib/components/fabro-workflow/src/handler/command.rs b/lib/components/fabro-workflow/src/handler/command.rs index db9682a1c..49c43230c 100644 --- a/lib/components/fabro-workflow/src/handler/command.rs +++ b/lib/components/fabro-workflow/src/handler/command.rs @@ -363,20 +363,21 @@ mod tests { Ok(RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: std::collections::HashMap::default(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: std::collections::HashMap::default(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), )) @@ -464,23 +465,24 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await @@ -1075,7 +1077,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1106,7 +1108,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1133,7 +1135,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1160,7 +1162,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1185,7 +1187,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1210,7 +1212,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1240,7 +1242,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap_err(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1267,7 +1269,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node = snapshot diff --git a/lib/components/fabro-workflow/src/handler/llm/acp.rs b/lib/components/fabro-workflow/src/handler/llm/acp.rs index cfd498beb..9e9f30c39 100644 --- a/lib/components/fabro-workflow/src/handler/llm/acp.rs +++ b/lib/components/fabro-workflow/src/handler/llm/acp.rs @@ -473,8 +473,9 @@ impl AgentAcpBackend { emitter.notice( RunNoticeLevel::Info, RunNoticeCode::GithubTokenRefreshLimited, - "ACP agent stages receive workflow env at process launch; stages running beyond \ - token expiry may need to be retried.", + "ACP agent stages receive workflow env at process launch; GITHUB_TOKEN access to \ + every declared repository expires together, so stages running beyond token \ + expiry may need to be retried.", ); } provider diff --git a/lib/components/fabro-workflow/src/handler/parallel.rs b/lib/components/fabro-workflow/src/handler/parallel.rs index 82773470b..fed0fd03e 100644 --- a/lib/components/fabro-workflow/src/handler/parallel.rs +++ b/lib/components/fabro-workflow/src/handler/parallel.rs @@ -944,24 +944,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await @@ -1404,7 +1405,7 @@ mod tests { .execute(&node, &context, &graph, Path::new("/tmp/test"), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); assert_eq!(outcome.status, StageOutcome::Succeeded); let results: Vec = diff --git a/lib/components/fabro-workflow/src/handler/prompt.rs b/lib/components/fabro-workflow/src/handler/prompt.rs index 630332878..f2d3df535 100644 --- a/lib/components/fabro-workflow/src/handler/prompt.rs +++ b/lib/components/fabro-workflow/src/handler/prompt.rs @@ -267,24 +267,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await @@ -569,7 +570,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("classify", 1)).unwrap(); diff --git a/lib/components/fabro-workflow/src/lib.rs b/lib/components/fabro-workflow/src/lib.rs index 3178542d7..29cd629ec 100644 --- a/lib/components/fabro-workflow/src/lib.rs +++ b/lib/components/fabro-workflow/src/lib.rs @@ -293,6 +293,7 @@ pub mod error; pub mod event; pub mod file_resolver; pub mod git; +pub(crate) mod git_bridge; pub(crate) mod graph; pub mod handler; mod hook_context; diff --git a/lib/components/fabro-workflow/src/lifecycle/git.rs b/lib/components/fabro-workflow/src/lifecycle/git.rs index b8e06088e..691690b21 100644 --- a/lib/components/fabro-workflow/src/lifecycle/git.rs +++ b/lib/components/fabro-workflow/src/lifecycle/git.rs @@ -782,6 +782,7 @@ mod tests { labels: BTreeMap::new(), source_directory: Some("/tmp/project".to_string()), workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/components/fabro-workflow/src/operations/archive.rs b/lib/components/fabro-workflow/src/operations/archive.rs index 92d9124cf..28e28ae77 100644 --- a/lib/components/fabro-workflow/src/operations/archive.rs +++ b/lib/components/fabro-workflow/src/operations/archive.rs @@ -215,24 +215,25 @@ mod tests { async fn seed_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { event::append_event(run_store, run_id, &Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: *run_id, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 7783a9b59..54f095bd2 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -17,7 +17,7 @@ use fabro_store::{Database, RunDatabase}; use fabro_template::TemplateContext; use fabro_types::{ AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance, - WorkflowSettings, + WorkflowSettings, WorkflowVersionId, }; use fabro_util::json::normalize_json_value; use tokio::task::spawn_blocking; @@ -100,6 +100,7 @@ impl CreateRunInput { run_id, storage_root, workflow_slug, + workflow_version_id: None, submitted_manifest_bytes, title, automation, @@ -133,6 +134,7 @@ pub struct CreateRunPersistenceMetadata { pub run_id: RunId, pub storage_root: PathBuf, pub workflow_slug: Option, + pub workflow_version_id: Option, pub submitted_manifest_bytes: Option>, pub title: Option, pub automation: Option, @@ -202,6 +204,7 @@ pub struct CreateRunPersistenceInput { run_id: RunId, run_dir: PathBuf, workflow_slug: Option, + workflow_version_id: Option, submitted_manifest_bytes: Option>, title: Option, automation: Option, @@ -229,6 +232,10 @@ impl CreateRunPersistenceInput { self.workflow_slug.as_deref() } + pub fn workflow_version_id(&self) -> Option { + self.workflow_version_id + } + pub fn submitted_manifest_bytes(&self) -> Option<&[u8]> { self.submitted_manifest_bytes.as_deref() } @@ -397,6 +404,7 @@ pub fn assemble_create_run_persistence_input( run_id, storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -417,6 +425,7 @@ pub fn assemble_create_run_persistence_input( run_id, run_dir, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -438,6 +447,7 @@ pub async fn persist_create_run( run_id, run_dir, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -465,6 +475,7 @@ pub async fn persist_create_run( graph: validated.graph().clone(), graph_source: Some(validated.source().to_string()), workflow_slug, + workflow_version_id, automation, source_directory: Some(source_directory), labels, @@ -552,6 +563,7 @@ async fn persist_created_run( .collect::>(), source_directory: record.source_directory.clone(), workflow_slug: record.workflow_slug.clone(), + workflow_version_id: record.workflow_version_id, automation: record.automation.clone(), provenance: record.provenance.clone(), manifest_blob, @@ -1709,6 +1721,7 @@ reasoning = false run_id: fixtures::RUN_1, storage_root: PathBuf::from("/tmp/storage"), workflow_slug: None, + workflow_version_id: None, submitted_manifest_bytes: None, title: None, automation: None, @@ -1761,7 +1774,9 @@ reasoning = false std::fs::write(&dot_path, "this is no longer a graph").unwrap(); let materialized = materialize_create_run(compiled, catalog.as_ref()).unwrap(); - let metadata = persistence_metadata(&request, fixtures::RUN_2, &storage_root); + let workflow_version_id = test_support::test_workflow_version_id(); + let mut metadata = persistence_metadata(&request, fixtures::RUN_2, &storage_root); + metadata.workflow_version_id = Some(workflow_version_id); let input = assemble_create_run_persistence_input(materialized, metadata); let store = memory_store(); let created = persist_create_run(store.as_ref(), input).await.unwrap(); @@ -1775,6 +1790,7 @@ reasoning = false let state = run_store.state().await.unwrap(); assert_eq!(state.spec.graph.goal(), "Compiled goal"); assert_eq!(state.spec.automation, Some(automation)); + assert_eq!(state.spec.workflow_version_id, Some(workflow_version_id)); let events = run_store.list_events().await.unwrap(); assert_eq!( events @@ -1790,6 +1806,7 @@ reasoning = false created.workflow_source.as_deref(), Some(compiled_source.as_str()) ); + assert_eq!(created.workflow_version_id, Some(workflow_version_id)); let manifest_blob = created .manifest_blob .as_ref() diff --git a/lib/components/fabro-workflow/src/operations/fork.rs b/lib/components/fabro-workflow/src/operations/fork.rs index 3f728211d..1439ab471 100644 --- a/lib/components/fabro-workflow/src/operations/fork.rs +++ b/lib/components/fabro-workflow/src/operations/fork.rs @@ -149,27 +149,28 @@ async fn persist_forked_run( .map_err(|err| Error::engine(err.to_string()))?; event::append_event(&run_store, &spec.run_id, &Event::RunCreated { - run_id: spec.run_id, - title: None, - settings: serde_json::to_value(&spec.settings) + run_id: spec.run_id, + title: None, + settings: serde_json::to_value(&spec.settings) .map_err(|err| Error::engine(err.to_string()))?, - graph: serde_json::to_value(&spec.graph) + graph: serde_json::to_value(&spec.graph) .map_err(|err| Error::engine(err.to_string()))?, - workflow_source: projection.spec.graph_source.clone(), - labels: spec.labels.clone().into_iter().collect(), - source_directory: spec.source_directory.clone(), - workflow_slug: spec.workflow_slug.clone(), - automation: spec.automation.clone(), - provenance: spec.provenance.clone(), - manifest_blob: spec.manifest_blob, + workflow_source: projection.spec.graph_source.clone(), + labels: spec.labels.clone().into_iter().collect(), + source_directory: spec.source_directory.clone(), + workflow_slug: spec.workflow_slug.clone(), + workflow_version_id: spec.workflow_version_id, + automation: spec.automation.clone(), + provenance: spec.provenance.clone(), + manifest_blob: spec.manifest_blob, // Content-addressed, so the forked run reads the source run's // unredacted spec bytes through the same id. - spec_blob: spec.spec_blob, - git: spec.git.clone(), - fork_source_ref: spec.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + spec_blob: spec.spec_blob, + git: spec.git.clone(), + fork_source_ref: spec.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .map_err(|err| Error::engine(err.to_string()))?; @@ -371,30 +372,32 @@ mod tests { let source = store.create_run(&source_run_id).await.unwrap(); let graph = Graph::new("fork-source"); let settings = WorkflowSettings::default(); + let workflow_version_id = test_support::test_workflow_version_id(); event::append_event(&source, &source_run_id, &Event::RunCreated { - run_id: source_run_id, - title: None, - settings: serde_json::to_value(&settings).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: Some("digraph fork_source {}".to_string()), - labels: BTreeMap::new(), - source_directory: Some("/client/source".to_string()), - workflow_slug: Some("fork-source".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: Some(fabro_types::GitContext { + run_id: source_run_id, + title: None, + settings: serde_json::to_value(&settings).unwrap(), + graph: serde_json::to_value(&graph).unwrap(), + workflow_source: Some("digraph fork_source {}".to_string()), + labels: BTreeMap::new(), + source_directory: Some("/client/source".to_string()), + workflow_slug: Some("fork-source".to_string()), + workflow_version_id: Some(workflow_version_id), + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: Some(fabro_types::GitContext { origin_url: "https://github.com/example/repo.git".to_string(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -463,6 +466,10 @@ mod tests { assert_eq!(node.response.as_deref(), Some("historical response")); assert_eq!(forked_state.checkpoints.len(), 1); + assert_eq!( + forked_state.spec.workflow_version_id, + Some(workflow_version_id) + ); assert_eq!( forked_state.spec.fork_source_ref.unwrap().source_run_id, source_run_id diff --git a/lib/components/fabro-workflow/src/operations/resume.rs b/lib/components/fabro-workflow/src/operations/resume.rs index 433947716..466ae5e0c 100644 --- a/lib/components/fabro-workflow/src/operations/resume.rs +++ b/lib/components/fabro-workflow/src/operations/resume.rs @@ -43,8 +43,7 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result>(), source_directory, workflow_slug, + workflow_version_id, automation, provenance: input.provenance.clone(), manifest_blob, @@ -123,7 +125,7 @@ mod tests { use fabro_types::{ AuthMethod, BlobHash, DirtyStatus, FailureReason, ForkSourceRef, GitContext, Graph, IdpIdentity, Principal, PullRequestLink, RunRunnableSource, RunServerProvenance, RunTiming, - WorkflowSettings, fixtures, + WorkflowSettings, fixtures, test_support, }; use object_store::memory::InMemory; @@ -186,6 +188,7 @@ mod tests { labels: labels.into_iter().collect(), source_directory: Some("/workspace/source".to_string()), workflow_slug: Some("retry-source".to_string()), + workflow_version_id: Some(test_support::test_workflow_version_id()), automation: None, provenance: provenance("source-user"), manifest_blob, @@ -393,6 +396,10 @@ mod tests { Some(&"test".to_string()) ); assert_eq!(retry_state.spec.graph.name, "retry_source"); + assert_eq!( + retry_state.spec.workflow_version_id, + Some(test_support::test_workflow_version_id()) + ); assert_eq!( retry_state.spec.graph_source.as_deref(), Some("digraph retry_source { start -> exit }") diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index c295ae145..897ee6039 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -1,4 +1,5 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; +use std::future::Future; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -17,10 +18,11 @@ use fabro_sandbox::{DockerSandboxOptions, SandboxSpec}; use fabro_static::EnvVars; use fabro_types::settings::run::{ ApprovalMode, McpServerSettings as ResolvedMcpServerSettings, PullRequestSettings, - ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, + ResolvedGithubIntegration, ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, RunPrepareSettings as ResolvedRunPrepareSettings, }; use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind}; +use fabro_util::error::collect_chain; use fabro_vault::Vault; use tokio::runtime::Handle; use tokio::sync::RwLock as AsyncRwLock; @@ -31,7 +33,8 @@ use crate::artifact_upload::ArtifactSink; use crate::context::Context; use crate::error::{self, Error}; use crate::event::{ - Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeLevel, append_event_to_sink, + Emitter, Event, EventBody, RunEventLogger, RunEventPersistenceError, RunEventSink, + RunNoticeLevel, append_event_to_sink, }; use crate::handler::HandlerRegistry; use crate::model_fallback::{ModelFallbackNotice, ResolvedModelFallbacks, resolve_model_fallbacks}; @@ -104,9 +107,10 @@ pub struct StartServices { pub artifact_sink: Option, pub run_control: Option>, pub github_app: Option, - /// Server-resolved GitHub integration permissions to inject into the - /// sandbox env. Empty when github integration has no permissions. - pub github_permissions: HashMap, + /// The resolved GitHub integration request (interpolated permissions + /// plus declared additional repositories) to inject into the sandbox + /// env. Empty when the github integration requests no token. + pub github_integration: ResolvedGithubIntegration, pub vault: Arc>, pub catalog: Arc, pub on_node: crate::OnNodeCallback, @@ -157,8 +161,7 @@ pub async fn start(run_dir: &Path, services: StartServices) -> Result Result Error { + cancel_token.cancel(); + error.into() +} + +/// Race a pipeline step against the first latched run-event persistence +/// failure. When the failure wins, the step future is dropped mid-flight and +/// the run token is cancelled. +async fn race_persistence( + logger: &RunEventLogger, + cancel_token: &CancellationToken, + step: impl Future, +) -> Result { + tokio::select! { + result = step => Ok(result), + failure = logger.wait_for_failure() => { + Err(stop_for_run_event_persistence_failure(cancel_token, failure)) + } + } +} + +async fn flush_or_stop( + logger: &RunEventLogger, + cancel_token: &CancellationToken, +) -> Result<(), Error> { + logger + .flush() + .await + .map_err(|failure| stop_for_run_event_persistence_failure(cancel_token, failure)) +} + impl RunSession { async fn new(persisted: &Persisted, services: StartServices) -> Result { let record = persisted.run_spec(); @@ -430,6 +472,7 @@ impl RunSession { run_id: Some(record.run_id), clone_origin_url: record.repo_origin_url().map(str::to_string), clone_branch: record.base_branch().map(str::to_string), + clone_commit_sha: None, }, SandboxProviderKind::Daytona => { let api_key = vault_guard @@ -441,6 +484,7 @@ impl RunSession { run_id: Some(record.run_id), clone_origin_url: record.repo_origin_url().map(str::to_string), clone_branch: record.base_branch().map(str::to_string), + clone_commit_sha: None, api_key, } } @@ -450,11 +494,13 @@ impl RunSession { .environment .resolve_env(secret_lookup) .map_err(|err| Error::engine_with_source("failed to resolve run environment", err))?; - let github_permissions: Option> = - (!services.github_permissions.is_empty()).then(|| services.github_permissions.clone()); + let github_integration = services + .github_integration + .is_token_requested() + .then(|| services.github_integration.clone()); let sandbox_env = SandboxEnvSpec { toml_env, - github_permissions, + github_integration, origin_url: record.repo_origin_url().map(str::to_string), }; @@ -589,7 +635,7 @@ fn resolve_sandbox_provider(settings: &ResolvedRunSettings) -> SandboxProviderKi } fn resolve_daytona_config(settings: &ResolvedRunSettings) -> DaytonaConfig { - daytona_config_from_environment(&settings.environment, !settings.clone.enabled) + daytona_config_from_environment(&settings.environment, &settings.clone) } fn resolve_docker_config( @@ -598,7 +644,7 @@ fn resolve_docker_config( ) -> Result { docker_config_from_environment_with_secrets( &settings.environment, - !settings.clone.enabled, + &settings.clone, secrets_lookup, ) .map_err(|err| Error::engine_with_source("failed to resolve Docker environment config", err)) @@ -689,6 +735,7 @@ impl RunSession { resume: Option, ) -> Result { let on_node = self.on_node.clone(); + let run_cancel_token = self.cancel_token.clone(); let record = persisted.run_spec(); let run_options = RunOptions { @@ -777,7 +824,19 @@ impl RunSession { seed_context: self.seed_context, fabro_run_tools: self.fabro_run_tools, }; - let mut initialized = Box::pin(pipeline::initialize(persisted, init_options)).await?; + let mut initialized = match race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(pipeline::initialize(persisted, init_options)), + ) + .await? + { + Ok(initialized) => initialized, + Err(err) => { + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; + return Err(err); + } + }; initialized.on_node = on_node; let sandbox_for_cleanup = Arc::clone(&initialized.engine.run.sandbox); @@ -801,8 +860,15 @@ impl RunSession { steering_hub_for_drain.drain_pending_at_run_end(); }); - let executed = pipeline::execute(initialized).await; - store_progress_logger.flush().await; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; + + let executed = race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(pipeline::execute(initialized)), + ) + .await?; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; let final_context = Some(executed.final_context.clone()); let finalize_opts = FinalizeOptions { @@ -822,16 +888,21 @@ impl RunSession { model: self.pr_model, }; - let concluding = async { - let concluded = Box::pin(pipeline::conclude(executed, &finalize_opts)).await?; - let published = Box::pin(pipeline::publish(concluded, &publish_opts)).await; - Box::pin(pipeline::finalize(published, &finalize_opts)).await - }; - let finalized = match concluding.await { + let concluding = race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(async { + let concluded = Box::pin(pipeline::conclude(executed, &finalize_opts)).await?; + let published = Box::pin(pipeline::publish(concluded, &publish_opts)).await; + Box::pin(pipeline::finalize(published, &finalize_opts)).await + }), + ) + .await?; + let finalized = match concluding { Ok(finalized) => finalized, Err(err) => { self.steering_hub.drain_pending_at_run_end(); - store_progress_logger.flush().await; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; return Err(err); } }; @@ -840,7 +911,7 @@ impl RunSession { // scopeguard above re-runs as a no-op (drain is idempotent on an // already-empty buffer) on the way out of scope. self.steering_hub.drain_pending_at_run_end(); - store_progress_logger.flush().await; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; scopeguard::ScopeGuard::into_inner(cleanup_guard); @@ -1000,13 +1071,20 @@ impl Drop for DetachedRunCompletionGuard { 0, ) .await; - let _ = append_event_to_sink(&event_sink, &run_id, &Event::RunNotice { + if let Err(err) = append_event_to_sink(&event_sink, &run_id, &Event::RunNotice { level: RunNoticeLevel::Error, code: code.to_string(), message: message.to_string(), exec_output_tail: None, }) - .await; + .await + { + let rendered_error = collect_chain(&err).join(": "); + tracing::warn!( + error = %rendered_error, + "Failed to append detached completion notice", + ); + } }); } } @@ -1030,7 +1108,11 @@ async fn persist_detached_failure( exec_output_tail: None, }; if let Err(err) = append_event_to_sink(event_sink, &run_id, &event).await { - tracing::warn!(error = %err, "Failed to append detached failure notice"); + let rendered_error = collect_chain(&err).join(": "); + tracing::warn!( + error = %rendered_error, + "Failed to append detached failure notice", + ); } Ok(()) @@ -1089,8 +1171,19 @@ mod tests { work -> exit }"#; + const BLOCKING_DOT: &str = r#"digraph Test { + graph [goal="Wait forever"] + start [shape=Mdiamond] + block [type="blocking"] + exit [shape=Msquare] + start -> block + block -> exit + }"#; + struct TimedOutcomeHandler; + struct BlockingHandler; + fn timed_success_outcome() -> Outcome { let mut outcome = Outcome::success(); outcome.timing = Some(StageTiming::new(0, 100, 50)); @@ -1122,6 +1215,20 @@ mod tests { } } + #[async_trait::async_trait] + impl Handler for BlockingHandler { + async fn execute( + &self, + _node: &fabro_graphviz::graph::Node, + _context: &Context, + _graph: &fabro_graphviz::graph::Graph, + _run_dir: &Path, + _services: &EngineServices, + ) -> Result { + std::future::pending().await + } + } + fn memory_store() -> Arc { Arc::new(Database::new( Arc::new(InMemory::new()), @@ -1283,6 +1390,7 @@ reasoning = false let settings = settings_from_run_layer(RunLayer { clone: Some(RunCloneLayer { enabled: Some(false), + depth: Some(1), }), ..RunLayer::default() }); @@ -1293,6 +1401,45 @@ reasoning = false .skip_clone ); assert!(resolve_daytona_config(&settings.run).skip_clone); + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, Some(1)); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + Some(1) + ); + } + + #[test] + fn zero_clone_depth_requests_full_history_from_clone_providers() { + let settings = settings_from_run_layer(RunLayer { + clone: Some(RunCloneLayer { + enabled: None, + depth: Some(0), + }), + ..RunLayer::default() + }); + + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, None); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + None + ); + } + + #[test] + fn clone_providers_default_to_depth_100() { + let settings = settings_from_run_layer(RunLayer::default()); + + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, Some(100)); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + Some(100) + ); } #[test] @@ -1723,7 +1870,7 @@ reasoning = false artifact_sink: None, run_control: None, github_app: None, - github_permissions: HashMap::new(), + github_integration: ResolvedGithubIntegration::default(), vault: Arc::new(AsyncRwLock::new(start_vault(&[]))), catalog: test_catalog(), on_node: None, @@ -2085,6 +2232,74 @@ reasoning = false assert!(run_store.state().await.unwrap().conclusion.is_some()); } + #[tokio::test] + async fn event_persistence_failure_stops_execution_and_fails_run() { + let temp = tempfile::tempdir().unwrap(); + let (storage_root, run_dir) = storage_root_and_run_dir(&temp); + let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); + let mut registry = test_registry(); + registry.register("blocking", Box::new(BlockingHandler)); + let (_persisted, store) = persisted_workflow(BLOCKING_DOT, &storage_root).await; + let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); + let canonical_sink = RunEventSink::store(run_store.clone()); + let mut services = test_start_services(&store, &run_dir, emitter, Arc::new(registry)).await; + let cancel_token = services.cancel_token.clone(); + services.event_sink = RunEventSink::callback(move |event| { + let canonical_sink = canonical_sink.clone(); + async move { + if matches!(&event.body, EventBody::StageStarted(_)) + && event.node_id.as_deref() == Some("block") + { + return Err(anyhow::anyhow!( + "request failed with status 413 Payload Too Large" + ) + .context("worker lost canonical run store during append run event")); + } + canonical_sink.write_run_event(&event).await + } + }); + + let result = tokio::time::timeout(Duration::from_secs(2), start(&run_dir, services)) + .await + .expect("event persistence failure should stop the blocking stage"); + let Err(error) = result else { + panic!("event persistence failure should fail the run"); + }; + + assert!(cancel_token.is_cancelled()); + let rendered = error.display_with_causes(); + assert!( + rendered.contains("run event persistence failed"), + "{rendered}" + ); + assert!(rendered.contains("stage.started"), "{rendered}"); + assert!(rendered.contains("413 Payload Too Large"), "{rendered}"); + + let projection = run_store.state().await.unwrap(); + assert!(matches!(projection.status, RunStatus::Failed { .. })); + let events = run_store.list_events().await.unwrap(); + let run_failed = events + .iter() + .find_map(|event| match &event.event.body { + EventBody::RunFailed(properties) => Some(properties), + _ => None, + }) + .expect("persistence failure should emit run.failed"); + assert!( + run_failed + .failure + .detail + .causes + .iter() + .any(|cause| cause.contains("413 Payload Too Large")) + ); + assert!( + events + .iter() + .all(|event| !matches!(&event.event.body, EventBody::RunCompleted(_))) + ); + } + #[tokio::test] async fn start_can_run_bundle_backed_child_workflow_without_workflow_bundle_json() { let temp = tempfile::tempdir().unwrap(); diff --git a/lib/components/fabro-workflow/src/operations/timeline.rs b/lib/components/fabro-workflow/src/operations/timeline.rs index 83319745e..b3f51350e 100644 --- a/lib/components/fabro-workflow/src/operations/timeline.rs +++ b/lib/components/fabro-workflow/src/operations/timeline.rs @@ -241,20 +241,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, Utc::now(), ) diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index 7bc1d6dff..6a2f63078 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -156,6 +156,7 @@ fn persisted_workflow(graph: Graph, source: String, run_dir: &Path, run_id: RunI graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some( std::env::current_dir() @@ -208,23 +209,24 @@ async fn seed_created_and_starting( graph: &Graph, ) { append_event(run_store, &run_options.run_id, &Event::RunCreated { - run_id: run_options.run_id, - title: None, - settings: serde_json::to_value(&run_options.settings).unwrap(), - graph: serde_json::to_value(graph).unwrap(), - workflow_source: None, - labels: run_options.labels.clone().into_iter().collect(), - source_directory: Some(std::env::current_dir().unwrap().display().to_string()), - workflow_slug: run_options.workflow_slug.clone(), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: run_options.pre_run_git.clone(), - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + run_id: run_options.run_id, + title: None, + settings: serde_json::to_value(&run_options.settings).unwrap(), + graph: serde_json::to_value(graph).unwrap(), + workflow_source: None, + labels: run_options.labels.clone().into_iter().collect(), + source_directory: Some(std::env::current_dir().unwrap().display().to_string()), + workflow_slug: run_options.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: run_options.pre_run_git.clone(), + fork_source_ref: run_options.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -285,7 +287,7 @@ async fn execute_test_run_with_options( hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -302,7 +304,7 @@ async fn execute_test_run_with_options( .unwrap(); let executed = execute(initialized).await; - store_logger.flush().await; + store_logger.flush().await.unwrap(); executed } @@ -347,7 +349,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -486,7 +488,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() { hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -597,7 +599,7 @@ async fn run_with_lifecycle( hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), diff --git a/lib/components/fabro-workflow/src/pipeline/finalize.rs b/lib/components/fabro-workflow/src/pipeline/finalize.rs index 9bdb18c39..76dde359f 100644 --- a/lib/components/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/components/fabro-workflow/src/pipeline/finalize.rs @@ -787,23 +787,24 @@ mod tests { async fn seeded_run_store() -> RunDatabase { let run_store = test_store().create_run(&test_run_id()).await.unwrap(); append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("metadata")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: Some("/tmp/project".to_string()), - workflow_slug: Some("metadata".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: test_run_id(), + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(fabro_types::Graph::new("metadata")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: Some("/tmp/project".to_string()), + workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -906,20 +907,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: test_run_id(), - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: test_run_id(), + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), ) @@ -1112,8 +1114,8 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); - let inner_store = test_store().create_run(&test_run_id()).await.unwrap(); - let run_store = inner_store; + let run_store = seeded_run_store().await; + crate::test_support::mark_run_running(&run_store, &test_run_id()).await; let emitter = Arc::new(Emitter::new(test_run_id())); let store_logger = StoreProgressLogger::new(run_store.clone()); store_logger.register(&emitter); @@ -1156,7 +1158,7 @@ mod tests { }) .await .unwrap(); - store_logger.flush().await; + store_logger.flush().await.unwrap(); assert_eq!(concluded.conclusion.status, StageOutcome::Succeeded); } diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index c692b9627..d1e864e88 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -24,6 +24,7 @@ use super::types::{InitOptions, Initialized, LlmSpec, Persisted, SandboxEnvSpec} use crate::error::Error; use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::git::GitAuthor; +use crate::git_bridge; use crate::handler::llm::{AgentAcpBackend, AgentApiBackend, BackendRouter, routing}; use crate::handler::{HandlerRegistry, default_registry}; #[cfg(test)] @@ -37,10 +38,14 @@ use crate::services::{ use crate::stage_execution::{StageExecutionSeed, StageExecutionTracker}; use crate::steering_hub::SteeringHub; -type BuiltSandboxEnv = ( - HashMap, - Option>, -); +struct BuiltSandboxEnv { + env: HashMap, + github_token: Option>, + /// The validated effective repository set behind `github_token`. + /// Present only in App mode or when additional repositories are + /// declared; drives the eager access validation at initialization. + github_access: Option, +} async fn run_hooks( hook_runner: Option<&HookRunner>, @@ -91,41 +96,113 @@ fn build_sandbox_env( spec: &SandboxEnvSpec, github_app: Option<&fabro_github::GitHubCredentials>, ) -> Result { - let env = spec.toml_env.clone(); + let mut env = spec.toml_env.clone(); - let Some(permissions) = spec.github_permissions.as_ref().filter(|p| !p.is_empty()) else { - return Ok((env, None)); + let no_token = |env| BuiltSandboxEnv { + env, + github_token: None, + github_access: None, }; + let Some(integration) = spec + .github_integration + .as_ref() + .filter(|integration| integration.is_token_requested()) + else { + return Ok(no_token(env)); + }; + let declares_additional = integration.has_additional_repositories(); let Some(creds) = github_app else { - return Ok((env, None)); + if declares_additional { + // Legacy permissions-only configuration stays best-effort, but a + // declared additional set is an explicit access requirement. + return Err(Error::Precondition( + "run.integrations.github.additional_repositories requires GitHub credentials, \ + but none are configured" + .to_string(), + )); + } + return Ok(no_token(env)); }; - let source = match creds { - fabro_github::GitHubCredentials::Pat(token) => { - Some(InstallationTokenSource::pat(token.clone())) - } - fabro_github::GitHubCredentials::Installation(token) => { - Some(InstallationTokenSource::installation(token.clone())) - } - fabro_github::GitHubCredentials::App(_) => { - let Some(origin_url) = spec.origin_url.as_deref() else { - return Ok((env, None)); - }; - let https_url = fabro_github::ssh_url_to_https(origin_url); - let (owner, repo) = fabro_github::parse_github_owner_repo(&https_url) - .map_err(|err| Error::engine_with_anyhow("Failed to parse GitHub origin", err))?; - let permissions = serde_json::to_value(permissions).map_err(|err| { - Error::engine_with_source("Failed to serialize GitHub permissions", err) - })?; - Some( - InstallationTokenSource::for_repository(creds, owner, repo, permissions).map_err( - |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), - )?, + // Validate the effective repository set whenever it matters: App mode + // scopes the mint to it, and any declared additional set must hold its + // invariants regardless of credential kind. Legacy PAT/static + // permissions-only runs skip it to preserve their origin-agnostic + // behavior. + let github_access = + if declares_additional || matches!(creds, fabro_github::GitHubCredentials::App(_)) { + fabro_github::GitHubRepositoryAccess::new( + spec.origin_url.as_deref(), + &integration.additional_repositories, + integration.permissions.clone(), ) - } + .map_err(|err| { + Error::engine_with_anyhow("Failed to validate GitHub repository access", err) + })? + } else { + None + }; + + let github_token = match github_access.as_ref() { + Some(access) => Some(InstallationTokenSource::for_access(creds, access).map_err( + |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), + )?), + None => match creds { + fabro_github::GitHubCredentials::Pat(token) => { + Some(InstallationTokenSource::pat(token.clone())) + } + fabro_github::GitHubCredentials::Installation(token) => { + Some(InstallationTokenSource::installation(token.clone())) + } + // No origin URL and nothing declared: keep the legacy App-mode + // best-effort skip. + fabro_github::GitHubCredentials::App(_) => None, + }, }; - Ok((env, source)) + if declares_additional { + let access = github_access + .as_ref() + .expect("access is always constructed when additional repositories are declared"); + git_bridge::merge_git_bridge_env(&mut env, &access.targets())?; + } + + Ok(BuiltSandboxEnv { + env, + github_token, + github_access, + }) +} + +/// When additional repositories are declared, resolve their token before the +/// first workflow stage. App-backed sources first check that every target is +/// on one installation. Static credentials resolve locally; the first Git +/// operation remains their access check. Legacy permissions-only runs skip +/// eager resolution. +async fn resolve_declared_repository_token(built: &BuiltSandboxEnv) -> Result<(), Error> { + let Some(_) = built + .github_access + .as_ref() + .filter(|access| access.has_additional_repositories()) + else { + return Ok(()); + }; + // `build_sandbox_env` guarantees a token source whenever additional + // repositories are declared; fail closed if that ever breaks. + let Some(source) = built.github_token.as_ref() else { + return Err(Error::Precondition( + "run.integrations.github.additional_repositories requires GitHub credentials, but \ + none are configured" + .to_string(), + )); + }; + source.resolve().await.map_err(|err| { + Error::engine_with_anyhow( + "Failed to resolve the GitHub token for the declared repository set", + err, + ) + })?; + Ok(()) } async fn build_registry( @@ -440,10 +517,16 @@ pub async fn initialize( }); } - let (base_env, github_token) = build_sandbox_env( + let built_env = build_sandbox_env( &options.sandbox_env, options.run_options.github_app.as_ref(), )?; + resolve_declared_repository_token(&built_env).await?; + let BuiltSandboxEnv { + env: base_env, + github_token, + github_access: _, + } = built_env; let tool_env_provider = Arc::new(WorkflowToolEnvProvider { base_env: base_env.clone(), github_token: github_token.clone(), @@ -665,7 +748,7 @@ mod tests { use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_interview::AutoApproveInterviewer; use fabro_sandbox::SandboxSpec; - use fabro_store::Database; + use fabro_store::{Database, RunDatabase}; use fabro_types::settings::run::RunModelControls; use fabro_types::{ EventBody, ForkSourceRef, RunEvent, RunId, WorkflowSettings, fixtures, test_support, @@ -710,6 +793,37 @@ mod tests { )) } + async fn seed_run_created( + run_store: &RunDatabase, + settings: serde_json::Value, + graph: serde_json::Value, + source_directory: Option, + fork_source_ref: Option, + ) { + crate::event::append_event(run_store, &test_run_id(), &Event::RunCreated { + run_id: test_run_id(), + title: None, + settings, + graph, + workflow_source: None, + labels: BTreeMap::new(), + source_directory, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref, + retried_from: None, + parent_id: None, + web_url: None, + }) + .await + .unwrap(); + } + fn simple_graph() -> (Graph, String) { let source = r"digraph test { start [shape=Mdiamond]; @@ -815,7 +929,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -851,6 +965,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some(std::env::current_dir().unwrap().display().to_string()), git: Some(fabro_types::GitContext { @@ -944,7 +1059,7 @@ mod tests { let initialized = initialize(persisted, InitOptions { sandbox_env: SandboxEnvSpec { toml_env: HashMap::from([("TEST_KEY".to_string(), "value".to_string())]), - github_permissions: None, + github_integration: None, origin_url: None, }, ..test_init_options( @@ -1035,27 +1150,14 @@ mod tests { let mut run_options = test_settings(&run_dir); run_options.settings = settings; run_options.fork_source_ref = fork_source_ref; - crate::event::append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(&run_options.settings).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: Some(workspace.display().to_string()), - workflow_slug: Some("test".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, - }) - .await - .unwrap(); + seed_run_created( + &run_store, + serde_json::to_value(&run_options.settings).unwrap(), + serde_json::to_value(&graph).unwrap(), + Some(workspace.display().to_string()), + run_options.fork_source_ref.clone(), + ) + .await; initialize(persisted, InitOptions { resume: Some(ResumeState::for_test( @@ -1266,7 +1368,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault, @@ -1321,10 +1423,18 @@ mod tests { let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); - let persisted = test_persisted(graph, source, &run_dir); + let persisted = test_persisted(graph.clone(), source, &run_dir); let emitter = Arc::new(crate::event::Emitter::new(test_run_id())); let store = memory_store(); let run_store = store.create_run(&test_run_id()).await.unwrap(); + seed_run_created( + &run_store, + serde_json::to_value(WorkflowSettings::default()).unwrap(), + serde_json::to_value(graph).unwrap(), + None, + None, + ) + .await; let store_logger = StoreProgressLogger::new(run_store.clone()); let seen = Arc::new(std::sync::Mutex::new(Vec::new())); emitter.on_event({ @@ -1361,7 +1471,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -1375,7 +1485,7 @@ mod tests { }) .await .unwrap(); - store_logger.flush().await; + store_logger.flush().await.unwrap(); assert_eq!(initialized.run_options.run_dir, run_dir); assert!( @@ -1503,7 +1613,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -1519,4 +1629,169 @@ mod tests { assert!(matches!(result, Err(Error::Cancelled))); } + + mod github_integration_env { + //! Focused tests for `build_sandbox_env` / + //! `resolve_declared_repository_token` around declared additional + //! repositories. Installation-resolution failure naming is covered + //! by `fabro_github::access` tests; these prove the initialization + //! wiring: hard errors for declared sets, best-effort behavior for + //! legacy permissions-only configuration. + + use fabro_github::test_support::{InstallationTokenMinter, installation_token_source}; + use fabro_github::{GitHubAppCredentials, GitHubCredentials, InstallationToken}; + use fabro_types::settings::run::ResolvedGithubIntegration; + + use super::*; + + fn integration(additional: &[&str]) -> ResolvedGithubIntegration { + ResolvedGithubIntegration { + permissions: HashMap::from([( + "contents".to_string(), + "read".to_string(), + )]), + additional_repositories: additional + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect(), + } + } + + fn spec( + origin: Option<&str>, + github_integration: Option, + ) -> SandboxEnvSpec { + SandboxEnvSpec { + toml_env: HashMap::new(), + github_integration, + origin_url: origin.map(str::to_string), + } + } + + #[test] + fn declared_additional_repositories_require_credentials() { + let spec = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&["fabro-sh/keystone"])), + ); + let Err(err) = build_sandbox_env(&spec, None) else { + panic!("declared additional repositories without credentials must fail"); + }; + assert!( + err.to_string().contains("requires GitHub credentials"), + "{err}" + ); + } + + #[test] + fn declared_additional_repositories_require_an_origin() { + let spec = spec(None, Some(integration(&["fabro-sh/keystone"]))); + let creds = GitHubCredentials::Pat("ghp_x".to_string()); + let Err(err) = build_sandbox_env(&spec, Some(&creds)) else { + panic!("declared additional repositories without an origin must fail"); + }; + assert!( + err.to_string().contains("GitHub repository access"), + "{err}" + ); + } + + #[test] + fn declared_repositories_inject_bridge_entries_and_keep_the_pat_source() { + let spec = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&["fabro-sh/keystone"])), + ); + let creds = GitHubCredentials::Pat("ghp_x".to_string()); + let built = build_sandbox_env(&spec, Some(&creds)).unwrap(); + + assert!(built.github_token.is_some()); + let access = built.github_access.expect("access should be constructed"); + assert!(access.has_additional_repositories()); + // Helper entry plus two SSH rewrites for each of the two + // effective repositories (origin + declared additional). + assert_eq!( + built.env.get("GIT_CONFIG_COUNT").map(String::as_str), + Some("5") + ); + assert_eq!( + built.env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("credential.https://github.com.helper") + ); + assert_eq!( + built.env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("0") + ); + } + + #[test] + fn legacy_permissions_only_configuration_stays_best_effort() { + // No credentials: no error, no token source, no bridge entries. + let no_creds = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&[])), + ); + let built = build_sandbox_env(&no_creds, None).unwrap(); + assert!(built.github_token.is_none()); + assert!(!built.env.contains_key("GIT_CONFIG_COUNT")); + + // App credentials without an origin: legacy best-effort skip. + let creds = GitHubCredentials::App(GitHubAppCredentials { + app_id: "1".to_string(), + private_key_pem: "unused".to_string(), + slug: None, + }); + let no_origin = spec(None, Some(integration(&[]))); + let built = build_sandbox_env(&no_origin, Some(&creds)).unwrap(); + assert!(built.github_token.is_none()); + assert!(built.github_access.is_none()); + } + + struct FailingMinter; + + #[async_trait::async_trait] + impl InstallationTokenMinter for FailingMinter { + async fn mint(&self) -> anyhow::Result { + Err(anyhow::anyhow!("scripted mint failure")) + } + } + + #[tokio::test] + async fn eager_validation_fails_when_the_declared_token_cannot_resolve() { + let access = fabro_github::GitHubRepositoryAccess::new( + Some("https://github.com/fabro-sh/fabro"), + &["fabro-sh/keystone".parse().unwrap()].into_iter().collect(), + HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap(); + let built = BuiltSandboxEnv { + env: HashMap::new(), + github_token: Some(installation_token_source( + "fabro-sh/fabro (+1 additional)", + Arc::new(FailingMinter), + )), + github_access: access, + }; + + let err = resolve_declared_repository_token(&built).await.unwrap_err(); + let message = err.to_string(); + assert!(message.contains("declared repository set"), "{message}"); + } + + #[tokio::test] + async fn eager_validation_skips_legacy_permissions_only_runs() { + let built = BuiltSandboxEnv { + env: HashMap::new(), + github_token: Some(installation_token_source( + "fabro-sh/fabro", + Arc::new(FailingMinter), + )), + github_access: None, + }; + + resolve_declared_repository_token(&built) + .await + .expect("legacy permissions-only runs must not resolve eagerly"); + } + } } diff --git a/lib/components/fabro-workflow/src/pipeline/persist.rs b/lib/components/fabro-workflow/src/pipeline/persist.rs index 303241ff5..95cab3c32 100644 --- a/lib/components/fabro-workflow/src/pipeline/persist.rs +++ b/lib/components/fabro-workflow/src/pipeline/persist.rs @@ -171,6 +171,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("ship".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/tmp/project".to_string()), git: Some(fabro_types::GitContext { @@ -220,6 +221,7 @@ mod tests { labels: record.labels.clone().into_iter().collect(), source_directory: record.source_directory.clone(), workflow_slug: record.workflow_slug.clone(), + workflow_version_id: None, automation: record.automation.clone(), provenance: record.provenance.clone(), manifest_blob: None, diff --git a/lib/components/fabro-workflow/src/pipeline/pull_request.rs b/lib/components/fabro-workflow/src/pipeline/pull_request.rs index f3f7e4a78..c0e35acfc 100644 --- a/lib/components/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/components/fabro-workflow/src/pipeline/pull_request.rs @@ -819,20 +819,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, Utc::now(), ) @@ -1096,44 +1097,46 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: Some(fabro_types::GitContext { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: Some(fabro_types::GitContext { origin_url: String::new(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: run_spec.workflow_version_id, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1165,44 +1168,46 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: Some(fabro_types::GitContext { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: Some(fabro_types::GitContext { origin_url: String::new(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: run_spec.workflow_version_id, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1589,39 +1594,41 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: run_spec.workflow_version_id, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1808,39 +1815,41 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); // Seed a completed run so the PR body can include run details. let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - git: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + git: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: None, - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/pipeline/types.rs b/lib/components/fabro-workflow/src/pipeline/types.rs index ebbe3f4f8..c65f1fdf6 100644 --- a/lib/components/fabro-workflow/src/pipeline/types.rs +++ b/lib/components/fabro-workflow/src/pipeline/types.rs @@ -8,7 +8,9 @@ use fabro_mcp::config::McpServerSettings; use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxSpec; use fabro_template::TemplateContext; -use fabro_types::settings::run::{PullRequestSettings, RunModelControls}; +use fabro_types::settings::run::{ + PullRequestSettings, ResolvedGithubIntegration, RunModelControls, +}; use fabro_types::{ManifestPath, RunId, RunProjection}; use fabro_validate::{Diagnostic, Severity}; use fabro_vault::Vault; @@ -246,7 +248,10 @@ pub struct LlmSpec { #[derive(Clone)] pub struct SandboxEnvSpec { pub toml_env: HashMap, - pub github_permissions: Option>, + /// The resolved GitHub integration request (interpolated permissions + /// plus declared additional repositories). `None` when the run requests + /// no `GITHUB_TOKEN`. + pub github_integration: Option, pub origin_url: Option, } diff --git a/lib/components/fabro-workflow/src/run_lookup.rs b/lib/components/fabro-workflow/src/run_lookup.rs index 8a66dbf91..2b37861cf 100644 --- a/lib/components/fabro-workflow/src/run_lookup.rs +++ b/lib/components/fabro-workflow/src/run_lookup.rs @@ -490,23 +490,24 @@ mod tests { let run_spec = sample_run_spec(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: None, - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: run_spec.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: run_spec.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/run_metadata.rs b/lib/components/fabro-workflow/src/run_metadata.rs index 942b662f1..c4ae9951a 100644 --- a/lib/components/fabro-workflow/src/run_metadata.rs +++ b/lib/components/fabro-workflow/src/run_metadata.rs @@ -680,25 +680,26 @@ mod tests { let projection = RunProjection::new( "Metadata".to_string(), RunSpec { - run_id: fabro_types::fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: fabro_types::Graph::new("metadata"), - graph_source: None, - workflow_slug: Some("metadata".to_string()), - automation: None, - source_directory: Some("/Users/client/project".to_string()), - git: Some(GitContext { + run_id: fabro_types::fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: fabro_types::Graph::new("metadata"), + graph_source: None, + workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/Users/client/project".to_string()), + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: None, dirty: DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }, chrono::Utc::now(), ); diff --git a/lib/components/fabro-workflow/src/runtime_store.rs b/lib/components/fabro-workflow/src/runtime_store.rs index 2e506400c..b68caa27a 100644 --- a/lib/components/fabro-workflow/src/runtime_store.rs +++ b/lib/components/fabro-workflow/src/runtime_store.rs @@ -146,23 +146,24 @@ mod tests { async fn append_created_event(run_store: &fabro_store::RunDatabase) { let record = test_run_spec(); append_event(run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&record.settings).unwrap(), - graph: serde_json::to_value(&record.graph).unwrap(), - workflow_source: Some("digraph test {}".to_string()), - labels: std::collections::BTreeMap::new(), - source_directory: Some("/tmp/test".to_string()), - workflow_slug: Some("test".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&record.settings).unwrap(), + graph: serde_json::to_value(&record.graph).unwrap(), + workflow_source: Some("digraph test {}".to_string()), + labels: std::collections::BTreeMap::new(), + source_directory: Some("/tmp/test".to_string()), + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/stage_execution.rs b/lib/components/fabro-workflow/src/stage_execution.rs index ec755127f..dfdf8a3ba 100644 --- a/lib/components/fabro-workflow/src/stage_execution.rs +++ b/lib/components/fabro-workflow/src/stage_execution.rs @@ -198,20 +198,21 @@ mod tests { fn projection_with_stages(stages: &[(&str, u32, u32)]) -> RunProjection { let spec = RunSpec { - run_id: RunId::new(), - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: std::collections::HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: RunId::new(), + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: std::collections::HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }; let mut projection = RunProjection::new(String::new(), spec, Utc::now()); for (node_id, visit, seq) in stages { diff --git a/lib/components/fabro-workflow/src/test_support.rs b/lib/components/fabro-workflow/src/test_support.rs index c7407b7b9..6c9f55b99 100644 --- a/lib/components/fabro-workflow/src/test_support.rs +++ b/lib/components/fabro-workflow/src/test_support.rs @@ -52,7 +52,11 @@ pub(crate) fn test_configured_provider_ids( /// persisted before tests reopen the run store. async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { let executed = Box::pin(pipeline::execute(initialized.initialized)).await; - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); let state = executed.engine.run.run_store.state().await.ok(); let billing = state.as_ref().and_then(billing_from_projection); let event = build_terminal_event( @@ -65,7 +69,11 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { billing, ); executed.engine.run.emitter.emit(&event); - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); executed } @@ -182,34 +190,35 @@ async fn initialized( .expect("failed to create slate-backed test run store"); let run_store = inner_store; append_event(&run_store, &run_options.run_id, &Event::RunCreated { - run_id: run_options.run_id, - title: None, - settings: serde_json::to_value(&run_options.settings) + run_id: run_options.run_id, + title: None, + settings: serde_json::to_value(&run_options.settings) .expect("failed to serialize settings"), - graph: serde_json::to_value(graph).expect("failed to serialize graph"), - workflow_source: None, - labels: run_options + graph: serde_json::to_value(graph).expect("failed to serialize graph"), + workflow_source: None, + labels: run_options .labels .clone() .into_iter() .collect::>(), - source_directory: Some(sandbox.working_directory().to_string()), - workflow_slug: run_options.workflow_slug.clone(), - automation: None, - provenance: fabro_types::RunProvenance { + source_directory: Some(sandbox.working_directory().to_string()), + workflow_slug: run_options.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: fabro_types::RunProvenance { server: None, client: None, subject: fabro_types::Principal::System { system_kind: fabro_types::SystemActorKind::Engine, }, }, - manifest_blob: None, - spec_blob: None, - git: run_options.pre_run_git.clone(), - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + manifest_blob: None, + spec_blob: None, + git: run_options.pre_run_git.clone(), + fork_source_ref: run_options.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("failed to seed run.created event in run store"); @@ -480,7 +489,11 @@ pub async fn run_graph_with_state_and_llm_source( ) .await; let executed = pipeline::execute(initialized.initialized).await; - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); let outcome = executed.outcome?; let state = executed .engine diff --git a/lib/components/fabro-workflow/tests/it/daytona_integration.rs b/lib/components/fabro-workflow/tests/it/daytona_integration.rs index 9932f5a52..fda3aabf7 100644 --- a/lib/components/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/components/fabro-workflow/tests/it/daytona_integration.rs @@ -195,9 +195,17 @@ fn test_artifact_store(run_dir: &Path) -> ArtifactStore { async fn create_env_with_github_app( github_app: Option, ) -> DaytonaSandbox { - DaytonaSandbox::new(DaytonaConfig::default(), github_app, None, None, None, None) - .await - .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?") + DaytonaSandbox::new( + DaytonaConfig::default(), + github_app, + None, + None, + None, + None, + None, + ) + .await + .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?") } fn load_github_app_credentials() -> fabro_github::GitHubCredentials { @@ -400,7 +408,7 @@ async fn daytona_snapshot_sandbox() { }; let creds = load_github_app_credentials(); - let env = DaytonaSandbox::new(config, Some(creds), None, None, None, None) + let env = DaytonaSandbox::new(config, Some(creds), None, None, None, None, None) .await .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?"); env.initialize().await.unwrap(); @@ -1605,7 +1613,7 @@ async fn daytona_computer_use_browser_screenshot() { skip_clone: true, ..DaytonaConfig::default() }; - let env = DaytonaSandbox::new(config, None, None, None, None, None) + let env = DaytonaSandbox::new(config, None, None, None, None, None, None) .await .expect("DAYTONA_API_KEY must be set"); env.initialize().await.unwrap(); @@ -1753,7 +1761,7 @@ async fn daytona_playwright_mcp_sandbox_transport() { skip_clone: true, ..DaytonaConfig::default() }; - let sandbox = DaytonaSandbox::new(config, None, None, None, None, None) + let sandbox = DaytonaSandbox::new(config, None, None, None, None, None, None) .await .expect("DAYTONA_API_KEY must be set"); sandbox.initialize().await.unwrap(); diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 42ea14630..8104f7802 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -13332,7 +13332,7 @@ async fn asset_collection_docker_sandbox() { ..Default::default() }; let sandbox: Arc = Arc::new( - fabro_agent::DockerSandbox::new(config, None, None, None, None) + fabro_agent::DockerSandbox::new(config, None, None, None, None, None) .expect("Docker not available"), ); sandbox.initialize().await.expect("Docker init failed"); diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index 0fb3cccd0..73f0bfd19 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -208,6 +208,19 @@ fn main() { ("DiffSummary", "fabro_types::DiffSummary", &[]), ("RepositoryRef", "fabro_types::RepositoryRef", &[]), ("WorkflowSettings", "fabro_types::WorkflowSettings", &[]), + // Run-level GitHub integration settings reuse the canonical resolved + // types instead of generating parallel API DTOs; the wire shape is + // identical (InterpString serializes as its source string). + ( + "RunIntegrationsSettings", + "fabro_types::settings::run::RunIntegrationsSettings", + &[], + ), + ( + "RunIntegrationsGithubSettings", + "fabro_types::settings::run::RunIntegrationsGithubSettings", + &[], + ), ("ServerSettings", "fabro_types::ServerSettings", &[]), ( "ServerNamespace", diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index c53e682c7..a845a8252 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -25,7 +25,10 @@ pub mod types { ReasoningEffortFeature, Speed as BillingSpeed, TokenCounts as CompletionUsage, }; pub use fabro_types::run_event::AgentSessionActivatedProps; - pub use fabro_types::settings::run::{McpHttpProtocol, RunModelControls, RunModelSettings}; + pub use fabro_types::settings::run::{ + McpHttpProtocol, RunIntegrationsGithubSettings, RunIntegrationsSettings, RunModelControls, + RunModelSettings, + }; pub use fabro_types::settings::server::{ GithubIntegrationSettings, GithubIntegrationStrategy, IntegrationWebhooksSettings, LogDestination, ObjectStoreSettings, ServerApiSettings, ServerArtifactsSettings, diff --git a/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs b/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs index 243268f1c..bd5af9330 100644 --- a/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs +++ b/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs @@ -1,8 +1,10 @@ -//! JSON parity test for `RunIntegrationsGithubSettings`. +//! JSON parity and type-identity tests for `RunIntegrationsGithubSettings`. //! -//! Asserts that the API-side generated `RunIntegrationsGithubSettings` and -//! the canonical Rust resolved type round-trip through the same JSON shape. -//! Covers both the populated and empty-permissions cases. +//! Asserts that the API-side `RunIntegrationsGithubSettings` is the +//! canonical Rust resolved type (via `with_replacement` in `build.rs`) and +//! that both names round-trip through the same JSON shape. Covers the +//! populated and empty permissions cases as well as populated and empty +//! `additional_repositories` sets. use fabro_api::types::{ RunIntegrationsGithubSettings as ApiRunIntegrationsGithubSettings, @@ -11,6 +13,22 @@ use fabro_api::types::{ use fabro_types::settings::run::{RunIntegrationsGithubSettings, RunIntegrationsSettings}; use serde_json::json; +/// Type-identity witnesses: the generated API names are the canonical Rust +/// types, not parallel DTOs. Compiles only when they are the same type. +#[expect(dead_code, reason = "compile-time type-identity witness")] +fn github_settings_type_identity( + value: ApiRunIntegrationsGithubSettings, +) -> RunIntegrationsGithubSettings { + value +} + +#[expect(dead_code, reason = "compile-time type-identity witness")] +fn integrations_settings_type_identity( + value: ApiRunIntegrationsSettings, +) -> RunIntegrationsSettings { + value +} + #[test] fn run_integrations_github_settings_round_trips_with_permissions() { let json_value = json!({ @@ -44,6 +62,43 @@ fn run_integrations_github_settings_round_trips_empty_permissions() { assert_eq!(serde_json::to_value(&canonical).unwrap(), json_value); } +#[test] +fn run_integrations_github_settings_round_trips_additional_repositories() { + let json_value = json!({ + "permissions": { "contents": "read" }, + "additional_repositories": ["fabro-sh/arc", "fabro-sh/keystone"], + }); + + let api: ApiRunIntegrationsGithubSettings = + serde_json::from_value(json_value.clone()).expect("api type should parse repositories"); + let canonical: RunIntegrationsGithubSettings = + serde_json::from_value(json_value.clone()).expect("canonical type should parse"); + + assert_eq!(serde_json::to_value(&api).unwrap(), json_value); + assert_eq!(serde_json::to_value(&canonical).unwrap(), json_value); +} + +#[test] +fn run_integrations_github_settings_omits_an_empty_repository_set() { + // An absent field and an explicit empty array both deserialize to the + // empty set, and the empty set serializes back with the field omitted — + // keeping single-repository settings byte-identical to older releases. + let empty_array = json!({ + "permissions": {}, + "additional_repositories": [], + }); + let omitted = json!({ "permissions": {} }); + + let api: ApiRunIntegrationsGithubSettings = + serde_json::from_value(empty_array).expect("api type should parse an empty array"); + let canonical: RunIntegrationsGithubSettings = + serde_json::from_value(omitted.clone()).expect("canonical type should parse"); + + assert!(api.additional_repositories.is_empty()); + assert_eq!(serde_json::to_value(&api).unwrap(), omitted); + assert_eq!(serde_json::to_value(&canonical).unwrap(), omitted); +} + #[test] fn run_integrations_settings_round_trips() { let json_value = json!({ diff --git a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs index f1a00a5a5..e09a867a6 100644 --- a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs +++ b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs @@ -130,6 +130,7 @@ fn run_projection_round_trips_with_pending_control_unset() { fn run_spec_json() -> serde_json::Value { serde_json::to_value(RunSpec { graph_source: Some("digraph test {}".to_string()), + workflow_version_id: Some(test_support::test_workflow_version_id()), ..test_support::test_run_spec() }) .unwrap() diff --git a/lib/foundation/fabro-config/src/layers/run.rs b/lib/foundation/fabro-config/src/layers/run.rs index 8b020666f..c18b72015 100644 --- a/lib/foundation/fabro-config/src/layers/run.rs +++ b/lib/foundation/fabro-config/src/layers/run.rs @@ -88,13 +88,23 @@ pub struct RunIntegrationsLayer { #[serde(deny_unknown_fields)] pub struct RunIntegrationsGithubLayer { #[serde(default, skip_serializing_if = "Option::is_none")] - pub permissions: Option>, + pub permissions: Option>, + /// Extra `owner/repository` slugs the minted `GITHUB_TOKEN` must cover in + /// addition to the implicit run origin. Kept as raw strings in this + /// sparse layer; slug validation happens at resolve time so diagnostics + /// can carry indexed paths. The higher-precedence list replaces the lower + /// one wholesale (`Some(vec![])` is an explicit clear); no `...` splice. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub additional_repositories: Option>, } impl Combine for RunIntegrationsGithubLayer { fn combine(self, other: Self) -> Self { Self { - permissions: self.permissions.or(other.permissions), + permissions: self.permissions.or(other.permissions), + additional_repositories: self + .additional_repositories + .or(other.additional_repositories), } } } @@ -303,6 +313,8 @@ pub struct RunCheckpointLayer { pub struct RunCloneLayer { #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub depth: Option, } /// `[run.run_branch]` — Fabro-managed checkpoint branch policy. diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 9010f25f8..ae529224e 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -1,6 +1,8 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use fabro_types::GitHubRepositorySlug; use fabro_types::settings::InterpString; +use fabro_types::settings::interp::ResolveCtx; use fabro_types::settings::run::{ ArtifactsSettings, GitAuthorSettings, HookDefinition, HookType, InterviewProviderSettings, McpServerSettings, McpTransport, MergeStrategy, NotificationProviderSettings, @@ -17,9 +19,9 @@ use crate::{ EnvironmentLayer, HookAgentMarker, HookEntry, HookTlsMode, InterviewProviderLayer, InterviewsLayer, McpEntryLayer, MergeMap, ModelRefOrSplice, NotificationProviderLayer, NotificationRouteLayer, RunAgentLayer, RunArtifactsLayer, RunCheckpointLayer, RunCloneLayer, - RunExecutionLayer, RunGitLayer, RunGoalLayer, RunIntegrationsLayer, RunLayer, - RunMetaBranchLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, RunRunBranchLayer, - RunScmLayer, StickyMap, StringOrSplice, + RunExecutionLayer, RunGitLayer, RunGoalLayer, RunIntegrationsGithubLayer, RunIntegrationsLayer, + RunLayer, RunMetaBranchLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, + RunRunBranchLayer, RunScmLayer, StickyMap, StringOrSplice, }; pub fn resolve_run( @@ -28,7 +30,7 @@ pub fn resolve_run( mcp_server_catalog: &HashMap, errors: &mut Vec, ) -> RunNamespace { - let clone = resolve_clone(layer.clone.as_ref()); + let clone = resolve_clone(layer.clone.as_ref(), errors); let run_branch = resolve_run_branch(layer.run_branch.as_ref()); let mut meta_branch = resolve_meta_branch(layer.meta_branch.as_ref()); if !run_branch.enabled { @@ -85,23 +87,140 @@ pub fn resolve_run( scm: resolve_scm(layer.scm.as_ref()), pull_request, artifacts: resolve_artifacts(layer.artifacts.as_ref(), errors), - integrations: resolve_integrations(layer.integrations.as_ref()), + integrations: resolve_integrations(layer.integrations.as_ref(), errors), } } -fn resolve_integrations(layer: Option<&RunIntegrationsLayer>) -> RunIntegrationsSettings { +fn resolve_integrations( + layer: Option<&RunIntegrationsLayer>, + errors: &mut Vec, +) -> RunIntegrationsSettings { let github = layer .and_then(|integrations| integrations.github.as_ref()) - .map(|github| RunIntegrationsGithubSettings { - // Collapse `Option>` -> `HashMap<...>`: both `None` - // and `Some({})` resolve to an empty map (no token requested). - // The presence distinction is only meaningful at merge time. - permissions: github.permissions.clone().unwrap_or_default(), - }) + .map(|github| resolve_integrations_github(github, errors)) .unwrap_or_default(); RunIntegrationsSettings { github } } +/// GitHub caps one installation token at 500 repositories; the implicit run +/// origin takes one slot. +const MAX_ADDITIONAL_REPOSITORIES: usize = 499; + +fn resolve_integrations_github( + github: &RunIntegrationsGithubLayer, + errors: &mut Vec, +) -> RunIntegrationsGithubSettings { + // Collapse `Option>` -> `HashMap<...>`: both `None` + // and `Some({})` resolve to an empty map (no token requested). + // The presence distinction is only meaningful at merge time. The same + // collapse applies to `additional_repositories` (`Some(vec![])` is an + // explicit clear that resolves to the empty set). + let permissions = github.permissions.clone().unwrap_or_default(); + let raw_repositories = github + .additional_repositories + .as_deref() + .unwrap_or_default(); + + if raw_repositories.len() > MAX_ADDITIONAL_REPOSITORIES { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.additional_repositories".to_string(), + reason: format!( + "at most {MAX_ADDITIONAL_REPOSITORIES} additional repositories are supported (the \ + run origin takes the remaining slot of GitHub's 500-repository token limit), got \ + {}", + raw_repositories.len() + ), + }); + } + + let mut additional_repositories: BTreeSet = BTreeSet::new(); + for (index, value) in raw_repositories.iter().enumerate() { + let path = format!("run.integrations.github.additional_repositories[{index}]"); + let Ok(slug) = value.parse::() else { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` is not a full GitHub `owner/repository` slug (no scheme, host, \ + ref, or extra path component)" + ), + }); + continue; + }; + if let Some(existing) = additional_repositories.get(&slug) { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` duplicates `{existing}` (repository identity is case-insensitive)" + ), + }); + continue; + } + if let Some(first) = additional_repositories.first() { + if !first.same_owner(&slug) { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` has owner `{}` but `{first}` has owner `{}`; all repositories \ + must share one owner because one GitHub App installation covers one \ + account", + slug.owner(), + first.owner() + ), + }); + continue; + } + } + additional_repositories.insert(slug); + } + + if !additional_repositories.is_empty() { + validate_additional_repository_permissions(&permissions, errors); + } + + RunIntegrationsGithubSettings { + permissions, + additional_repositories, + } +} + +/// A non-empty additional-repository set needs a token that can reach +/// repository contents. Only a literal `contents` value is checked here; a +/// templated value is re-checked after interpolation at the runtime boundary. +fn validate_additional_repository_permissions( + permissions: &HashMap, + errors: &mut Vec, +) { + if permissions.is_empty() { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.additional_repositories".to_string(), + reason: "additional repositories require [run.integrations.github.permissions] with \ + a `contents` permission; a higher layer may have cleared the permissions" + .to_string(), + }); + return; + } + let Some(contents) = permissions.get("contents") else { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.permissions".to_string(), + reason: "additional repositories require the `contents` permission (`read` or \ + `write`)" + .to_string(), + }); + return; + }; + if let Ok(literal) = contents.resolve_with(&mut ResolveCtx::new()) { + if !RunIntegrationsGithubSettings::contents_permission_allows_repository_access(&literal) { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.permissions.contents".to_string(), + reason: format!( + "additional repositories require `contents = \"read\"` or `contents = \ + \"write\"`, got `{literal}`" + ), + }); + } + } +} + fn resolve_goal(goal: Option<&RunGoalLayer>) -> Option { match goal? { RunGoalLayer::Inline(value) => Some(RunGoal::Inline(value.clone())), @@ -244,9 +363,24 @@ fn resolve_checkpoint(checkpoint: Option<&RunCheckpointLayer>) -> RunCheckpointS } } -fn resolve_clone(clone: Option<&RunCloneLayer>) -> RunCloneSettings { +fn resolve_clone( + clone: Option<&RunCloneLayer>, + errors: &mut Vec, +) -> RunCloneSettings { + let mut depth = clone + .and_then(|clone| clone.depth) + .unwrap_or(RunCloneSettings::DEFAULT_DEPTH); + if depth < 0 { + errors.push(ResolveError::Invalid { + path: "run.clone.depth".to_string(), + reason: "depth must be at least 0".to_string(), + }); + depth = RunCloneSettings::DEFAULT_DEPTH; + } + RunCloneSettings { enabled: clone.and_then(|clone| clone.enabled).unwrap_or(true), + depth, } } diff --git a/lib/foundation/fabro-config/src/tests/resolve_run.rs b/lib/foundation/fabro-config/src/tests/resolve_run.rs index d416eb651..bc6fe8bc4 100644 --- a/lib/foundation/fabro-config/src/tests/resolve_run.rs +++ b/lib/foundation/fabro-config/src/tests/resolve_run.rs @@ -126,6 +126,7 @@ fn resolves_run_defaults_from_empty_settings() { assert!(!settings.environment.lifecycle.preserve); assert!(settings.environment.lifecycle.stop_on_terminal); assert!(settings.clone.enabled); + assert_eq!(settings.clone.depth, 100); assert!(settings.run_branch.enabled); assert!(settings.run_branch.push); assert!(settings.meta_branch.enabled); @@ -296,6 +297,7 @@ _version = 1 [run.clone] enabled = false +depth = 1 [run.run_branch] enabled = true @@ -310,12 +312,52 @@ push = false .run; assert!(!settings.clone.enabled); + assert_eq!(settings.clone.depth, 1); assert!(settings.run_branch.enabled); assert!(!settings.run_branch.push); assert!(settings.meta_branch.enabled); assert!(!settings.meta_branch.push); } +#[test] +fn zero_clone_depth_requests_full_history() { + let settings = super::workflow_settings_from_toml( + r" +_version = 1 + +[run.clone] +depth = 0 +", + ) + .expect("zero clone depth should resolve") + .run; + + assert_eq!(settings.clone.depth, 0); +} + +#[test] +fn rejects_negative_clone_depth() { + let error = super::workflow_settings_from_toml( + r" +_version = 1 + +[run.clone] +depth = -1 +", + ) + .expect_err("negative clone depth should not resolve"); + + let message = error.to_string(); + assert!( + message.contains("run.clone.depth"), + "unexpected error: {message}" + ); + assert!( + message.contains("at least 0"), + "unexpected error: {message}" + ); +} + #[test] fn disabling_run_branch_forces_meta_branch_off() { let settings = super::workflow_settings_from_toml( @@ -879,6 +921,447 @@ issues = "{{ env.GH_PERM_LEVEL }}" } } +mod run_integrations_github_additional_repositories { + //! Layer + resolver tests for + //! `[run.integrations.github].additional_repositories`. + //! + //! The list replaces wholesale across layers (`[]` is an explicit clear), + //! resolves independently from `permissions`, and validates each entry as + //! a full `owner/repository` slug with indexed error paths. + + use crate::SettingsLayer; + use crate::layers::Combine; + + fn parse_settings(source: &str) -> SettingsLayer { + source + .parse::() + .expect("fixture should parse via SettingsLayer") + } + + fn invalid_paths_and_reasons(error: crate::Error) -> Vec<(String, String)> { + let errors = match error { + crate::Error::Resolve { errors, .. } => errors, + other => panic!("expected structured resolve errors, got {other:#}"), + }; + errors + .into_iter() + .map(|error| match error { + crate::ResolveError::Invalid { path, reason } => (path, reason), + other => panic!("expected invalid-value error, got {other}"), + }) + .collect() + } + + fn resolved_repositories(settings: &fabro_types::WorkflowSettings) -> Vec { + settings + .run + .integrations + .github + .additional_repositories + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn resolves_one_and_multiple_repositories() { + let one = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ) + .expect("one additional repository should resolve"); + assert_eq!(resolved_repositories(&one), vec!["fabro-sh/keystone"]); + assert!(one.run.integrations.github.has_additional_repositories()); + + let many = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "fabro-sh/arc"] +permissions = { contents = "write" } +"#, + ) + .expect("multiple additional repositories should resolve"); + assert_eq!(resolved_repositories(&many), vec![ + "fabro-sh/arc", + "fabro-sh/keystone", + ]); + } + + #[test] + fn rejects_malformed_slugs_with_indexed_paths() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = [ + "fabro-sh/keystone", + "https://github.com/fabro-sh/arc", + "git@github.com:fabro-sh/arc.git", + "fabro-sh/arc@main", + "not-a-slug", +] +permissions = { contents = "read" } +"#, + ) + .expect_err("malformed slugs should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec![ + "run.integrations.github.additional_repositories[1]", + "run.integrations.github.additional_repositories[2]", + "run.integrations.github.additional_repositories[3]", + "run.integrations.github.additional_repositories[4]", + ] + ); + assert!( + invalid[0].1.contains("owner/repository"), + "reason should explain the slug grammar: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_duplicate_and_case_variant_duplicate_slugs() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "Fabro-SH/Keystone"] +permissions = { contents = "read" } +"#, + ) + .expect_err("case-variant duplicate slugs should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories[1]"] + ); + assert!( + invalid[0].1.contains("case-insensitive"), + "reason should mention case-insensitive identity: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_cross_owner_additional_repositories() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "lithoscomputer/conveyor"] +permissions = { contents = "read" } +"#, + ) + .expect_err("cross-owner additional repositories should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories[1]"] + ); + assert!( + invalid[0].1.contains("share one owner"), + "reason should explain the single-owner requirement: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_more_than_the_installation_token_repository_limit() { + let repositories = (0..500) + .map(|index| format!("\"owner/repo-{index}\"")) + .collect::>() + .join(", "); + let error = super::workflow_settings_from_toml(&format!( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = [{repositories}] +permissions = {{ contents = "read" }} +"#, + )) + .expect_err("500 additional repositories should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid[0].0, + "run.integrations.github.additional_repositories" + ); + assert!(invalid[0].1.contains("499"), "{}", invalid[0].1); + } + + #[test] + fn rejects_additional_repositories_without_permissions() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +"#, + ) + .expect_err("additional repositories without permissions should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories"] + ); + } + + #[test] + fn rejects_additional_repositories_without_the_contents_permission() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { issues = "read" } +"#, + ) + .expect_err("additional repositories require the contents permission"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.permissions"] + ); + } + + #[test] + fn rejects_a_literal_contents_permission_that_is_not_read_or_write() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "admin" } +"#, + ) + .expect_err("literal contents permission must be read or write"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.permissions.contents"] + ); + } + + #[test] + fn defers_a_templated_contents_permission_to_the_runtime_boundary() { + let settings = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "{{ vars.GH_CONTENTS }}" } +"#, + ) + .expect("templated contents permission resolves; the value is re-checked at runtime"); + + assert_eq!(resolved_repositories(&settings), vec!["fabro-sh/keystone"]); + } + + #[test] + fn higher_layer_replaces_the_repository_list_wholesale() { + let workflow = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +"#, + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/arc", "fabro-sh/widgets"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + + // The lists never union: the higher layer's single entry wins, while + // the permission map inherits independently from the lower layer. + assert_eq!(resolved_repositories(&resolved), vec!["fabro-sh/keystone"]); + assert_eq!( + resolved.run.integrations.github.permissions.len(), + 1, + "permissions should inherit from the lower layer" + ); + } + + #[test] + fn absent_higher_layer_inherits_the_lower_repository_list() { + let workflow = parse_settings("_version = 1\n"); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + assert_eq!(resolved_repositories(&resolved), vec!["fabro-sh/keystone"]); + } + + #[test] + fn empty_higher_layer_list_clears_inherited_repositories() { + let workflow = parse_settings( + r" +_version = 1 + +[run.integrations.github] +additional_repositories = [] +", + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + + assert!( + resolved + .run + .integrations + .github + .additional_repositories + .is_empty(), + "explicit [] should clear the inherited repository list" + ); + // Permissions survive the repository clear: each field resolves + // independently. + assert!(resolved.run.integrations.github.is_token_requested()); + } + + #[test] + fn rejects_repositories_that_survive_a_cross_layer_permission_clear() { + let workflow = parse_settings( + r" +_version = 1 + +[run.integrations.github] +permissions = {} +", + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let error = super::workflow_settings_from_layer(merged) + .map(|_| ()) + .expect_err("repositories with cleared permissions should not resolve"); + + let message = error.to_string(); + assert!( + message.contains("additional_repositories"), + "error should name the invalid combination: {message}" + ); + } + + #[test] + fn permissions_only_and_fully_empty_shapes_are_preserved() { + let permissions_only = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github.permissions] +issues = "read" +"#, + ) + .expect("permissions-only settings should resolve"); + assert!( + permissions_only + .run + .integrations + .github + .additional_repositories + .is_empty() + ); + assert!( + permissions_only + .run + .integrations + .github + .is_token_requested() + ); + + let empty = super::workflow_settings_from_toml("_version = 1\n") + .expect("empty settings should resolve"); + assert!( + empty + .run + .integrations + .github + .additional_repositories + .is_empty() + ); + assert!(!empty.run.integrations.github.is_token_requested()); + } +} + mod run_agent { use crate::SettingsLayer; use crate::layers::Combine; diff --git a/lib/foundation/fabro-dev/src/commands/bench_tests.rs b/lib/foundation/fabro-dev/src/commands/bench_tests.rs index 39616c4b8..81264fb9d 100644 --- a/lib/foundation/fabro-dev/src/commands/bench_tests.rs +++ b/lib/foundation/fabro-dev/src/commands/bench_tests.rs @@ -8,7 +8,7 @@ use clap::Args; use quick_xml::events::{BytesStart, Event}; use quick_xml::reader::Reader; -use super::{PlannedCommand, capture_command, command, workspace_root}; +use super::{PlannedCommand, command, workspace_root}; const TOOL_CONFIG_RELATIVE: &str = "target/bench-tests/nextest-tool.toml"; const TOOL_CONFIG_BODY: &str = "\ @@ -67,7 +67,7 @@ pub(crate) fn bench_tests(args: BenchTestsArgs) -> Result<()> { } let root = workspace_root(); - let git_sha = resolve_head_sha(&root)?; + let git_sha = super::resolve_git_revision(&root, "HEAD")?; let tool_config = ensure_tool_config(&root)?; let junit_path = root .join("target") @@ -180,21 +180,6 @@ fn ensure_tool_config(root: &Path) -> Result { Ok(path) } -fn resolve_head_sha(root: &Path) -> Result { - let cmd = PlannedCommand::new("git").arg("rev-parse").arg("HEAD"); - let output = capture_command(root, &cmd)?; - if !output.status.success() { - bail!( - "git rev-parse HEAD failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(String::from_utf8(output.stdout) - .context("git rev-parse HEAD returned non-UTF-8")? - .trim() - .to_string()) -} - #[expect( clippy::disallowed_methods, reason = "fabro-dev opens the CSV file synchronously; bench-tests is a CLI tool, not a Tokio runtime" diff --git a/lib/foundation/fabro-dev/src/commands/mod.rs b/lib/foundation/fabro-dev/src/commands/mod.rs index a59142898..cb2d9c8cf 100644 --- a/lib/foundation/fabro-dev/src/commands/mod.rs +++ b/lib/foundation/fabro-dev/src/commands/mod.rs @@ -148,6 +148,24 @@ pub(crate) fn capture_command(cwd: &Path, planned: &PlannedCommand) -> Result Result { + let command = PlannedCommand::new("git") + .arg("rev-parse") + .arg("--verify") + .arg(revision); + let output = capture_command(root, &command)?; + if !output.status.success() { + anyhow::bail!( + "git rev-parse --verify {revision} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8(output.stdout) + .with_context(|| format!("git rev-parse --verify {revision} returned non-UTF-8"))? + .trim() + .to_string()) +} + #[expect( clippy::disallowed_methods, reason = "dev CLI sanitizes inherited Cargo build-script env before spawning nested cargo" diff --git a/lib/foundation/fabro-dev/src/commands/release.rs b/lib/foundation/fabro-dev/src/commands/release.rs index e088f38d3..9883f91af 100644 --- a/lib/foundation/fabro-dev/src/commands/release.rs +++ b/lib/foundation/fabro-dev/src/commands/release.rs @@ -8,6 +8,9 @@ use super::{PlannedCommand, capture_command, run_command, spa_refresh, workspace const RELEASE_EPOCH: &str = "2026-01-01"; const RELEASE_TEST_SEGMENT_WRITE_KEY: &str = "fake-for-local-smoke"; +const MAX_PUSH_ATTEMPTS: u32 = 4; +const RELEASE_BRANCH: &str = "main"; +const RELEASE_REMOTE: &str = "origin"; #[derive(Debug, Args)] pub(crate) struct ReleaseArgs { @@ -36,6 +39,55 @@ struct ReleasePlan { root: PathBuf, } +struct ReleaseVersions { + current: String, + next: String, +} + +impl ReleaseVersions { + fn tag(&self) -> String { + format!("v{}", self.next) + } +} + +struct RemoteReleaseState { + main: String, + tag: Option, +} + +#[derive(Debug, PartialEq, Eq)] +enum PushFailureDisposition { + Published, + Retryable, + Unchanged, + Inconsistent, +} + +impl RemoteReleaseState { + fn classify_push_failure( + &self, + remote_main_before: &str, + release_head: &str, + local_contains_remote_before: bool, + ) -> PushFailureDisposition { + let main_is_release = self.main == release_head; + let tag_is_release = self.tag.as_deref() == Some(release_head); + + if main_is_release && tag_is_release { + PushFailureDisposition::Published + } else if main_is_release || tag_is_release { + PushFailureDisposition::Inconsistent + } else if !local_contains_remote_before + || self.main != remote_main_before + || self.tag.is_some() + { + PushFailureDisposition::Retryable + } else { + PushFailureDisposition::Unchanged + } + } +} + #[expect( clippy::print_stdout, reason = "dev release command reports progress and dry-run commands directly" @@ -52,64 +104,21 @@ pub(crate) fn release(args: ReleaseArgs) -> Result<()> { }; let cargo_toml = plan.root.join("Cargo.toml"); - let current_version = read_current_version(&cargo_toml)?; - println!("Current version: {current_version}"); - - let base_version = plan.next_base_version()?; - let new_version = plan.compute_release_version(&base_version)?; - let tag = format!("v{new_version}"); - println!("Releasing {new_version} (tag {tag})"); + let versions = plan.compute_versions(&cargo_toml)?; + let tag = versions.tag(); + println!("Current version: {}", versions.current); + println!("Releasing {} (tag {tag})", versions.next); if plan.dry_run { - plan.print_dry_run(¤t_version, &new_version, &tag); + plan.print_dry_run(&versions); return Ok(()); } + plan.ensure_main_branch()?; plan.ensure_clean_worktree()?; spa_refresh::spa_refresh_root(&plan.root)?; plan.verify_release_tests()?; - update_version(&cargo_toml, ¤t_version, &new_version)?; - println!("Updated {}", cargo_toml.display()); - - run_command( - &plan.root, - &PlannedCommand::new("cargo") - .arg("update") - .arg("--workspace"), - )?; - println!("Updated Cargo.lock"); - - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("add") - .arg("Cargo.toml") - .arg("Cargo.lock"), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("commit") - .arg("-m") - .arg(format!("Bump version to {new_version}")), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("tag") - .arg("-a") - .arg(&tag) - .arg("-m") - .arg(&tag), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("push") - .arg("origin") - .arg("main") - .arg(&tag), - )?; + let tag = plan.commit_tag_and_push(&cargo_toml, versions)?; println!(); println!("Released {tag}"); @@ -156,6 +165,347 @@ impl ReleasePlan { } } + fn compute_versions(&self, cargo_toml: &Path) -> Result { + let current = read_current_version(cargo_toml)?; + let base_version = self.next_base_version()?; + let next = self.compute_release_version(&base_version)?; + Ok(ReleaseVersions { current, next }) + } + + /// Commits the version bump, tags it, and pushes `main` plus the tag + /// atomically. When the push is rejected because origin/main moved while + /// the release ran, rebuilds the bump commit and tag on the fresh tip + /// and retries. + #[expect( + clippy::print_stdout, + reason = "dev release command reports push retry progress directly" + )] + fn commit_tag_and_push( + &self, + cargo_toml: &Path, + mut versions: ReleaseVersions, + ) -> Result { + let mut attempt = 1; + loop { + self.ensure_main_branch()?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running")?; + + let start_head = super::resolve_git_revision(&self.root, "HEAD")?; + let remote_tracking_ref = Self::remote_tracking_ref(); + let remote_main_before = + super::resolve_git_revision(&self.root, &remote_tracking_ref).with_context(|| { + format!( + "release requires a local {} tracking ref; fetch {RELEASE_REMOTE} and retry", + Self::remote_branch() + ) + })?; + let local_contains_remote_before = + self.commit_is_ancestor(&remote_main_before, &start_head)?; + let tag = versions.tag(); + self.create_bump_commit_and_tag(cargo_toml, &versions)?; + let release_head = super::resolve_git_revision(&self.root, "HEAD")?; + let Err(push_error) = self.push_main_and_tag(&tag) else { + return Ok(tag); + }; + + let disposition = self + .push_failure_disposition( + &tag, + &remote_main_before, + &release_head, + local_contains_remote_before, + ) + .context("push failed and origin could not be verified")?; + match disposition { + PushFailureDisposition::Published => { + println!( + "Push reported an error, but {RELEASE_REMOTE} contains {RELEASE_BRANCH} \ + and {tag} at {release_head}" + ); + return Ok(tag); + } + PushFailureDisposition::Unchanged => return Err(push_error), + PushFailureDisposition::Inconsistent => { + return Err(push_error.context( + "origin contains only part of the release; inspect the remote refs before \ + retrying", + )); + } + PushFailureDisposition::Retryable => { + if attempt == MAX_PUSH_ATTEMPTS { + return Err(push_error); + } + } + } + + let remote_branch = Self::remote_branch(); + println!( + "Origin changed during push attempt {attempt} of {MAX_PUSH_ATTEMPTS}; rebuilding \ + the release on the latest {remote_branch}" + ); + self.resync_with_origin_main(&tag, &start_head, &release_head)?; + versions = self.compute_versions(cargo_toml)?; + println!("Retrying as {} (tag {})", versions.next, versions.tag()); + attempt += 1; + } + } + + #[expect( + clippy::print_stdout, + reason = "dev release command reports version bump progress directly" + )] + fn create_bump_commit_and_tag( + &self, + cargo_toml: &Path, + versions: &ReleaseVersions, + ) -> Result<()> { + update_version(cargo_toml, &versions.current, &versions.next)?; + println!("Updated {}", cargo_toml.display()); + + let [cargo_update, git_add, git_commit, git_tag] = Self::bump_commands(versions); + run_command(&self.root, &cargo_update)?; + println!("Updated Cargo.lock"); + + for command in [git_add, git_commit, git_tag] { + run_command(&self.root, &command)?; + } + Ok(()) + } + + fn bump_commands(versions: &ReleaseVersions) -> [PlannedCommand; 4] { + let tag = versions.tag(); + [ + PlannedCommand::new("cargo") + .arg("update") + .arg("--workspace"), + PlannedCommand::new("git") + .arg("add") + .arg("Cargo.toml") + .arg("Cargo.lock"), + PlannedCommand::new("git") + .arg("commit") + .arg("-m") + .arg(format!("Bump version to {}", versions.next)), + PlannedCommand::new("git") + .arg("tag") + .arg("-a") + .arg(&tag) + .arg("-m") + .arg(tag), + ] + } + + fn push_main_and_tag(&self, tag: &str) -> Result<()> { + run_command(&self.root, &Self::push_command(tag)) + } + + fn push_failure_disposition( + &self, + tag: &str, + remote_main_before: &str, + release_head: &str, + local_contains_remote_before: bool, + ) -> Result { + Ok(self.remote_release_state(tag)?.classify_push_failure( + remote_main_before, + release_head, + local_contains_remote_before, + )) + } + + fn commit_is_ancestor(&self, ancestor: &str, descendant: &str) -> Result { + let output = capture_command( + &self.root, + &PlannedCommand::new("git") + .arg("merge-base") + .arg("--is-ancestor") + .arg(ancestor) + .arg(descendant), + )?; + if output.status.success() { + return Ok(true); + } + if output.status.code() == Some(1) { + return Ok(false); + } + bail!( + "failed to compare git commits {ancestor} and {descendant}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + } + + fn remote_release_state(&self, tag: &str) -> Result { + let branch_ref = Self::branch_ref(); + let tag_ref = Self::tag_ref(tag); + let peeled_tag_ref = format!("{tag_ref}^{{}}"); + let output = capture_command( + &self.root, + &PlannedCommand::new("git") + .arg("ls-remote") + .arg(RELEASE_REMOTE) + .arg(&branch_ref) + .arg(&tag_ref) + .arg(&peeled_tag_ref), + )?; + if !output.status.success() { + bail!( + "failed to inspect release refs on {RELEASE_REMOTE}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + let stdout = + String::from_utf8(output.stdout).context("git ls-remote returned non-UTF-8 output")?; + let mut main = None; + let mut tag_object = None; + let mut peeled_tag = None; + for line in stdout.lines() { + let (object_id, reference) = line + .split_once('\t') + .with_context(|| format!("unexpected git ls-remote output: {line}"))?; + if reference == branch_ref { + main = Some(object_id.to_string()); + } else if reference == tag_ref { + tag_object = Some(object_id.to_string()); + } else if reference == peeled_tag_ref { + peeled_tag = Some(object_id.to_string()); + } + } + + let remote_branch = Self::remote_branch(); + let main = main + .with_context(|| format!("{remote_branch} was not present in git ls-remote output"))?; + Ok(RemoteReleaseState { + main, + tag: peeled_tag.or(tag_object), + }) + } + + /// Drops the bump commit and tag this run created, then fast-forwards + /// onto the updated origin/main. `--keep` preserves worktree edits, and + /// `--ff-only` refuses to discard commits that did not come from origin. + fn resync_with_origin_main( + &self, + tag: &str, + start_head: &str, + release_head: &str, + ) -> Result<()> { + self.ensure_main_branch()?; + let current_head = super::resolve_git_revision(&self.root, "HEAD")?; + if current_head != release_head { + bail!( + "local {RELEASE_BRANCH} changed while the release was running; refusing to move \ + it from {current_head} back to {start_head}" + ); + } + + let tag_ref = Self::tag_ref(tag); + let tag_object = super::resolve_git_revision(&self.root, &tag_ref) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + let tag_commit = super::resolve_git_revision(&self.root, &format!("{tag_ref}^{{commit}}")) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + if tag_commit != release_head { + bail!( + "local release tag {tag} changed while the release was running; refusing to \ + delete it" + ); + } + + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("reset") + .arg("--keep") + .arg(start_head), + ) + .context( + "working tree changed while the release was running; local edits were preserved", + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("update-ref") + .arg("-d") + .arg(&tag_ref) + .arg(tag_object), + ) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running")?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("fetch") + .arg("--tags") + .arg(RELEASE_REMOTE) + .arg(RELEASE_BRANCH), + )?; + let remote_branch = Self::remote_branch(); + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("merge") + .arg("--ff-only") + .arg(&remote_branch), + ) + .with_context(|| { + format!( + "local {RELEASE_BRANCH} has diverged from {remote_branch}; reconcile manually and \ + rerun the release" + ) + })?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running") + } + + fn branch_ref() -> String { + format!("refs/heads/{RELEASE_BRANCH}") + } + + fn remote_branch() -> String { + format!("{RELEASE_REMOTE}/{RELEASE_BRANCH}") + } + + fn remote_tracking_ref() -> String { + format!("refs/remotes/{RELEASE_REMOTE}/{RELEASE_BRANCH}") + } + + fn tag_ref(tag: &str) -> String { + format!("refs/tags/{tag}") + } + + fn push_command(tag: &str) -> PlannedCommand { + PlannedCommand::new("git") + .arg("push") + .arg("--atomic") + .arg(RELEASE_REMOTE) + .arg(RELEASE_BRANCH) + .arg(tag) + } + + fn ensure_main_branch(&self) -> Result<()> { + let output = capture_command( + &self.root, + &PlannedCommand::new("git") + .arg("symbolic-ref") + .arg("--quiet") + .arg("HEAD"), + )?; + if !output.status.success() { + bail!("release must run from {RELEASE_BRANCH}; HEAD is not a local branch"); + } + + let head_ref = String::from_utf8(output.stdout) + .context("git symbolic-ref returned non-UTF-8 output")?; + let head_ref = head_ref.trim(); + if head_ref != Self::branch_ref() { + let branch = head_ref.strip_prefix("refs/heads/").unwrap_or(head_ref); + bail!("release must run from {RELEASE_BRANCH}; current branch is {branch}"); + } + Ok(()) + } + fn ensure_clean_worktree(&self) -> Result<()> { let output = capture_command( &self.root, @@ -195,7 +545,7 @@ impl ReleasePlan { clippy::print_stdout, reason = "dev release command reports dry-run commands directly" )] - fn print_dry_run(&self, current_version: &str, new_version: &str, tag: &str) { + fn print_dry_run(&self, versions: &ReleaseVersions) { println!("DRY RUN: would refresh SPA assets:"); println!("{}", Self::spa_refresh_command().to_shell_line()); @@ -206,31 +556,15 @@ impl ReleasePlan { println!("{}", Self::release_tests_command().to_shell_line()); } - println!("DRY RUN: would update Cargo.toml version {current_version} -> {new_version}"); - for command in [ - PlannedCommand::new("cargo") - .arg("update") - .arg("--workspace"), - PlannedCommand::new("git") - .arg("add") - .arg("Cargo.toml") - .arg("Cargo.lock"), - PlannedCommand::new("git") - .arg("commit") - .arg("-m") - .arg(format!("Bump version to {new_version}")), - PlannedCommand::new("git") - .arg("tag") - .arg("-a") - .arg(tag) - .arg("-m") - .arg(tag), - PlannedCommand::new("git") - .arg("push") - .arg("origin") - .arg("main") - .arg(tag), - ] { + println!( + "DRY RUN: would update Cargo.toml version {} -> {}", + versions.current, versions.next + ); + let tag = versions.tag(); + for command in Self::bump_commands(versions) + .into_iter() + .chain(std::iter::once(Self::push_command(&tag))) + { println!("{}", command.to_shell_line()); } } @@ -321,3 +655,396 @@ fn workspace_package_version<'a>( ) }) } + +#[cfg(test)] +mod tests { + use super::*; + + const WORKSPACE_MANIFEST: &str = r#"[workspace] +members = ["app"] + +[workspace.package] +version = "0.1.0" +"#; + + const MEMBER_MANIFEST: &str = r#"[package] +name = "app" +edition = "2021" +version.workspace = true +"#; + + fn git(root: &Path, args: &[&str]) -> String { + let mut command = PlannedCommand::new("git"); + for arg in args { + command = command.arg(*arg); + } + let output = capture_command(root, &command).expect("git should spawn"); + assert!( + output.status.success(), + "git {args:?} failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + fn configure_identity(repo: &Path) { + git(repo, &["config", "user.name", "Release Test"]); + git(repo, &["config", "user.email", "release-test@example.com"]); + } + + /// A fixture with a bare `origin`, a `work` clone releases run from, and + /// an `other` clone that simulates concurrent pushes. + struct RaceFixture { + _dir: tempfile::TempDir, + origin: PathBuf, + work: PathBuf, + other: PathBuf, + } + + #[expect( + clippy::disallowed_methods, + reason = "release tests build git fixture repositories synchronously" + )] + fn race_fixture() -> RaceFixture { + let dir = tempfile::tempdir().expect("creating fixture"); + let origin = dir.path().join("origin.git"); + let work = dir.path().join("work"); + let other = dir.path().join("other"); + + std::fs::create_dir(&origin).expect("creating origin dir"); + git(&origin, &["init", "--bare", "-b", "main"]); + + std::fs::create_dir(&work).expect("creating work dir"); + git(&work, &["init", "-b", "main"]); + configure_identity(&work); + std::fs::write(work.join("Cargo.toml"), WORKSPACE_MANIFEST).expect("writing manifest"); + std::fs::create_dir_all(work.join("app/src")).expect("creating member dirs"); + std::fs::write(work.join("app/Cargo.toml"), MEMBER_MANIFEST) + .expect("writing member manifest"); + std::fs::write(work.join("app/src/lib.rs"), "").expect("writing member lib"); + git(&work, &["add", "."]); + git(&work, &["commit", "-m", "initial"]); + git(&work, &[ + "remote", + "add", + "origin", + origin.to_str().expect("origin path should be utf-8"), + ]); + git(&work, &["push", "-u", "origin", "main"]); + + git(dir.path(), &[ + "clone", + origin.to_str().expect("origin path should be utf-8"), + "other", + ]); + configure_identity(&other); + + RaceFixture { + _dir: dir, + origin, + work, + other, + } + } + + #[expect( + clippy::disallowed_methods, + reason = "release tests write fixture files synchronously" + )] + fn write_file(path: &Path, contents: &str) { + std::fs::write(path, contents).expect("writing fixture file"); + } + + fn nightly_plan(root: &Path) -> ReleasePlan { + ReleasePlan { + nightly: true, + release_date: NaiveDate::from_ymd_opt(2026, 1, 1).expect("valid release date"), + dry_run: false, + skip_tests: true, + root: root.to_path_buf(), + } + } + + fn push_concurrent_commit(fixture: &RaceFixture) { + write_file(&fixture.other.join("README.md"), "concurrent\n"); + git(&fixture.other, &["add", "README.md"]); + git(&fixture.other, &["commit", "-m", "concurrent work"]); + git(&fixture.other, &["push", RELEASE_REMOTE, RELEASE_BRANCH]); + } + + #[test] + fn failed_push_check_recognizes_a_published_release() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + git(&fixture.work, &[ + "push", + "--atomic", + RELEASE_REMOTE, + RELEASE_BRANCH, + tag.as_str(), + ]); + + assert_eq!( + plan.push_failure_disposition(&tag, &start_head, &release_head, true) + .expect("inspecting published refs"), + PushFailureDisposition::Published + ); + } + + #[test] + fn push_failure_classification_rejects_an_unchanged_remote() { + let state = RemoteReleaseState { + main: "start".to_string(), + tag: None, + }; + + assert_eq!( + state.classify_push_failure("start", "release", true), + PushFailureDisposition::Unchanged + ); + } + + #[test] + fn release_requires_the_main_branch() { + let fixture = race_fixture(); + git(&fixture.work, &["switch", "-c", "feature"]); + + let error = nightly_plan(&fixture.work) + .ensure_main_branch() + .expect_err("a release from another branch should fail"); + + assert!( + format!("{error:#}").contains("release must run from main; current branch is feature"), + "error should identify the required and current branches: {error:#}" + ); + } + + #[test] + fn resync_preserves_worktree_edits() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + write_file(&fixture.work.join("app/src/lib.rs"), "user edit\n"); + + let error = plan + .resync_with_origin_main(&tag, &start_head, &release_head) + .expect_err("resync should stop when the worktree changes"); + + assert!( + format!("{error:#}").contains("working tree changed while the release was running"), + "error should explain why resync stopped: {error:#}" + ); + let diff = git(&fixture.work, &["diff", "--", "app/src/lib.rs"]); + assert!( + diff.contains("+user edit"), + "resync should preserve the worktree edit:\n{diff}" + ); + } + + #[test] + fn resync_preserves_a_commit_created_after_the_release_commit() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + write_file(&fixture.work.join("local.txt"), "concurrent local work\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "concurrent local work"]); + + let error = plan + .resync_with_origin_main(&tag, &start_head, &release_head) + .expect_err("resync should stop when local main changes"); + + assert!( + format!("{error:#}").contains("local main changed while the release was running"), + "error should explain why resync stopped: {error:#}" + ); + let subject = git(&fixture.work, &["log", "-1", "--format=%s"]); + assert_eq!(subject, "concurrent local work"); + git(&fixture.work, &[ + "rev-parse", + "--verify", + &ReleasePlan::tag_ref(&tag), + ]); + } + + #[test] + fn push_does_not_retry_when_remote_refs_are_unchanged() { + let fixture = race_fixture(); + write_file(&fixture.work.join("local.txt"), "unpushed local work\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "unpushed local work"]); + + let missing_push_remote = fixture + .origin + .parent() + .expect("origin should have a parent") + .join("missing.git"); + git(&fixture.work, &[ + "remote", + "set-url", + "--push", + RELEASE_REMOTE, + missing_push_remote + .to_str() + .expect("push remote path should be utf-8"), + ]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + plan.commit_tag_and_push(&cargo_toml, versions) + .expect_err("an unrelated push failure should be returned"); + + let subjects = git(&fixture.work, &["log", "--format=%s"]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "unpushed local work", + "initial" + ]); + } + + #[test] + fn push_rebuilds_bump_commit_when_origin_main_moves() { + let fixture = race_fixture(); + push_concurrent_commit(&fixture); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue itself when origin/main moves"); + + assert_eq!(tag, "v0.100.0-nightly.0"); + let subjects = git(&fixture.origin, &["log", "--format=%s", "main"]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "concurrent work", + "initial" + ]); + git(&fixture.origin, &[ + "rev-parse", + "--verify", + "refs/tags/v0.100.0-nightly.0", + ]); + } + + #[test] + fn push_rebuilds_when_origin_main_was_fetched_but_not_merged() { + let fixture = race_fixture(); + push_concurrent_commit(&fixture); + git(&fixture.work, &["fetch", RELEASE_REMOTE, RELEASE_BRANCH]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue a local main behind its tracking branch"); + + assert_eq!(tag, "v0.100.0-nightly.0"); + let subjects = git(&fixture.origin, &["log", "--format=%s", RELEASE_BRANCH]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "concurrent work", + "initial" + ]); + } + + #[test] + fn push_recomputes_version_when_tag_is_taken() { + let fixture = race_fixture(); + + git(&fixture.other, &[ + "tag", + "-a", + "v0.100.0-nightly.0", + "-m", + "v0.100.0-nightly.0", + ]); + git(&fixture.other, &["push", "origin", "v0.100.0-nightly.0"]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue itself when the tag is taken"); + + assert_eq!(tag, "v0.100.0-nightly.1"); + git(&fixture.origin, &[ + "rev-parse", + "--verify", + "refs/tags/v0.100.0-nightly.1", + ]); + let subject = git(&fixture.origin, &["log", "-1", "--format=%s", "main"]); + assert_eq!(subject, "Bump version to 0.100.0-nightly.1"); + } + + #[test] + fn push_preserves_unpushed_local_commits_on_divergence() { + let fixture = race_fixture(); + + write_file(&fixture.work.join("local.txt"), "local\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "unpushed local work"]); + + push_concurrent_commit(&fixture); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let error = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect_err("diverged local main should fail instead of being reset away"); + + assert!( + format!("{error:#}").contains("diverged"), + "error should explain the divergence: {error:#}" + ); + let subject = git(&fixture.work, &["log", "-1", "--format=%s"]); + assert_eq!(subject, "unpushed local work"); + } +} diff --git a/lib/foundation/fabro-dev/tests/it/release.rs b/lib/foundation/fabro-dev/tests/it/release.rs index 28fb4cf94..a548e25ed 100644 --- a/lib/foundation/fabro-dev/tests/it/release.rs +++ b/lib/foundation/fabro-dev/tests/it/release.rs @@ -34,7 +34,7 @@ members = [] version = "0.1.0" "#, ); - git(fixture.path(), &["init"]); + git(fixture.path(), &["init", "-b", "main"]); git(fixture.path(), &["config", "user.name", "Release Test"]); git(fixture.path(), &[ "config", @@ -107,6 +107,10 @@ fn dry_run_computes_stable_version_from_date() { stdout.contains("git tag -a v0.100.0 -m v0.100.0"), "dry-run should print release tag command:\n{stdout}" ); + assert!( + stdout.contains("git push --atomic origin main v0.100.0"), + "dry-run should print the atomic push command:\n{stdout}" + ); assert!( stdout.contains( "unset GH_TOKEN GITHUB_TOKEN && SEGMENT_WRITE_KEY=fake-for-local-smoke cargo nextest run --locked" diff --git a/lib/foundation/fabro-model/src/catalog/providers/venice.toml b/lib/foundation/fabro-model/src/catalog/providers/venice.toml index dc97c4ab4..9791591e0 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/venice.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/venice.toml @@ -1,46 +1,212 @@ +# Model IDs, capabilities, contexts, and prices are from Venice's published +# model catalog, verified 2026-08-21: +# https://github.com/veniceai/api-docs/blob/59a300b1d036c0c0acc0e5f75c0ab0dd07c40c1c/data/static-models.json + [providers.venice] display_name = "Venice" adapter = "openai_compatible" base_url = "https://api.venice.ai/api/v1" priority = 35 aliases = ["venice-ai"] +billing_policy = "openai" [providers.venice.auth] credentials = ["env:VENICE_API_KEY", "vault:VENICE_API_KEY"] -[providers.venice.models."venice-uncensored-1-2"] -display_name = "Venice Uncensored 1.2" -family = "venice-uncensored" +[providers.venice.models."kimi-k3"] +display_name = "Kimi K3" +family = "kimi-k3" +agent_profile = "kimi" +aliases = ["kimi"] + +[providers.venice.models."kimi-k3".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."kimi-k3".features] +tools = true +vision = true +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."kimi-k3".costs] +input_cost_per_mtok = 3.75 +output_cost_per_mtok = 18.75 +cache_input_cost_per_mtok = 0.375 + +[providers.venice.models."kimi-k3-fast"] +api_id = "kimi-k3-fast-api" +display_name = "Kimi K3 Fast" +family = "kimi-k3" +agent_profile = "kimi" +aliases = ["kimi-fast"] + +[providers.venice.models."kimi-k3-fast".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."kimi-k3-fast".features] +tools = true +vision = true +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."kimi-k3-fast".costs] +input_cost_per_mtok = 4.5 +output_cost_per_mtok = 22.5 +cache_input_cost_per_mtok = 0.45 + +[providers.venice.models."grok-4.6"] +api_id = "grok-4-6" +display_name = "Grok 4.6" +family = "grok-4" +aliases = ["grok", "grok46", "grok-46"] + +[providers.venice.models."grok-4.6".limits] +context_window = 500000 +max_output = 32000 + +[providers.venice.models."grok-4.6".features] +tools = true +vision = true +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true + +[providers.venice.models."grok-4.6".controls] +reasoning_effort = ["low", "medium", "high", "xhigh"] + +[providers.venice.models."grok-4.6".costs] +input_cost_per_mtok = 2.27 +output_cost_per_mtok = 6.8 +cache_input_cost_per_mtok = 0.57 + +[providers.venice.models."glm-5.3"] +api_id = "z-ai-glm-5-3" +display_name = "GLM 5.3" +family = "glm-5" +aliases = ["glm", "glm5", "glm53", "glm5.3", "glm-5-3"] + +[providers.venice.models."glm-5.3".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."glm-5.3".features] +tools = true +vision = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true + +[providers.venice.models."glm-5.3".controls] +reasoning_effort = ["low", "high", "max"] + +[providers.venice.models."glm-5.3".costs] +input_cost_per_mtok = 1.75 +output_cost_per_mtok = 5.5 +cache_input_cost_per_mtok = 0.325 + +[providers.venice.models."deepseek-v4-flash"] +api_id = "deepseek-v4-flash-0731" +display_name = "DeepSeek V4 Flash" +family = "deepseek-v4" +agent_profile = "openai" default = true -aliases = ["venice-uncensored", "vu"] +aliases = ["deepseek-v4", "deepseek", "deepseek-flash"] -[providers.venice.models."venice-uncensored-1-2".limits] -context_window = 128000 -max_output = 8192 +[providers.venice.models."deepseek-v4-flash".limits] +context_window = 1000000 +max_output = 32768 -[providers.venice.models."venice-uncensored-1-2".features] +[providers.venice.models."deepseek-v4-flash".features] +tools = true +vision = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."deepseek-v4-flash".controls] +reasoning_effort = ["low", "high", "max"] + +[providers.venice.models."deepseek-v4-flash".costs] +input_cost_per_mtok = 0.175 +output_cost_per_mtok = 0.35 +cache_input_cost_per_mtok = 0.035 + +[providers.venice.models."deepseek-v4-pro"] +api_id = "deepseek-v4-pro-0813" +display_name = "DeepSeek V4 Pro" +family = "deepseek-v4" +agent_profile = "openai" +aliases = ["deepseek-pro"] + +[providers.venice.models."deepseek-v4-pro".limits] +context_window = 1000000 +max_output = 32768 + +[providers.venice.models."deepseek-v4-pro".features] +tools = true +vision = false +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."deepseek-v4-pro".costs] +input_cost_per_mtok = 1.65 +output_cost_per_mtok = 4.95 +cache_input_cost_per_mtok = 0.165 + +[providers.venice.models."qwen3.8-max"] +api_id = "qwen-3-8-max" +display_name = "Qwen 3.8 Max" +family = "qwen3" +aliases = ["qwen", "qwen-max", "qwen3.8", "qwen-3.8", "qwen38", "qwen-3.8-max", "qwen38-max"] + +[providers.venice.models."qwen3.8-max".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."qwen3.8-max".features] tools = true vision = true -reasoning = false +reasoning = true +reasoning_by_default = true +prompt_cache = true -[providers.venice.models."venice-uncensored-1-2".costs] -input_cost_per_mtok = 0.2 -output_cost_per_mtok = 0.9 +[providers.venice.models."qwen3.8-max".costs] +input_cost_per_mtok = 2.5 +output_cost_per_mtok = 7.5 +cache_input_cost_per_mtok = 0.3125 -[providers.venice.models."venice-uncensored-role-play"] -display_name = "Venice Uncensored Role Play" -family = "venice-uncensored" -aliases = ["venice-roleplay", "vrp"] +[providers.venice.models."qwen3.8-27b"] +api_id = "qwen-3-8-27b" +display_name = "Qwen 3.8 27B" +family = "qwen3.8" +aliases = ["qwen-27b", "qwen-3.8-27b", "qwen38-27b"] -[providers.venice.models."venice-uncensored-role-play".limits] -context_window = 128000 -max_output = 4096 +[providers.venice.models."qwen3.8-27b".limits] +context_window = 262144 +max_output = 131072 -[providers.venice.models."venice-uncensored-role-play".features] +[providers.venice.models."qwen3.8-27b".features] tools = true vision = true -reasoning = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true -[providers.venice.models."venice-uncensored-role-play".costs] -input_cost_per_mtok = 0.5 -output_cost_per_mtok = 2.0 +[providers.venice.models."qwen3.8-27b".controls] +reasoning_effort = ["low", "medium", "xhigh"] + +[providers.venice.models."qwen3.8-27b".costs] +input_cost_per_mtok = 0.45 +output_cost_per_mtok = 3.2 diff --git a/lib/foundation/fabro-types/src/repository.rs b/lib/foundation/fabro-types/src/repository.rs index 778b3ebf0..fcddc9bea 100644 --- a/lib/foundation/fabro-types/src/repository.rs +++ b/lib/foundation/fabro-types/src/repository.rs @@ -1,3 +1,8 @@ +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::str::FromStr; + use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -25,8 +30,10 @@ impl RepositoryRef { /// /// Construction enforces GitHub's owner and repository name syntax on the /// exact submitted bytes; no trimming, case folding, or other normalization -/// is performed. -#[derive(Debug, Clone, PartialEq, Eq)] +/// is performed. The original spelling is preserved for `Display` and +/// serialization, while identity (`Eq`, `Ord`, `Hash`) is case-insensitive +/// to match GitHub's treatment of owner and repository names. +#[derive(Debug, Clone)] pub struct GitHubRepositorySlug { owner: String, repo: String, @@ -57,6 +64,105 @@ impl GitHubRepositorySlug { pub fn repo(&self) -> &str { &self.repo } + + /// Canonical credential-free HTTPS URL for this GitHub repository. + #[must_use] + pub fn https_url(&self) -> String { + format!("https://github.com/{self}") + } + + /// Whether `other` names the same repository owner, ignoring ASCII case. + /// Owner and repository names are validated ASCII, so ASCII folding is + /// exact. + #[must_use] + pub fn same_owner(&self, other: &Self) -> bool { + self.owner.eq_ignore_ascii_case(&other.owner) + } +} + +/// Case-folded bytes for identity comparisons without allocating; owner and +/// repository names are validated ASCII, so ASCII folding is exact. +fn folded_bytes(value: &str) -> impl Iterator + '_ { + value.bytes().map(|byte| byte.to_ascii_lowercase()) +} + +impl PartialEq for GitHubRepositorySlug { + fn eq(&self, other: &Self) -> bool { + self.owner.eq_ignore_ascii_case(&other.owner) && self.repo.eq_ignore_ascii_case(&other.repo) + } +} + +impl Eq for GitHubRepositorySlug {} + +impl PartialOrd for GitHubRepositorySlug { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for GitHubRepositorySlug { + fn cmp(&self, other: &Self) -> Ordering { + folded_bytes(&self.owner) + .cmp(folded_bytes(&other.owner)) + .then_with(|| folded_bytes(&self.repo).cmp(folded_bytes(&other.repo))) + } +} + +impl Hash for GitHubRepositorySlug { + fn hash(&self, state: &mut H) { + for byte in folded_bytes(&self.owner) { + state.write_u8(byte); + } + // `/` cannot appear in a validated owner, so the folded + // `owner/repo` encoding stays unambiguous. + state.write_u8(b'/'); + for byte in folded_bytes(&self.repo) { + state.write_u8(byte); + } + } +} + +impl fmt::Display for GitHubRepositorySlug { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.owner, self.repo) + } +} + +/// Parse failure for [`GitHubRepositorySlug`]. The offending input is not +/// echoed back because config surfaces already attach the value and its path. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error( + "expected a GitHub `owner/repository` slug with no scheme, host, ref, or extra path component" +)] +pub struct GitHubRepositorySlugError; + +impl FromStr for GitHubRepositorySlug { + type Err = GitHubRepositorySlugError; + + fn from_str(value: &str) -> Result { + Self::try_new(value).ok_or(GitHubRepositorySlugError) + } +} + +impl Serialize for GitHubRepositorySlug { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for GitHubRepositorySlug { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error as _; + + let value = String::deserialize(deserializer)?; + value.parse().map_err(D::Error::custom) + } } fn valid_github_owner(value: &str) -> bool { @@ -234,6 +340,77 @@ mod tests { assert!(GitHubRepositorySlug::try_new(&over_repo).is_none()); } + #[test] + fn slug_identity_is_case_insensitive_but_display_preserves_case() { + let mixed: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + let lower: GitHubRepositorySlug = "fabro-sh/keystone".parse().unwrap(); + + assert_eq!(mixed, lower); + assert_eq!(mixed.cmp(&lower), std::cmp::Ordering::Equal); + assert!(mixed.same_owner(&lower)); + assert_eq!(mixed.to_string(), "Fabro-SH/Keystone"); + + let mut hashes = std::collections::HashSet::new(); + hashes.insert(mixed.clone()); + assert!( + !hashes.insert(lower.clone()), + "case variants share identity" + ); + + let mut ordered = std::collections::BTreeSet::new(); + ordered.insert(mixed); + assert!(!ordered.insert(lower), "case variants share ordering"); + } + + #[test] + fn slug_https_url_preserves_spelling_and_has_no_credentials() { + let slug: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + + assert_eq!(slug.https_url(), "https://github.com/Fabro-SH/Keystone"); + assert!(!slug.https_url().contains('@')); + } + + #[test] + fn slug_ordering_sorts_by_canonical_form() { + let mut slugs: Vec = ["owner/Zeta", "Owner/alpha", "owner/Beta"] + .iter() + .map(|value| value.parse().unwrap()) + .collect(); + slugs.sort(); + let rendered: Vec = slugs.iter().map(ToString::to_string).collect(); + assert_eq!(rendered, ["Owner/alpha", "owner/Beta", "owner/Zeta"]); + } + + #[test] + fn slug_from_str_rejects_urls_and_hosts() { + let cases = [ + "https://github.com/owner/repo", + "git@github.com:owner/repo.git", + "ssh://git@github.com/owner/repo", + "github.com/owner/repo", + "owner/repo@main", + "owner/repo#ref", + " owner/repo", + "owner/repo ", + ]; + for input in cases { + assert!(input.parse::().is_err(), "{input}"); + } + } + + #[test] + fn slug_serde_round_trips_as_a_string() { + let slug: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + let json = serde_json::to_string(&slug).unwrap(); + assert_eq!(json, "\"Fabro-SH/Keystone\""); + + let parsed: GitHubRepositorySlug = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, slug); + + let err = serde_json::from_str::("\"not a slug\"").unwrap_err(); + assert!(err.to_string().contains("owner/repository"), "{err}"); + } + #[test] fn valid_ref_selectors_are_accepted() { let max = "a".repeat(255); diff --git a/lib/foundation/fabro-types/src/run.rs b/lib/foundation/fabro-types/src/run.rs index 86f1910d7..34e56ffaf 100644 --- a/lib/foundation/fabro-types/src/run.rs +++ b/lib/foundation/fabro-types/src/run.rs @@ -8,6 +8,7 @@ use crate::graph::Graph; use crate::principal::Principal; use crate::run_id::RunId; use crate::run_summary::AutomationRef; +use crate::workflow_version_id::WorkflowVersionId; #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct RunServerProvenance { @@ -58,33 +59,35 @@ pub struct ForkSourceRef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RunSpec { - pub run_id: RunId, - pub settings: WorkflowSettings, - pub graph: Graph, + pub run_id: RunId, + pub settings: WorkflowSettings, + pub graph: Graph, #[serde(default, skip_serializing_if = "Option::is_none")] - pub graph_source: Option, + pub graph_source: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_slug: Option, + pub workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub automation: Option, + pub workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_directory: Option, + pub automation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_directory: Option, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub labels: HashMap, - pub provenance: RunProvenance, + pub labels: HashMap, + pub provenance: RunProvenance, #[serde(default, skip_serializing_if = "Option::is_none")] - pub manifest_blob: Option, + pub manifest_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub definition_blob: Option, + pub definition_blob: Option, /// Unredacted copy of this spec in the blob store. Stored events pass /// through secret redaction, so the spec folded from them is display /// data; execution must load the spec from this blob. #[serde(default, skip_serializing_if = "Option::is_none")] - pub spec_blob: Option, + pub spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub git: Option, + pub git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub fork_source_ref: Option, + pub fork_source_ref: Option, } impl RunSpec { diff --git a/lib/foundation/fabro-types/src/run_event/run.rs b/lib/foundation/fabro-types/src/run_event/run.rs index c7a17d990..8ef5a361e 100644 --- a/lib/foundation/fabro-types/src/run_event/run.rs +++ b/lib/foundation/fabro-types/src/run_event/run.rs @@ -7,42 +7,45 @@ use crate::status::{BlockedReason, PendingReason, SuccessReason}; use crate::{ AutomationRef, BlobHash, DiffSummary, ForkSourceRef, GitContext, Graph, PairId, PairTarget, RunControlAction, RunFailure, RunId, RunProvenance, RunTiming, WorkflowSettings, + WorkflowVersionId, }; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCreatedProps { #[serde(default, skip_serializing_if = "Option::is_none")] - pub title: Option, - pub settings: WorkflowSettings, - pub graph: Graph, + pub title: Option, + pub settings: WorkflowSettings, + pub graph: Graph, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_source: Option, + pub workflow_source: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, + pub labels: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_directory: Option, + pub source_directory: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_slug: Option, + pub workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub automation: Option, - pub provenance: RunProvenance, + pub workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub manifest_blob: Option, + pub automation: Option, + pub provenance: RunProvenance, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest_blob: Option, /// Unredacted copy of the run spec in the blob store. The settings and /// graph on this event are redacted at the sink; execution loads the /// spec from this blob instead. #[serde(default, skip_serializing_if = "Option::is_none")] - pub spec_blob: Option, + pub spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub git: Option, + pub git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub fork_source_ref: Option, + pub fork_source_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub retried_from: Option, + pub retried_from: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_id: Option, + pub parent_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub web_url: Option, + pub web_url: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index 8e589a57a..6d2b3ebeb 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -6,7 +6,7 @@ //! notifications, interviews, agent knobs, hooks, SCM targeting, pull-request //! behavior, and artifact collection. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; use std::time::Duration as StdDuration; @@ -603,9 +603,19 @@ pub struct RunIntegrationsSettings { /// presence-vs-clear distinction is only meaningful at the layer-merge /// stage; the resolved form collapses both `None` and `Some({})` into an /// empty map. +/// +/// `additional_repositories` lists repositories, beyond the implicit run +/// origin, that the minted `GITHUB_TOKEN` must cover. Configuration +/// resolution guarantees a non-empty set comes with a non-empty permission +/// map that includes `contents`; runs persisted before the field existed +/// deserialize to an empty set via the serde default. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct RunIntegrationsGithubSettings { - pub permissions: HashMap, + pub permissions: HashMap, + /// Omitted when empty so settings serialized by this release stay + /// byte-identical to earlier releases for single-repository runs. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub additional_repositories: BTreeSet, } impl RunIntegrationsGithubSettings { @@ -616,17 +626,68 @@ impl RunIntegrationsGithubSettings { !self.permissions.is_empty() } + /// Whether the run declares additional repositories beyond the origin. + pub fn has_additional_repositories(&self) -> bool { + !self.additional_repositories.is_empty() + } + + /// Whether a resolved `contents` permission level lets the token reach + /// repository contents — the level a declared `additional_repositories` + /// set requires. The one definition shared by config-time validation and + /// the runtime re-check after interpolation, so the accepted levels + /// cannot drift between the two layers. + #[must_use] + pub fn contents_permission_allows_repository_access(value: &str) -> bool { + value == "read" || value == "write" + } + /// Resolve every `permissions` value. `{{ vars.* }}` is substituted /// server-side at run creation, so values are literal by this point; a /// still-unresolved token fails closed rather than reaching the GitHub API /// as literal text. - pub fn resolve_permissions(&self) -> Result, ResolveError> { + fn resolve_permissions(&self) -> Result, ResolveError> { let mut ctx = ResolveCtx::new(); self.permissions .iter() .map(|(name, value)| Ok((name.clone(), value.resolve_with(&mut ctx)?))) .collect() } + + /// Resolve the whole runtime integration request: interpolated + /// permissions plus the declared additional repositories, produced + /// together so consumers cannot pick up one without the other. + pub fn resolve_integration(&self) -> Result { + Ok(ResolvedGithubIntegration { + permissions: self.resolve_permissions()?, + additional_repositories: self.additional_repositories.clone(), + }) + } +} + +/// The resolved runtime GitHub integration request for one run: interpolated +/// permission values plus the declared additional repositories. +/// +/// This is the single value carried from run materialization into workflow +/// startup, replacing parallel permission/repository collections that could +/// drift apart. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ResolvedGithubIntegration { + pub permissions: HashMap, + pub additional_repositories: BTreeSet, +} + +impl ResolvedGithubIntegration { + /// Mirrors [`RunIntegrationsGithubSettings::is_token_requested`] for the + /// resolved form. + #[must_use] + pub fn is_token_requested(&self) -> bool { + !self.permissions.is_empty() + } + + #[must_use] + pub fn has_additional_repositories(&self) -> bool { + !self.additional_repositories.is_empty() + } } #[cfg(test)] @@ -635,10 +696,11 @@ mod run_integrations_github_tests { fn settings(permissions: &[(&str, &str)]) -> RunIntegrationsGithubSettings { RunIntegrationsGithubSettings { - permissions: permissions + permissions: permissions .iter() .map(|(k, v)| ((*k).to_string(), InterpString::parse(v))) .collect(), + additional_repositories: std::collections::BTreeSet::new(), } } @@ -670,6 +732,53 @@ mod run_integrations_github_tests { fn resolve_permissions_is_empty_for_empty_settings() { assert!(settings(&[]).resolve_permissions().unwrap().is_empty()); } + + #[test] + fn settings_without_additional_repositories_field_deserialize_to_empty_set() { + // Persisted run.created events from releases before + // `additional_repositories` existed omit the field entirely. + let parsed: RunIntegrationsGithubSettings = serde_json::from_value(serde_json::json!({ + "permissions": { "contents": "read" } + })) + .expect("legacy settings should deserialize"); + + assert!(parsed.additional_repositories.is_empty()); + assert!(!parsed.has_additional_repositories()); + } + + #[test] + fn resolve_integration_carries_permissions_and_repositories_together() { + let mut s = settings(&[("contents", "read")]); + s.additional_repositories + .insert("fabro-sh/keystone".parse().unwrap()); + + let resolved = s.resolve_integration().unwrap(); + + assert!(resolved.is_token_requested()); + assert!(resolved.has_additional_repositories()); + assert_eq!( + resolved.permissions.get("contents"), + Some(&"read".to_string()) + ); + assert_eq!( + resolved + .additional_repositories + .iter() + .map(ToString::to_string) + .collect::>(), + vec!["fabro-sh/keystone"] + ); + } + + #[test] + fn resolve_integration_fails_on_an_unresolved_permission_token() { + let mut s = settings(&[("contents", "{{ env.GH_PERM_LEVEL }}")]); + s.additional_repositories + .insert("fabro-sh/keystone".parse().unwrap()); + + let err = s.resolve_integration().unwrap_err(); + assert_eq!(err.namespace, Namespace::Env); + } } /// The resolved source of a run goal. @@ -961,14 +1070,32 @@ impl Default for RunCheckpointSettings { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCloneSettings { pub enabled: bool, + #[serde(default = "default_clone_depth")] + pub depth: i32, +} + +impl RunCloneSettings { + pub const DEFAULT_DEPTH: i32 = 100; + + /// Git history depth to fetch, or `None` to fetch full history. + pub fn depth_limit(&self) -> Option { + (self.depth > 0).then_some(self.depth) + } } impl Default for RunCloneSettings { fn default() -> Self { - Self { enabled: true } + Self { + enabled: true, + depth: Self::DEFAULT_DEPTH, + } } } +fn default_clone_depth() -> i32 { + RunCloneSettings::DEFAULT_DEPTH +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunBranchSettings { pub enabled: bool, diff --git a/lib/foundation/fabro-types/src/test_support.rs b/lib/foundation/fabro-types/src/test_support.rs index 41af963f1..c5065cb76 100644 --- a/lib/foundation/fabro-types/src/test_support.rs +++ b/lib/foundation/fabro-types/src/test_support.rs @@ -1,7 +1,8 @@ use std::collections::HashMap; use crate::{ - AuthMethod, Graph, IdpIdentity, Principal, RunProvenance, RunSpec, WorkflowSettings, fixtures, + AuthMethod, BlobHash, Graph, IdpIdentity, Principal, RunProvenance, RunSpec, WorkflowSettings, + WorkflowVersionId, fixtures, }; #[must_use] @@ -38,19 +39,25 @@ pub fn test_run_provenance() -> RunProvenance { #[must_use] pub fn test_run_spec() -> RunSpec { RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, } } + +#[must_use] +pub fn test_workflow_version_id() -> WorkflowVersionId { + WorkflowVersionId::from(BlobHash::new(b"workflow")) +} diff --git a/lib/foundation/fabro-types/tests/run_event_serde.rs b/lib/foundation/fabro-types/tests/run_event_serde.rs index f287a6acc..2645b9ca4 100644 --- a/lib/foundation/fabro-types/tests/run_event_serde.rs +++ b/lib/foundation/fabro-types/tests/run_event_serde.rs @@ -6,7 +6,7 @@ use fabro_types::run_event::run::{RunCreatedProps, RunParentLinkedProps, RunPare use fabro_types::run_event::{RunSessionTurnFailedCode, RunSessionTurnFailedProps}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; -use fabro_types::test_support::test_run_provenance; +use fabro_types::test_support::{test_run_provenance, test_workflow_version_id}; use fabro_types::{AutomationRef, EventBody, TurnId, WorkflowSettings, fixtures}; fn templated_settings() -> WorkflowSettings { @@ -18,34 +18,37 @@ fn templated_settings() -> WorkflowSettings { #[test] fn run_created_props_round_trip_templated_settings() { let props = RunCreatedProps { - title: Some("Ship task".to_string()), - settings: templated_settings(), - graph: Graph::new("ship"), - workflow_source: Some("digraph Ship { start -> exit }".to_string()), - labels: BTreeMap::from([("team".to_string(), "platform".to_string())]), - source_directory: Some("/Users/client/project".to_string()), - workflow_slug: Some("demo".to_string()), - automation: Some(AutomationRef { + title: Some("Ship task".to_string()), + settings: templated_settings(), + graph: Graph::new("ship"), + workflow_source: Some("digraph Ship { start -> exit }".to_string()), + labels: BTreeMap::from([("team".to_string(), "platform".to_string())]), + source_directory: Some("/Users/client/project".to_string()), + workflow_slug: Some("demo".to_string()), + workflow_version_id: Some(test_workflow_version_id()), + automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }), - provenance: test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: Some(GitContext { + provenance: test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: None, dirty: DirtyStatus::Unknown, }), - fork_source_ref: Some(ForkSourceRef { + fork_source_ref: Some(ForkSourceRef { source_run_id: fixtures::RUN_2, checkpoint_sha: "def456".to_string(), }), - retried_from: Some(fixtures::RUN_1), - parent_id: Some(fixtures::RUN_2), - web_url: Some("http://localhost:3000/runs/01JNQVR7M0EJ5GKAT2SC4ERS1Z".to_string()), + retried_from: Some(fixtures::RUN_1), + parent_id: Some(fixtures::RUN_2), + web_url: Some( + "http://localhost:3000/runs/01JNQVR7M0EJ5GKAT2SC4ERS1Z".to_string(), + ), }; let json = serde_json::to_value(&props).expect("props should serialize"); @@ -67,6 +70,10 @@ fn run_created_props_round_trip_templated_settings() { assert_eq!(json["parent_id"], fixtures::RUN_2.to_string()); assert_eq!(json["automation"]["id"], "nightly"); assert_eq!(json["automation"]["trigger_id"], "schedule_1"); + assert_eq!( + json["workflow_version_id"], + test_workflow_version_id().to_string() + ); let round_trip: RunCreatedProps = serde_json::from_value(json.clone()).expect("props should deserialize"); @@ -84,22 +91,23 @@ fn run_created_props_round_trip_templated_settings() { #[test] fn run_created_props_omits_web_url_when_absent() { let props = RunCreatedProps { - title: None, - settings: WorkflowSettings::default(), - graph: Graph::new("ship"), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + title: None, + settings: WorkflowSettings::default(), + graph: Graph::new("ship"), + workflow_source: None, + labels: BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }; let json = serde_json::to_value(&props).expect("props should serialize"); @@ -115,6 +123,10 @@ fn run_created_props_omits_web_url_when_absent() { json.get("retried_from").is_none(), "retried_from must be omitted when None, got {json}" ); + assert!( + json.get("workflow_version_id").is_none(), + "workflow_version_id must be omitted when None, got {json}" + ); let round_trip: RunCreatedProps = serde_json::from_value(json.clone()).expect("props should deserialize"); @@ -137,6 +149,7 @@ fn run_created_props_defaults_additive_fields_for_legacy_events() { serde_json::from_value(json).expect("legacy props should deserialize"); assert_eq!(props.retried_from, None); assert_eq!(props.automation, None); + assert_eq!(props.workflow_version_id, None); } #[test] diff --git a/lib/foundation/fabro-types/tests/run_spec_serde.rs b/lib/foundation/fabro-types/tests/run_spec_serde.rs index ec7a7ef40..49432940c 100644 --- a/lib/foundation/fabro-types/tests/run_spec_serde.rs +++ b/lib/foundation/fabro-types/tests/run_spec_serde.rs @@ -4,7 +4,7 @@ use fabro_types::graph::Graph; use fabro_types::run::{DirtyStatus, ForkSourceRef, GitContext, RunSpec}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; -use fabro_types::test_support::test_run_provenance; +use fabro_types::test_support::{test_run_provenance, test_workflow_version_id}; use fabro_types::{AutomationRef, WorkflowSettings, fixtures}; fn templated_settings() -> WorkflowSettings { @@ -16,29 +16,30 @@ fn templated_settings() -> WorkflowSettings { #[test] fn run_spec_round_trips_templated_settings() { let record = RunSpec { - run_id: fixtures::RUN_1, - settings: templated_settings(), - graph: Graph::new("ship"), - graph_source: None, - workflow_slug: Some("demo".to_string()), - automation: Some(AutomationRef { + run_id: fixtures::RUN_1, + settings: templated_settings(), + graph: Graph::new("ship"), + graph_source: None, + workflow_slug: Some("demo".to_string()), + workflow_version_id: Some(test_workflow_version_id()), + automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }), - source_directory: Some("/Users/client/project".to_string()), - labels: HashMap::from([("team".to_string(), "platform".to_string())]), - provenance: test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: Some(GitContext { + source_directory: Some("/Users/client/project".to_string()), + labels: HashMap::from([("team".to_string(), "platform".to_string())]), + provenance: test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: Some("abc123".to_string()), dirty: DirtyStatus::Clean, }), - fork_source_ref: Some(ForkSourceRef { + fork_source_ref: Some(ForkSourceRef { source_run_id: fixtures::RUN_2, checkpoint_sha: "def456".to_string(), }), @@ -59,6 +60,10 @@ fn run_spec_round_trips_templated_settings() { assert_eq!(json["fork_source_ref"]["checkpoint_sha"], "def456"); assert_eq!(json["automation"]["id"], "nightly"); assert_eq!(json["automation"]["trigger_id"], "schedule_1"); + assert_eq!( + json["workflow_version_id"], + test_workflow_version_id().to_string() + ); let round_trip: RunSpec = serde_json::from_value(json.clone()).expect("record should deserialize"); @@ -85,4 +90,8 @@ fn run_spec_defaults_automation_for_legacy_specs() { let record: RunSpec = serde_json::from_value(json).expect("legacy spec should deserialize"); assert_eq!(record.automation, None); + assert_eq!(record.workflow_version_id, None); + + let round_trip = serde_json::to_value(&record).expect("record should serialize"); + assert!(round_trip.get("workflow_version_id").is_none()); } diff --git a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts index 7258d9d8d..79791f502 100644 --- a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts @@ -16,4 +16,8 @@ export interface RunCloneSettings { 'enabled': boolean; + /** + * Git history depth. Set to 0 to clone full history. + */ + 'depth'?: number; } diff --git a/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts b/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts index a28ae5b4b..df9e432ba 100644 --- a/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts @@ -16,4 +16,8 @@ export interface RunIntegrationsGithubSettings { 'permissions': { [key: string]: string; }; + /** + * Additional GitHub repositories, beyond the implicit run origin, that the minted GITHUB_TOKEN must cover. Each entry is a full `owner/repository` slug; every repository must share one owner with the run origin. Omitted when empty; settings persisted before this field existed deserialize to an empty set. + */ + 'additional_repositories'?: Array; } diff --git a/lib/packages/fabro-api-client/src/models/run-spec.ts b/lib/packages/fabro-api-client/src/models/run-spec.ts index e83be609b..a9e41bec6 100644 --- a/lib/packages/fabro-api-client/src/models/run-spec.ts +++ b/lib/packages/fabro-api-client/src/models/run-spec.ts @@ -38,6 +38,10 @@ export interface RunSpec { 'graph': { [key: string]: any; }; 'graph_source'?: string | null; 'workflow_slug'?: string | null; + /** + * SHA-256 identity of validated canonical workflow-version bytes. Hex input is case-insensitive; Fabro emits the canonical lowercase form. + */ + 'workflow_version_id'?: string | null; 'automation'?: AutomationRef | null; 'source_directory'?: string | null; 'labels'?: { [key: string]: string; };