diff --git a/.github/workflows/internal-deps.yml b/.github/workflows/internal-deps.yml new file mode 100644 index 000000000..6d5c55066 --- /dev/null +++ b/.github/workflows/internal-deps.yml @@ -0,0 +1,331 @@ +name: Internal dependencies + +# Every internal Git dependency names `branch = "main"`, and Cargo.lock picks +# the commit CI builds and Fabro ships. Each night this job moves the lock to +# the current main of each internal library with `cargo update -p`, then runs +# the Linux test suite from rust.yml against it, so drift is noticed without +# anyone asking. A passing lock goes to one pull request from +# `bot/internal-deps`, opened or updated here and never merged here. A failure +# opens one tracking issue labeled `internal-deps`, or comments on the open +# one; the next passing run closes it. Nothing is pushed to main. + +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +# Never cancel a run midway: the report and pull request jobs must see how +# the check ended. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: {} + +env: + CARGO_TERM_COLOR: always + CARGO_NET_RETRY: "10" + +jobs: + check: + name: Test (Linux) + runs-on: ubuntu-24.04-x86-32-cores + timeout-minutes: 60 + permissions: + contents: read + outputs: + changed: ${{ steps.update.outputs.changed }} + summary: ${{ steps.update.outputs.summary }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable + with: + toolchain: 1.97.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + cache-on-failure: true + - uses: taiki-e/install-action@773334c0e05d7e699e4d78234494308223f3a2cf # nextest + # One package per internal repository: updating one package from a Git + # repository moves every package from that repository. daytona-sdk comes + # through sandbox-driver, but its repository is separate, so it moves on + # its own. The summary names each repository whose locked commit moved, + # for the job summary, the pull request, and the tracking issue. + - name: Update internal dependencies + id: update + env: + INTERNAL_CRATES: sandbox-driver pebble-agent petri-runtime lithos-llm twin-openai daytona-sdk + run: | + set -euo pipefail + cp Cargo.lock "$RUNNER_TEMP/Cargo.lock.before" + args=() + for crate in $INTERNAL_CRATES; do + args+=(-p "$crate") + done + cargo update "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/cargo-update.log" + + python3 - "$RUNNER_TEMP/Cargo.lock.before" Cargo.lock > "$RUNNER_TEMP/summary.md" <<'PY' + import re + import sys + import tomllib + + SOURCE = re.compile(r"git\+https://github\.com/([^?#]+?)(?:\.git)?\?[^#]*#([0-9a-f]+)$") + + def commits(path): + found = {} + with open(path, "rb") as lock: + for package in tomllib.load(lock).get("package", []): + match = SOURCE.match(package.get("source", "")) + if match: + found.setdefault(match[1], set()).add(match[2]) + return found + + before, after = commits(sys.argv[1]), commits(sys.argv[2]) + rows = [] + for repo in sorted(set(before) | set(after)): + old, new = sorted(before.get(repo, ())), sorted(after.get(repo, ())) + if old == new: + continue + if len(old) == 1 and len(new) == 1: + moved = f"[`{old[0][:7]}...{new[0][:7]}`](https://github.com/{repo}/compare/{old[0]}...{new[0]})" + else: + moved = " ".join(f"`{c[:7]}`" for c in old) + " -> " + " ".join(f"`{c[:7]}`" for c in new) + rows.append(f"| `{repo}` | {moved} |") + if rows: + print("| Repository | Commits |\n|---|---|") + print("\n".join(rows)) + else: + print("No internal commit moved.") + PY + { + echo + echo "
cargo update output" + echo + echo '```' + cat "$RUNNER_TEMP/cargo-update.log" + echo '```' + echo + echo "
" + } >> "$RUNNER_TEMP/summary.md" + cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY" + + # Only a moved internal commit counts: `cargo update` can also + # rewrite unrelated entries of a lock that `--locked` accepts. + if grep -q '^| `' "$RUNNER_TEMP/summary.md"; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + git checkout -- Cargo.lock + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Cargo.lock already locks every internal main; nothing to do." + fi + delimiter="SUMMARY_$(openssl rand -hex 16)" + { + echo "summary<<$delimiter" + cat "$RUNNER_TEMP/summary.md" + echo "$delimiter" + } >> "$GITHUB_OUTPUT" + + # The rest is rust.yml's Test (Linux) job on the updated lock. + # Every Petri run takes its scope through a sandbox-driver plugin + # executable that Petri finds on PATH: `sandbox-driver-host` for the + # `local` provider, `sandbox-driver-docker` for `docker`. Installed + # at the commit the updated Cargo.lock resolves sandbox-driver to, so + # the plugins and the in-process driver are one build. + - name: Read the sandbox-driver commit Cargo.lock resolves + if: steps.update.outputs.changed == 'true' + id: sandbox-driver + run: | + rev="$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "sandbox-driver") | .source' | sed 's/.*#//' | sort -u)" + [[ "$rev" =~ ^[0-9a-f]{40}$ ]] + echo "rev=$rev" >> "$GITHUB_OUTPUT" + - name: Restore the sandbox-driver plugin executables + if: steps.update.outputs.changed == 'true' + id: sandbox-driver-cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/bin/sandbox-driver-host + ~/.cargo/bin/sandbox-driver-docker + key: sandbox-driver-plugins-${{ runner.os }}-${{ steps.sandbox-driver.outputs.rev }} + - name: Install the sandbox-driver plugin executables + if: steps.update.outputs.changed == 'true' && steps.sandbox-driver-cache.outputs.cache-hit != 'true' + env: + SANDBOX_DRIVER_REV: ${{ steps.sandbox-driver.outputs.rev }} + run: cargo install --locked --git https://github.com/lithoscomputer/sandbox-driver --rev "$SANDBOX_DRIVER_REV" sandbox-driver-host sandbox-driver-docker + # The images the suite's Docker tests run; see rust.yml. + - name: Pull the images the Docker tests run + if: steps.update.outputs.changed == 'true' + run: | + backend="$(dirname "$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "petri-executor-sandbox") | .manifest_path')")/src/backend.rs" + pin="$(sed -n 's/^const RUNNER_PIN: &str = "\([0-9a-f]*\)";$/\1/p' "$backend")" + test -n "$pin" + docker pull "ghcr.io/lithoscomputer/ubuntu-24.04:slim-$pin" + docker pull ghcr.io/lithoscomputer/ubuntu-22.04:slim + - name: Test + if: steps.update.outputs.changed == 'true' + run: cargo nextest run --locked --workspace --status-level slow --profile ci + # The pull request job commits exactly the lock that passed. + - name: Keep the tested lock + if: steps.update.outputs.changed == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: internal-deps-lock + path: Cargo.lock + if-no-files-found: error + retention-days: 7 + + pull-request: + name: Open the update pull request + needs: check + if: needs.check.outputs.changed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + # The release App's credentials live in this environment. A pull request + # opened with the App's token, unlike one opened with GITHUB_TOKEN, + # triggers rust.yml on it. + environment: nightly + permissions: + contents: read # check out the tested commit; writes go through the App token + steps: + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ vars.FABRO_RELEASES_APP_CLIENT_ID }} + private-key: ${{ secrets.FABRO_RELEASES_APP_PRIVATE_KEY }} + # Narrowed to what this job does: push the branch, open the PR. + permission-contents: write + permission-pull-requests: write + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 # the branch merges the tested commit into its history + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: internal-deps-lock + path: ${{ runner.temp }}/internal-deps-lock + + # The branch is only ever added to, never rewritten: an open pull + # request's branch merges the tested commit and takes the tested lock; + # with no open pull request, a leftover branch is deleted and a new one + # starts from the tested commit. The push names the branch explicitly, + # so it can never reach main. + - name: Push the lock and open or update the pull request + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BRANCH: bot/internal-deps + SUMMARY: ${{ needs.check.outputs.summary }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + lock="$RUNNER_TEMP/internal-deps-lock/Cargo.lock" + repo_api="repos/$GITHUB_REPOSITORY" + remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config user.name "fabro-releases[bot]" + git config user.email "fabro-releases[bot]@users.noreply.github.com" + + pr="$(gh api "$repo_api/pulls?head=$GITHUB_REPOSITORY_OWNER:$BRANCH&base=main&state=open" --jq '.[0].number // empty')" + if [ -n "$pr" ]; then + git fetch --no-tags "$remote" "refs/heads/$BRANCH" + git checkout -B "$BRANCH" FETCH_HEAD + # Cargo.lock is the only file the branch changes, so it is the + # only possible conflict, and the tested lock resolves it. + git merge --no-ff --no-commit "$GITHUB_SHA" || true + cp "$lock" Cargo.lock + git add Cargo.lock + if [ -n "$(git diff --name-only --diff-filter=U)" ]; then + echo "::error::$BRANCH conflicts with main outside Cargo.lock; close its pull request and rerun." + exit 1 + fi + else + if git ls-remote --exit-code --heads "$remote" "$BRANCH" > /dev/null; then + git push "$remote" --delete "$BRANCH" + fi + git checkout -B "$BRANCH" "$GITHUB_SHA" + cp "$lock" Cargo.lock + git add Cargo.lock + fi + if git rev-parse -q --verify MERGE_HEAD > /dev/null || ! git diff --cached --quiet; then + git commit -m "Update internal dependencies to their current main" + git push "$remote" "HEAD:refs/heads/$BRANCH" + else + echo "$BRANCH already carries this lock." + fi + + body="$RUNNER_TEMP/body.md" + { + echo "Moves Cargo.lock to the current main of each internal library with \`cargo update -p\`. The Linux test suite passed on this lock: $RUN_URL" + echo + echo "${SUMMARY:-No summary was recorded.}" + echo + echo "Opened by the Internal dependencies workflow (\`.github/workflows/internal-deps.yml\`), which updates this branch each night the update passes. Merge it when CI is green." + } > "$body" + if [ -n "$pr" ]; then + gh api -X PATCH "$repo_api/pulls/$pr" -F body=@"$body" --jq '"Updated \(.html_url)"' + else + gh api "$repo_api/pulls" -f title="Update internal dependencies" -f head="$BRANCH" -f base=main \ + -F body=@"$body" --jq '"Opened \(.html_url)"' + fi + + report: + name: report + needs: check + if: always() && (needs.check.result == 'success' || needs.check.result == 'failure') + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + issues: write # open, comment on, and close the tracking issue; create its label + steps: + - name: Update the tracking issue + env: + GH_TOKEN: ${{ github.token }} + RESULT: ${{ needs.check.result }} + CHANGED: ${{ needs.check.outputs.changed }} + SUMMARY: ${{ needs.check.outputs.summary }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + label=internal-deps + repo_api="repos/$GITHUB_REPOSITORY" + issue="$(gh api "$repo_api/issues?labels=$label&state=open&per_page=100" \ + --jq '[.[] | select(.pull_request == null)] | sort_by(.number) | .[0].number // empty')" + body="$RUNNER_TEMP/body.md" + + if [ "$RESULT" = "success" ]; then + if [ "$CHANGED" = "true" ]; then + echo "The updated lock passed the Linux test suite; the pull request job carries it." >> "$GITHUB_STEP_SUMMARY" + fi + if [ -n "$issue" ]; then + { + echo "The nightly internal dependency update passed again: $RUN_URL" + echo + echo "${SUMMARY:-No summary was recorded.}" + } > "$body" + gh api "$repo_api/issues/$issue/comments" -F body=@"$body" --silent + gh api -X PATCH "$repo_api/issues/$issue" -f state=closed -f state_reason=completed --silent + echo "Closed #$issue." + fi + exit 0 + fi + + { + echo "The nightly internal dependency update failed: $RUN_URL" + echo + echo "It moved Cargo.lock to the current main of each internal library with \`cargo update -p\` and ran the Linux test suite against it. Whoever broke an API this repository uses fixes it here promptly." + echo + echo "${SUMMARY:-No summary was recorded: the run failed before \`cargo update\` finished.}" + } > "$body" + if [ -n "$issue" ]; then + gh api "$repo_api/issues/$issue/comments" -F body=@"$body" --silent + echo "Commented on #$issue." + else + if ! gh api "$repo_api/labels/$label" --silent 2>/dev/null; then + gh api "$repo_api/labels" -f name="$label" -f color=d93f0b \ + -f description="Nightly internal dependency update" --silent + fi + gh api "$repo_api/issues" -f title="Nightly internal dependency update failed" \ + -F body=@"$body" -f "labels[]=$label" --jq '"Opened #\(.number)."' + fi diff --git a/Cargo.lock b/Cargo.lock index 2dbaa4d6c..8189726dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,7 +1990,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "chrono", "fabro-automation", @@ -2016,7 +2016,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2041,7 +2041,7 @@ dependencies = [ [[package]] name = "fabro-automation" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2062,11 +2062,11 @@ dependencies = [ [[package]] name = "fabro-build-support" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" [[package]] name = "fabro-checkpoint" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "fabro-config", "fabro-types", @@ -2074,7 +2074,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2170,7 +2170,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -2199,7 +2199,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2228,7 +2228,7 @@ dependencies = [ [[package]] name = "fabro-db" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2241,7 +2241,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2260,7 +2260,7 @@ dependencies = [ [[package]] name = "fabro-dot" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "fabro-template", "fabro-types", @@ -2272,7 +2272,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -2286,7 +2286,7 @@ dependencies = [ [[package]] name = "fabro-environment" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2308,7 +2308,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2333,7 +2333,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "fabro-dot", @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "fabro-static", "http 1.4.0", @@ -2353,7 +2353,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "fabro-config", @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "async-trait", "fabro-types", @@ -2382,7 +2382,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "async-trait", "bytes", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "clap", "fabro-options-metadata", @@ -2418,7 +2418,7 @@ dependencies = [ [[package]] name = "fabro-manifest" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2447,7 +2447,7 @@ dependencies = [ [[package]] name = "fabro-mcp-server" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "fabro-client", @@ -2467,7 +2467,7 @@ dependencies = [ [[package]] name = "fabro-mcp-store" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "chrono", "fabro-db", @@ -2485,7 +2485,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "axum", @@ -2507,7 +2507,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "serde", "serde_json", @@ -2515,7 +2515,7 @@ dependencies = [ [[package]] name = "fabro-petri" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2526,6 +2526,7 @@ dependencies = [ "fabro-checkpoint", "fabro-client", "fabro-db", + "fabro-github", "fabro-http", "fabro-interview", "fabro-llm", @@ -2543,6 +2544,7 @@ dependencies = [ "lithos-llm", "object_store", "pebble-coding-agent", + "percent-encoding", "petri-attractor-steps", "petri-execution", "petri-frontend-attractor", @@ -2556,6 +2558,7 @@ dependencies = [ "sandbox-driver-host", "serde", "serde_json", + "smol_str", "sqlx", "tempfile", "thiserror 2.0.18", @@ -2566,7 +2569,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "libc", "tempfile", @@ -2578,7 +2581,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "aho-corasick", "pebble-coding-agent", @@ -2595,7 +2598,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2690,7 +2693,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "fabro-http", "fabro-interview", @@ -2711,18 +2714,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" [[package]] name = "fabro-store" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "bytes", "chrono", @@ -2750,7 +2753,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -2776,7 +2779,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "fabro-types", @@ -2790,7 +2793,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2815,7 +2818,7 @@ dependencies = [ [[package]] name = "fabro-tool" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2839,7 +2842,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2853,7 +2856,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "chrono", "clap", @@ -2880,7 +2883,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "console 0.15.11", @@ -2903,7 +2906,7 @@ dependencies = [ [[package]] name = "fabro-variable" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2920,7 +2923,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2939,7 +2942,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2984,7 +2987,7 @@ dependencies = [ [[package]] name = "fabro-workflow-version" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "fabro-config", "fabro-dot", @@ -5161,7 +5164,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petri-attractor-steps" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "globset", @@ -5192,7 +5195,7 @@ dependencies = [ [[package]] name = "petri-driver" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "getrandom 0.3.4", @@ -5212,7 +5215,7 @@ dependencies = [ [[package]] name = "petri-engine" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "petri-ir", "serde", @@ -5224,7 +5227,7 @@ dependencies = [ [[package]] name = "petri-execution" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "petri-driver", @@ -5248,7 +5251,7 @@ dependencies = [ [[package]] name = "petri-executor" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "libc", @@ -5263,7 +5266,7 @@ dependencies = [ [[package]] name = "petri-executor-sandbox" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "petri-executor", @@ -5285,7 +5288,7 @@ dependencies = [ [[package]] name = "petri-frontend" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "marked-yaml", "petri-ir", @@ -5299,7 +5302,7 @@ dependencies = [ [[package]] name = "petri-frontend-attractor" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "minijinja", "petri-frontend", @@ -5316,7 +5319,7 @@ dependencies = [ [[package]] name = "petri-frontend-fabro" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "petri-frontend", "petri-frontend-attractor", @@ -5332,7 +5335,7 @@ dependencies = [ [[package]] name = "petri-frontend-native" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "petri-frontend", "petri-ir", @@ -5343,7 +5346,7 @@ dependencies = [ [[package]] name = "petri-ir" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "regex", "serde", @@ -5356,7 +5359,7 @@ dependencies = [ [[package]] name = "petri-runtime" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "petri-driver", @@ -5377,7 +5380,7 @@ dependencies = [ [[package]] name = "petri-steps" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "petri-executor", @@ -5393,7 +5396,7 @@ dependencies = [ [[package]] name = "petri-store" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "getrandom 0.3.4", @@ -5408,7 +5411,7 @@ dependencies = [ [[package]] name = "petri-testkit" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/petri.git?branch=main#91d1b77b3927cb3fefcc4a412fdf89ce04857277" +source = "git+https://github.com/lithoscomputer/petri.git?branch=main#e46845bd0139dd04e9e795d3201b5b9b4b8b1026" dependencies = [ "async-trait", "petri-driver", @@ -5424,6 +5427,7 @@ dependencies = [ "serde_json", "smol_str", "tokio", + "tokio-util", ] [[package]] @@ -7896,7 +7900,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" dependencies = [ "axum", "base64", diff --git a/Cargo.toml b/Cargo.toml index 8604c92c0..156edb0d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.375.0-nightly.0" +version = "0.378.0-nightly.0" license = "MIT" [workspace.dependencies] @@ -70,6 +70,7 @@ jsonwebtoken = { version = "10", features = ["aws_lc_rs"] } hkdf = "0.12" hmac = "0.12" sha2 = "0.10" +smol_str = "0.3" hex = "0.4" insta = "1" fabro-test = { path = "lib/foundation/fabro-test" } diff --git a/docs/public/integrations/github.mdx b/docs/public/integrations/github.mdx index 52a22b6d9..602e7f782 100644 --- a/docs/public/integrations/github.mdx +++ b/docs/public/integrations/github.mdx @@ -260,7 +260,9 @@ contents = "write" pull_requests = "write" ``` -Only the listed permissions are requested — the token is scoped to the minimum access needed. If the GitHub App isn't configured or the repository lacks an installation, the run logs a warning and continues without the token. +Only the listed permissions are requested. If no GitHub credentials are configured, Fabro does not inject a managed token. If configured credentials cannot resolve a token, the execution scope fails to initialize. + +GitHub-target runs also receive Git read access to their origin without declaring an integration permission map. In App mode this default token requests only `contents = "read"`, and it is used by Git's credential helper. Declaring integration permissions additionally supplies `GITHUB_TOKEN` to command and agent processes. The managed token carries exactly the access the run declares, so it replaces any `GITHUB_TOKEN` set in the workflow environment, an ACP agent's environment or the sandbox's own environment. To give a stage different GitHub access, change the declared permissions or repositories instead. One-shot containers a stage runs (such as `docker://` steps) receive the managed `GITHUB_TOKEN` but not the Git credential helper, which reads a store inside the stage's sandbox. In App mode, the token covers only the run's origin repository unless the run declares [additional repositories](#additional-repositories). Injecting `GITHUB_TOKEN` alone does not make other private repositories reachable. @@ -279,8 +281,8 @@ The run origin stays implicit — never list it. Each entry is a full `owner/rep What works against every declared repository, within the granted permissions: - **`gh` CLI and raw GitHub API calls** through `GITHUB_TOKEN`. -- **Plain Git over HTTPS** (`git clone https://github.com/owner/repo`), through a secret-free credential helper that reads `$GITHUB_TOKEN` at invocation time. -- **The common SSH spellings** `git@github.com:owner/repo[.git]` and `ssh://git@github.com/owner/repo[.git]`, through per-repository SSH-to-HTTPS rewrites injected into the stage environment. +- **Plain Git over HTTPS** (`git clone https://github.com/owner/repo`), through a credential helper that reads a private, renewable store outside the workspace. +- **The common SSH spellings** `git@github.com:owner/repo[.git]` and `ssh://git@github.com/owner/repo[.git]`, through SSH-to-HTTPS rewrites injected into the stage environment. Fabro does not clone additional repositories for you; a workflow that needs one on disk adds its own clone step (`git clone https://github.com/owner/repo` or `gh repo clone owner/repo`). @@ -296,15 +298,15 @@ Behavior notes: - **Token strategy (PAT):** the configured PAT is used as-is. The repository list drives validation and preflight probes, but it cannot narrow the PAT's inherent GitHub scope — App mode remains the least-authority option. - **`GH_TOKEN` precedence:** `gh` checks `GH_TOKEN` before `GITHUB_TOKEN`. If the resolved run environment defines `GH_TOKEN`, `gh` uses it instead of the managed token; Fabro never sets or removes `GH_TOKEN`, and preflight warns when additional repositories are declared alongside one. -- **SSH rewrites match by prefix.** With `owner/repo` declared, the SSH spelling of `owner/repo-other` is also rewritten to HTTPS. The scoped token is invalid for undeclared repositories at GitHub, so authority is unchanged — but a private undeclared repository fails with a GitHub authorization error instead of a missing-credential or SSH error. +- **GitHub SSH URLs are rewritten to HTTPS.** The helper supplies credentials only for the origin and declared repository paths, including their `.git` spellings. An undeclared private repository does not receive a credential from this helper. #### Security boundary Workflow authors may name any repository reachable by the server's GitHub App installation; Fabro applies no second server-side repository intersection. The token is scoped server-side to exactly the declared set — a request to an undeclared repository fails at GitHub, and Fabro never mints an unscoped installation-wide token. With `contents = "write"`, **any stage can push to any declared repository**. Declare the smallest repository set and the weakest permissions that work. -Installation Access Tokens are short-lived. Fabro's own pushes present a fresh token on each call. Git commands the agent runs inside the sandbox read the token through a credential store the sandbox driver configures for the checkout; the token never appears in the repository's remote URL or configuration. For ACP/CLI agent turns launched with GitHub App push credentials, Fabro re-mints the token and rewrites that store before the ACP process starts, then every 45 minutes for the lifetime of that turn. Refresh failures are logged and do not fail the stage. +Installation Access Tokens are short-lived. Fabro reuses cached tokens until they are within ten minutes of expiry. Each process launch resolves its current credentials, and a background task checks the Git credential store every minute while the execution scope is acquired. Command stages, native agent tools, ACP agents, and resumed scopes use the same mechanism. A long-lived ACP agent's later Git operations read the renewed store. The token never appears in the repository's remote URL, Git configuration, or workspace snapshots; the private store is removed when the scope releases. Background refresh failures are logged and retried, and a still-valid cached token remains usable. -`FABRO_PUSH_CRED_REFRESH_AHEAD` defaults to enabled; set it to `0`, `false`, `off`, `no`, or an empty value to disable both turn-entry and background refresh. `FABRO_PUSH_CRED_REFRESH_INTERVAL_SECONDS` overrides the background interval, and `0` disables only the background loop. This refresh loop is ACP-specific; command and native/API agent stages do not run it. Reconnected sandboxes for resumed or parked runs currently lack the App credentials needed for ACP refresh, so the refresh is skipped there. +`GITHUB_TOKEN` is a process environment variable, so a process that is already running keeps its launch-time value. The renewable Git helper continues to work across token rotation, but a long-running process that calls the GitHub API through `GITHUB_TOKEN` must restart or obtain a new token separately. Static PATs cannot be renewed by Fabro. Publication uses a separate token source with the permissions needed to push and open pull requests. The permissions table follows the standard layer-merge order (workflow > project > user > defaults). Set defaults at `[run.integrations.github.permissions]` in `~/.fabro/settings.toml` so every run inherits a baseline; tighten or override per-workflow as needed. A higher layer that defines `permissions = {}` clears the inherited map (no token requested). diff --git a/lib/apps/fabro-cli/src/commands/run/petri_worker.rs b/lib/apps/fabro-cli/src/commands/run/petri_worker.rs index ba2a834ff..f59b415ed 100644 --- a/lib/apps/fabro-cli/src/commands/run/petri_worker.rs +++ b/lib/apps/fabro-cli/src/commands/run/petri_worker.rs @@ -84,6 +84,7 @@ use fabro_petri::providers::{DaytonaCredentials, SandboxProviderConfig}; use fabro_petri::runtime::{self, RuntimeSpec}; use fabro_petri::secrets::VaultSecrets; use fabro_petri::source::RunSource; +use fabro_petri::stage_credentials::StageCredentials; use fabro_petri::{HttpRunStore, admission}; use fabro_static::EnvVars; use fabro_store::RunProjection; @@ -171,7 +172,7 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { let run_tools = run_tool_services(&worker); let catalog = command_context::load_cli_catalog().context("failed to build worker LLM catalog")?; - let runtime = runtime_spec( + let mut runtime = runtime_spec( catalog.clone(), &vault, &worker.run_state, @@ -215,13 +216,16 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { None } }; + let read_tokens = publish::read_token_source(&worker.run_state.spec, github.as_ref()); + runtime.stage_credentials = + StageCredentials::for_run(&worker.run_state.spec, github.as_ref(), read_tokens.clone())?; let mut source = RunSource::for_run( worker.run_state.spec.target.as_ref(), &worker.run_state.spec.settings.run, None, ); if let Some(source) = &mut source { - source.credentials = publish::source_credentials(&worker.run_state.spec, github.as_ref()); + source.credentials = read_tokens.map(publish::source_credentials); } let publisher = publish::GitHubPublisher::for_run( run_id, @@ -657,6 +661,7 @@ async fn runtime_spec( .with_http_client(fabro_http::http_client().ok()) }); Ok(RuntimeSpec { + stage_credentials: None, sandbox: SandboxProviderConfig::from_lookup(daytona, crate::process_env_var), settings_toml: None, mcp_catalog_toml: None, diff --git a/lib/apps/fabro-cli/src/commands/run/publish.rs b/lib/apps/fabro-cli/src/commands/run/publish.rs index 092bf2d13..deba41843 100644 --- a/lib/apps/fabro-cli/src/commands/run/publish.rs +++ b/lib/apps/fabro-cli/src/commands/run/publish.rs @@ -107,15 +107,22 @@ fn token_source( } } -/// The read-only credentials the run's workspaces are fetched with. `None` -/// when the run has no GitHub target or no credentials resolve; a public -/// repository is then fetched anonymously. -pub(super) fn source_credentials( +/// The read-only token source the run's workspaces are fetched with, and +/// its stages' Git reads the origin with. `None` when the run has no GitHub +/// target or no credentials resolve; a public repository is then fetched +/// anonymously. +pub(super) fn read_token_source( spec: &RunSpec, credentials: Option<&GitHubCredentials>, -) -> Option> { - let tokens = token_source(spec, credentials, serde_json::json!({ "contents": "read" }))?; - Some(Arc::new(ReadCredentials(tokens))) +) -> Option> { + token_source(spec, credentials, serde_json::json!({ "contents": "read" })) +} + +/// Fetch credentials over the run's read-only token source. +pub(super) fn source_credentials( + tokens: Arc, +) -> Arc { + Arc::new(ReadCredentials(tokens)) } /// Fetch credentials resolved from the run's read-only token source. diff --git a/lib/apps/fabro-cli/tests/it/cmd/attach.rs b/lib/apps/fabro-cli/tests/it/cmd/attach.rs index c8978361e..ff4b4bb42 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/attach.rs @@ -1103,7 +1103,7 @@ fn attach_json_errors_without_prompting_for_human_input() { "recorded_at": "[EPOCH_MS]", "body": { "event": "run.started", - "format_version": 7, + "format_version": 8, "key": "[ULID]", "root": 0, "middleware_chain": [ diff --git a/lib/apps/fabro-server/src/manifest_validation.rs b/lib/apps/fabro-server/src/manifest_validation.rs index acbe81a3d..f47a63e26 100644 --- a/lib/apps/fabro-server/src/manifest_validation.rs +++ b/lib/apps/fabro-server/src/manifest_validation.rs @@ -56,13 +56,14 @@ fn offline_runtime(run: Option<&RunLayer>) -> RuntimeSpec { ..SettingsLayer::default() }; RuntimeSpec { - sandbox: SandboxProviderConfig::default(), - settings_toml: toml::to_string(&layer).ok(), - mcp_catalog_toml: None, - model_client: None, - dry_run: false, - fabro_home: None, - run_tools: None, + stage_credentials: None, + sandbox: SandboxProviderConfig::default(), + settings_toml: toml::to_string(&layer).ok(), + mcp_catalog_toml: None, + model_client: None, + dry_run: false, + fabro_home: None, + run_tools: None, } } diff --git a/lib/apps/fabro-server/src/server/petri_runs.rs b/lib/apps/fabro-server/src/server/petri_runs.rs index 734a58025..2c7c2d815 100644 --- a/lib/apps/fabro-server/src/server/petri_runs.rs +++ b/lib/apps/fabro-server/src/server/petri_runs.rs @@ -99,6 +99,7 @@ pub(crate) fn runtime_spec( } }; RuntimeSpec { + stage_credentials: None, sandbox, settings_toml, mcp_catalog_toml, diff --git a/lib/components/fabro-petri/Cargo.toml b/lib/components/fabro-petri/Cargo.toml index f9cd9a456..4034e8a0f 100644 --- a/lib/components/fabro-petri/Cargo.toml +++ b/lib/components/fabro-petri/Cargo.toml @@ -19,6 +19,9 @@ workspace = true test-support = ["dep:petri_testkit"] [dependencies] +fabro-github = { path = "../fabro-github" } +percent-encoding.workspace = true +smol_str.workspace = true fabro-api = { path = "../../foundation/fabro-api" } fabro-client = { path = "../../foundation/fabro-client" } fabro-db = { path = "../../foundation/fabro-db" } @@ -57,6 +60,7 @@ tokio-util.workspace = true tracing.workspace = true [dev-dependencies] +fabro-github = { path = "../fabro-github", features = ["test-support"] } fabro-macros = { path = "../../foundation/fabro-macros" } fabro-petri = { path = ".", features = ["test-support"] } fabro-tool = { path = "../fabro-tool" } diff --git a/lib/components/fabro-petri/src/engine.rs b/lib/components/fabro-petri/src/engine.rs index 369149be5..b833bc12d 100644 --- a/lib/components/fabro-petri/src/engine.rs +++ b/lib/components/fabro-petri/src/engine.rs @@ -226,6 +226,11 @@ pub async fn run(request: RunRequest) -> Result { if let Some(secrets) = request.secrets { runtime = runtime.secrets(SharedSecrets(secrets)); } + if let Some(credentials) = request.runtime.stage_credentials.clone() { + let masker = runtime.masker(); + runtime = + runtime.executor_layer(move |executor| credentials.executor(executor, masker.clone())); + } if let Some(blobs) = &request.blobs { runtime = runtime.capability(RunBlobs::output_store(Arc::clone(blobs))); } diff --git a/lib/components/fabro-petri/src/lib.rs b/lib/components/fabro-petri/src/lib.rs index 03cd854ec..e50758dfe 100644 --- a/lib/components/fabro-petri/src/lib.rs +++ b/lib/components/fabro-petri/src/lib.rs @@ -85,6 +85,7 @@ pub mod run_store; pub mod runtime; pub mod secrets; pub mod source; +pub mod stage_credentials; #[cfg(feature = "test-support")] pub mod test_support; pub mod workspace; diff --git a/lib/components/fabro-petri/src/projection/engine.rs b/lib/components/fabro-petri/src/projection/engine.rs index 12fae7891..a6c82b292 100644 --- a/lib/components/fabro-petri/src/projection/engine.rs +++ b/lib/components/fabro-petri/src/projection/engine.rs @@ -13,7 +13,7 @@ use fabro_types::{ use petri_execution::events::{Derived, RunEvent, Subject, ViewEvent, WaitState}; use petri_execution::{ExecutionId, InvocationId}; use petri_runtime::engine::{Admission, Event}; -use petri_runtime::ir::{Metrics, Status}; +use petri_runtime::ir::{Metrics, Status, UnderlyingFailure}; use serde_json::Value; use tracing::debug; @@ -382,9 +382,12 @@ pub(super) fn failure_message(status: &Status) -> Option { match status { Status::Failure(info) | Status::PartialSuccess { - underlying: Some(info), + underlying: Some(UnderlyingFailure::Failure(info)), } => Some(info.message.clone()), - Status::TimedOut => Some("the step timed out".to_string()), + Status::TimedOut + | Status::PartialSuccess { + underlying: Some(UnderlyingFailure::TimedOut), + } => Some("the step timed out".to_string()), Status::Cancelled => Some("the step was cancelled".to_string()), Status::Success | Status::PartialSuccess { underlying: None } | Status::Skipped => None, } diff --git a/lib/components/fabro-petri/src/runtime.rs b/lib/components/fabro-petri/src/runtime.rs index da6a3b129..0cb377727 100644 --- a/lib/components/fabro-petri/src/runtime.rs +++ b/lib/components/fabro-petri/src/runtime.rs @@ -32,38 +32,41 @@ use tracing::debug; use crate::host_tools; use crate::providers::{self, SandboxProviderConfig}; +use crate::stage_credentials::StageCredentials; /// What every Petri runtime Fabro builds is configured with. #[derive(Clone, Default)] pub struct RuntimeSpec { + /// Run-specific GitHub credentials applied to processes at execution. + pub stage_credentials: Option, /// Explicit provider configuration. Factories connect only at acquire. - pub sandbox: SandboxProviderConfig, + pub sandbox: SandboxProviderConfig, /// The operator's settings layer, as `~/.fabro/settings.toml` text: the /// lowest of the three layers the Fabro frontend reads (`[run.model]` /// defaults, `[[run.hooks]]`, `[run.agent.mcps]`, `[run.environment]` /// and the `[environments.]` catalog a bundle may name). - pub settings_toml: Option, + pub settings_toml: Option, /// The server's MCP catalog, as the TOML text the Fabro frontend /// resolves `[run.agent.mcps.] id = "..."` references against: a /// table keyed by catalog id, each entry in the inline /// `[run.agent.mcps.]` shape. `None` leaves every reference /// refused, as the standalone runner refuses it. - pub mcp_catalog_toml: Option, + pub mcp_catalog_toml: Option, /// The model client the native agent and prompt steps call, and the /// catalog the admission pass resolves model selectors against. `None` /// leaves every LLM node unpinned and every model call unconfigured. - pub model_client: Option, + pub model_client: Option, /// Simulate steps (Fabro's `--dry-run` handlers) in local workspaces, /// without acquiring the configured Docker or Daytona sandboxes. - pub dry_run: bool, + pub dry_run: bool, /// The Fabro home the skills step reads; `None` leaves it to Petri's /// own lookup (`FABRO_HOME`, else `$HOME/.fabro`). - pub fabro_home: Option, + pub fabro_home: Option, /// Fabro's run tools for every native agent session of the run, when /// the run enables them (`[run.agent] fabro_tools` and the worker /// token's `agent:run_tools` scope); `None` gives the sessions Pebble's /// tools alone. See [`crate::host_tools`]. - pub run_tools: Option, + pub run_tools: Option, } impl RuntimeSpec { diff --git a/lib/components/fabro-petri/src/stage_credentials.rs b/lib/components/fabro-petri/src/stage_credentials.rs new file mode 100644 index 000000000..547d529ae --- /dev/null +++ b/lib/components/fabro-petri/src/stage_credentials.rs @@ -0,0 +1,666 @@ +//! GitHub credentials for every process in a run's execution scopes. +//! +//! Git reads a private, renewable store outside the workspace. Processes get +//! the helper configuration and, when requested, a fresh `GITHUB_TOKEN` at +//! spawn. A long-lived agent's Git commands read the renewed store. A +//! one-shot container gets the token alone: the store lives in the scope's +//! sandbox, which the container does not share. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use fabro_github::token_source::{InstallationTokenSource, ResolvedToken}; +use fabro_github::{GITHUB_CREDENTIAL_HELPER_KEY, GitHubCredentials, GitHubRepositoryAccess}; +use fabro_static::EnvVars; +use fabro_types::settings::run::{RunIntegrationsGithubSettings, RunMode}; +use fabro_types::{GitHubRepositorySlug, RunSpec, RunTarget}; +use fabro_util::shell; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +use petri_runtime::executor::{ + AcquireContext, EnvError, EnvHandle, ExecEnv, Executor, Masker, ProcessSpec, ReleaseReport, + ScopeOutcome, ScopeSpec, SpawnEnv, SpawnTarget, +}; +use petri_runtime::ir::LogStream; +use smol_str::SmolStr; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time; +use tracing::warn; + +const REFRESH_INTERVAL: Duration = Duration::from_mins(1); +const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30); + +/// Token policy resolved by Fabro, without GitHub policy in Petri. +#[derive(Clone)] +pub struct StageCredentials { + git_tokens: Arc, + api_tokens: Option>, + repositories: Vec, +} + +impl StageCredentials { + /// Default origin access is `read_tokens`, the read-only source the + /// run's workspaces are fetched with. Declared permissions govern the + /// integration token and additional repositories. Static tokens retain + /// their existing scope; publication has a separate write-token source. + pub fn for_run( + spec: &RunSpec, + credentials: Option<&GitHubCredentials>, + read_tokens: Option>, + ) -> anyhow::Result> { + if spec.settings.run.execution.mode == RunMode::DryRun { + return Ok(None); + } + let Some(RunTarget::Git(target)) = &spec.target else { + return Ok(None); + }; + let repository = target.clone().validate()?.repository().clone(); + let integration = spec + .settings + .run + .integrations + .github + .resolve_integration()?; + let Some(access) = GitHubRepositoryAccess::new( + Some(&repository.https_url()), + &integration.additional_repositories, + integration.permissions.clone(), + )? + else { + return Ok(None); + }; + let Some(credentials) = credentials else { + return Ok(None); + }; + let api_tokens = integration + .is_token_requested() + .then(|| InstallationTokenSource::for_access(credentials, &access)) + .transpose()?; + let contents_declared = integration + .permissions + .get("contents") + .is_some_and(|value| { + RunIntegrationsGithubSettings::contents_permission_allows_repository_access(value) + }); + let (git_tokens, repositories) = match (&api_tokens, read_tokens) { + (Some(tokens), _) if contents_declared => ( + Arc::clone(tokens), + access.targets().into_iter().cloned().collect(), + ), + (_, Some(tokens)) => (tokens, vec![repository]), + (_, None) => return Ok(None), + }; + Ok(Some(Self { + git_tokens, + api_tokens, + repositories, + })) + } + + pub(crate) fn executor(&self, inner: Arc, masker: Masker) -> Arc { + Arc::new(CredentialExecutor { + inner, + credentials: self.clone(), + masker, + scopes: Mutex::new(HashMap::new()), + }) + } +} + +struct CredentialExecutor { + inner: Arc, + credentials: StageCredentials, + masker: Masker, + scopes: Mutex>, +} + +struct ScopeRefresh { + env: Arc, + task: Option>, +} + +impl Drop for ScopeRefresh { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} + +#[async_trait] +impl Executor for CredentialExecutor { + async fn acquire( + &self, + scope: &ScopeSpec, + ctx: &AcquireContext, + ) -> Result { + let handle = self.inner.acquire(scope, ctx).await?; + let env = match CredentialEnv::install( + handle.exec(), + self.credentials.clone(), + self.masker.clone(), + ) + .await + { + Ok(env) => Arc::new(env), + Err(error) => { + self.inner.release(handle, ScopeOutcome::Failed).await; + return Err(error); + } + }; + // Only minted tokens renew; a static token's store never changes. + let task = self + .credentials + .git_tokens + .mints_installation_tokens() + .then(|| { + let env = Arc::clone(&env); + tokio::spawn(async move { + loop { + time::sleep(REFRESH_INTERVAL).await; + if let Err(error) = env.refresh().await { + warn!(error = %env.masker.mask(&error.to_string()), "could not refresh the sandbox's GitHub credentials; retrying"); + } + } + }) + }); + self.scopes + .lock() + .await + .insert(handle.instance().to_string(), ScopeRefresh { + env: Arc::clone(&env), + task, + }); + Ok(handle.with_spawn_env(env)) + } + + async fn release(&self, handle: EnvHandle, outcome: ScopeOutcome) -> ReleaseReport { + let mut problems = Vec::new(); + let refresh = self.scopes.lock().await.remove(handle.instance()); + if let Some(mut refresh) = refresh { + if let Some(task) = refresh.task.take() { + task.abort(); + let _ = task.await; + } + if refresh.env.cleanup().await.is_err() { + problems.push("could not remove the sandbox's GitHub credential store".to_string()); + } + } + let mut report = self.inner.release(handle, outcome).await; + report.problems.extend(problems); + report + } +} + +struct CredentialEnv { + inner: Arc, + credentials: StageCredentials, + masker: Masker, + directory: String, + /// The token generation the store holds; `None` before the first write. + written: Mutex>, +} + +impl CredentialEnv { + /// Create the scope's private store directory and write the first store, + /// removing the directory again when that write fails. + async fn install( + inner: Arc, + credentials: StageCredentials, + masker: Masker, + ) -> Result { + let directory = command( + &inner, + "umask 077; mktemp -d /tmp/fabro-git-credentials.XXXXXXXX", + BTreeMap::new(), + ) + .await? + .trim() + .to_string(); + let env = Self { + inner, + credentials, + masker, + directory, + written: Mutex::new(None), + }; + if let Err(error) = env.refresh().await { + let _ = env.cleanup().await; + return Err(error); + } + Ok(env) + } + + fn store_path(&self) -> String { + shell::shell_quote(&format!("{}/store", self.directory)) + } + + async fn resolve( + &self, + tokens: &InstallationTokenSource, + operation: &'static str, + ) -> Result { + let resolved = time::timeout(CREDENTIAL_TIMEOUT, tokens.resolve()) + .await + .map_err(|_| EnvError::backend("github", operation, "token resolution timed out"))? + .map_err(|_| EnvError::backend("github", operation, "token resolution failed"))?; + self.masker.register_explicit(resolved.token.expose()); + Ok(resolved) + } + + /// Rewrite the store when the token source has minted a new generation. + async fn refresh(&self) -> Result<(), EnvError> { + let mut written = self.written.lock().await; + let token = self + .resolve(&self.credentials.git_tokens, "refresh") + .await?; + if *written == Some(token.snapshot.generation) { + return Ok(()); + } + let password = utf8_percent_encode(token.token.expose(), NON_ALPHANUMERIC).to_string(); + self.masker.register_explicit(&password); + let store = self + .credentials + .repositories + .iter() + .flat_map(|repo| { + [ + format!("https://x-access-token:{password}@github.com/{repo}\n"), + format!("https://x-access-token:{password}@github.com/{repo}.git\n"), + ] + }) + .collect::(); + let path = self.store_path(); + command( + &self.inner, + &format!("umask 077; printf '%s' \"$FABRO_GIT_CREDENTIAL_STORE\" > {path}.new && mv -f {path}.new {path}"), + BTreeMap::from([("FABRO_GIT_CREDENTIAL_STORE".into(), store.into())]), + ) + .await?; + *written = Some(token.snapshot.generation); + Ok(()) + } + + async fn cleanup(&self) -> Result<(), EnvError> { + command( + &self.inner, + &format!("rm -rf -- {}", shell::shell_quote(&self.directory)), + BTreeMap::new(), + ) + .await + .map(|_| ()) + } + + fn git_env(&self, env: &mut BTreeMap) -> Result<(), EnvError> { + let count = env + .get("GIT_CONFIG_COUNT") + .map(ToString::to_string) + .or_else(|| self.inner.ambient_env("GIT_CONFIG_COUNT")) + .unwrap_or_else(|| "0".to_string()) + .parse::() + .ok() + .filter(|count| *count <= 256) + .ok_or_else(|| { + EnvError::backend("github", "environment", "invalid GIT_CONFIG_COUNT") + })?; + // Preserve fetch/publish headers and workflow configuration. The store + // returns a credential only for an allowed repository path. + let entries = [ + ( + "credential.https://github.com.useHttpPath", + "true".to_string(), + ), + (GITHUB_CREDENTIAL_HELPER_KEY, String::new()), + ( + GITHUB_CREDENTIAL_HELPER_KEY, + format!("store --file={}", self.store_path()), + ), + ( + "url.https://github.com/.insteadOf", + "git@github.com:".to_string(), + ), + ( + "url.https://github.com/.insteadOf", + "ssh://git@github.com/".to_string(), + ), + ]; + for index in 0..count { + for prefix in ["GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_"] { + let key = format!("{prefix}{index}"); + if !env.contains_key(key.as_str()) { + if let Some(value) = self.inner.ambient_env(&key) { + env.insert(key.into(), value.into()); + } + } + } + } + for (offset, (key, value)) in entries.iter().enumerate() { + env.insert( + format!("GIT_CONFIG_KEY_{}", count + offset).into(), + (*key).into(), + ); + env.insert( + format!("GIT_CONFIG_VALUE_{}", count + offset).into(), + value.as_str().into(), + ); + } + env.insert( + "GIT_CONFIG_COUNT".into(), + (count + entries.len()).to_string().into(), + ); + env.entry("GIT_TERMINAL_PROMPT".into()) + .or_insert_with(|| "0".into()); + Ok(()) + } +} + +#[async_trait] +impl SpawnEnv for CredentialEnv { + async fn apply( + &self, + target: SpawnTarget, + env: &mut BTreeMap, + ) -> Result<(), EnvError> { + // The managed token carries exactly the access the run declared, so + // it replaces any `GITHUB_TOKEN` the process would otherwise see, as + // Fabro's stage environment did before Petri. + if let Some(tokens) = &self.credentials.api_tokens { + let resolved = self.resolve(tokens, "resolve").await?; + env.insert(EnvVars::GITHUB_TOKEN.into(), resolved.token.expose().into()); + } + // The store and the helper configuration that names it live in the + // scope's sandbox; a one-shot container cannot read either. + if target == SpawnTarget::Process { + self.refresh().await?; + self.git_env(env)?; + } + Ok(()) + } +} + +async fn command( + env: &Arc, + script: &str, + extra_env: BTreeMap, +) -> Result { + let spec = ProcessSpec::new("sh", &["-c", script]) + .with_timeout(Some(CREDENTIAL_TIMEOUT)) + .with_env(extra_env); + let mut handle = env.spawn(spec).await?; + let mut stdout = String::new(); + if let Some(mut lines) = handle.lines() { + while let Some(line) = lines.recv().await { + // Never echo stderr from an operation installing credentials. + if line.stream == LogStream::Stdout { + stdout.push_str(&line.line); + stdout.push('\n'); + } + } + } + if !handle.wait().await?.is_success() { + return Err(EnvError::backend( + "github", + "credential_store", + "credential store operation failed", + )); + } + Ok(stdout) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::os::unix::fs::PermissionsExt as _; + use std::path::Path; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use chrono::Utc; + use fabro_github::InstallationToken; + use fabro_github::test_support::{self, InstallationTokenMinter}; + use fabro_types::test_support as types_support; + use petri_runtime::executor::{MapSecrets, SecretProvider as _, StdinMode}; + use petri_runtime::ir::ScopeId; + use tokio::io::AsyncWriteExt as _; + + use super::*; + use crate::providers::{self, SandboxProviderConfig}; + + struct RotatingMinter(AtomicUsize); + + #[async_trait] + impl InstallationTokenMinter for RotatingMinter { + async fn mint(&self) -> anyhow::Result { + let generation = self.0.fetch_add(1, Ordering::SeqCst) + 1; + Ok(InstallationToken { + token: format!("scripted-token-generation-{generation}"), + // Each resolve exercises renewal without waiting an hour. + expires_at: Utc::now() + chrono::Duration::minutes(5), + }) + } + } + + const MANAGED_TOKEN_CHECK: &str = + "case $GITHUB_TOKEN in scripted-token-generation-*) exit 0;; *) exit 1;; esac"; + + /// A scope acquired through the credential layer over a local sandbox. + async fn acquire( + name: &str, + api: bool, + ) -> (tempfile::TempDir, Arc, EnvHandle, MapSecrets) { + let dir = tempfile::tempdir().expect("directory"); + let runtime = providers::standard_runtime(&SandboxProviderConfig::default()); + let router = runtime.sandbox_router_for(dir.path()).expect("router"); + let secrets = MapSecrets::empty(); + let executor = credentials(api).executor(router, secrets.masker()); + let handle = executor + .acquire( + &ScopeSpec::new(ScopeId::new(0), name), + &AcquireContext::bare(), + ) + .await + .expect("acquire"); + (dir, executor, handle, secrets) + } + + fn credentials(api: bool) -> StageCredentials { + let tokens = test_support::installation_token_source( + "acme/private", + Arc::new(RotatingMinter(AtomicUsize::new(0))), + ); + StageCredentials { + git_tokens: tokens.clone(), + api_tokens: api.then_some(tokens), + repositories: vec![GitHubRepositorySlug::try_new("acme/private").expect("slug")], + } + } + + #[test] + fn default_origin_access_does_not_request_an_api_token() { + let mut spec = types_support::test_run_spec(); + spec.target = Some( + serde_json::from_value( + serde_json::json!({"kind":"git", "repo":"acme/private", "branch":"main"}), + ) + .expect("target"), + ); + let pat = GitHubCredentials::Pat("scripted-personal-access-token".to_string()); + let read = InstallationTokenSource::pat("scripted-personal-access-token".to_string()); + let default = StageCredentials::for_run(&spec, Some(&pat), Some(Arc::clone(&read))) + .expect("policy") + .expect("credentials"); + assert!(default.api_tokens.is_none()); + assert_eq!(default.repositories.len(), 1); + spec.settings + .run + .integrations + .github + .permissions + .insert("contents".to_string(), "read".into()); + spec.settings + .run + .integrations + .github + .additional_repositories + .insert(GitHubRepositorySlug::try_new("acme/another").expect("slug")); + let declared = StageCredentials::for_run(&spec, Some(&pat), Some(read)) + .expect("policy") + .expect("credentials"); + assert!(declared.api_tokens.is_some()); + assert_eq!(declared.repositories.len(), 2); + } + + #[tokio::test] + async fn a_running_process_reads_renewed_git_credentials_and_release_cleans_up() { + let (_dir, executor, handle, secrets) = acquire("credentials", false).await; + let env = handle.exec(); + let script = "printf 'protocol=https\\nhost=github.com\\npath=acme/private\\n\\n' | git credential fill; printf 'READY\\n'; read answer; printf 'protocol=https\\nhost=github.com\\npath=acme/private\\n\\n' | git credential fill"; + let mut process = env + .spawn(ProcessSpec::new("sh", &["-c", script]).with_stdin(StdinMode::Piped)) + .await + .expect("launch"); + let mut lines = process.lines().expect("lines"); + let mut first = String::new(); + while let Some(line) = lines.recv().await { + if line.line == "READY" { + break; + } + first.push_str(&line.line); + } + assert!( + first.contains("password=scripted-token-generation-2"), + "first token is delivered (password present: {}, masked: {}, stderr present: {})", + first.contains("password="), + first.contains("***"), + first.contains("fatal:") + ); + // Another spawn rotates the store while the original process lives. + command(&env, "true", BTreeMap::new()) + .await + .expect("refresh via spawn"); + process + .stdin() + .expect("stdin") + .write_all(b"continue\n") + .await + .expect("continue"); + let mut second = String::new(); + while let Some(line) = lines.recv().await { + second.push_str(&line.line); + } + assert!( + second.contains("password=scripted-token-generation-3"), + "the same process uses the renewed helper" + ); + assert!(process.wait().await.expect("exit").is_success()); + assert!( + !secrets + .masker() + .mask(&format!("{first}{second}")) + .contains("scripted-token") + ); + assert!(command(&env, "printf 'protocol=https\\nhost=github.com\\npath=acme/unrelated\\n\\n' | git credential fill", BTreeMap::new()).await.is_err(), "the helper refuses an undeclared repository"); + command(&env, "printf 'protocol=https\\nhost=github.com\\npath=acme/private.git\\n\\n' | git credential fill >/dev/null", BTreeMap::new()).await.expect("the .git spelling is authenticated too"); + let probe = command( + &env, + "git config --get credential.https://github.com.helper", + BTreeMap::new(), + ) + .await + .expect("helper"); + let file = probe + .trim() + .strip_prefix("store --file=") + .expect("store path"); + assert_eq!( + fs::metadata(file).expect("file").permissions().mode() & 0o777, + 0o600 + ); + let report = executor.release(handle, ScopeOutcome::Succeeded).await; + assert!(report.is_clean(), "{report:?}"); + assert!( + !Path::new(file).exists(), + "release removes the credential store" + ); + } + + #[tokio::test] + async fn a_container_gets_the_managed_token_but_no_store_configuration() { + let dir = tempfile::tempdir().expect("directory"); + let runtime = providers::standard_runtime(&SandboxProviderConfig::default()); + let router = runtime.sandbox_router_for(dir.path()).expect("router"); + let handle = router + .acquire( + &ScopeSpec::new(ScopeId::new(0), "container-credentials"), + &AcquireContext::bare(), + ) + .await + .expect("acquire"); + let store = dir.path().join("store-directory"); + let secrets = MapSecrets::empty(); + let env = CredentialEnv { + inner: handle.exec(), + credentials: credentials(true), + masker: secrets.masker(), + directory: store.display().to_string(), + written: Mutex::new(None), + }; + let mut container = BTreeMap::from([("GITHUB_TOKEN".into(), "explicit".into())]); + env.apply(SpawnTarget::Container, &mut container) + .await + .expect("apply"); + assert!( + container["GITHUB_TOKEN"].starts_with("scripted-token-generation-"), + "the managed token replaces an explicit one in a container too" + ); + assert!( + !container.keys().any(|key| key.starts_with("GIT_CONFIG")), + "no helper configuration names the scope's store" + ); + assert!(!store.exists(), "a container spawn writes no store"); + let mut process = BTreeMap::new(); + fs::create_dir(&store).expect("store directory"); + env.apply(SpawnTarget::Process, &mut process) + .await + .expect("apply"); + assert!(process.contains_key("GIT_CONFIG_COUNT")); + assert!( + store.join("store").exists(), + "a process spawn refreshes the store" + ); + assert!( + router + .release(handle, ScopeOutcome::Succeeded) + .await + .is_clean() + ); + } + + #[tokio::test] + async fn declared_api_tokens_reach_processes_and_replace_explicit_values() { + let (_dir, executor, handle, secrets) = acquire("api-credentials", true).await; + let env = handle.exec(); + command(&env, MANAGED_TOKEN_CHECK, BTreeMap::new()) + .await + .expect("the integration token reaches the child"); + command( + &env, + MANAGED_TOKEN_CHECK, + BTreeMap::from([("GITHUB_TOKEN".into(), "command-token-override".into())]), + ) + .await + .expect("the managed token replaces an explicit one"); + assert!( + secrets + .masker() + .contains_secret("scripted-token-generation-3") + ); + assert!( + executor + .release(handle, ScopeOutcome::Succeeded) + .await + .is_clean() + ); + } +}