From 5b5d2a1cdae7253b2feb1730a601bb5de1115e84 Mon Sep 17 00:00:00 2001 From: Fabro Date: Wed, 27 May 2026 21:08:24 -0400 Subject: [PATCH] =?UTF-8?q?finalize=20run=20=E2=9A=92=EF=B8=8F=20Generated?= =?UTF-8?q?=20with=20[Fabro](https://fabro.sh)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run.json | 401 ++- stages/008-verify@1/diff.patch | 3628 ++++++++++++++++++++++++ stages/008-verify@1/output.log | 1 + stages/008-verify@1/script_timing.json | 8 + stages/008-verify@1/status.json | 6 + stages/009-exit@1/status.json | 6 + 6 files changed, 3938 insertions(+), 112 deletions(-) create mode 100644 stages/008-verify@1/diff.patch create mode 100644 stages/008-verify@1/output.log create mode 100644 stages/008-verify@1/script_timing.json create mode 100644 stages/008-verify@1/status.json create mode 100644 stages/009-exit@1/status.json diff --git a/run.json b/run.json index 8e3e85865..71bae418f 100644 --- a/run.json +++ b/run.json @@ -505,7 +505,7 @@ "kind": "running" }, "status_updated_at": "2026-05-28T00:17:17.965930Z", - "last_event_at": "2026-05-28T00:59:14.909882Z", + "last_event_at": "2026-05-28T01:08:23.965487Z", "pending_control": null, "checkpoints": [ { @@ -1356,9 +1356,9 @@ } }, { - "seq": 0, + "seq": 985, "checkpoint": { - "timestamp": "2026-05-28T01:08:19.665710Z", + "timestamp": "2026-05-28T01:08:23.965351Z", "current_node": "verify", "completed_nodes": [ "start", @@ -1372,53 +1372,67 @@ ], "node_retries": {}, "context_values": { - "response.simplify_opus": "Web UI and CLI remain untouched.\n\n## Summary\n\nReviewed 3 changed files via parallel reuse/quality/efficiency agents and applied the following cleanups:\n\n**Removed dead/redundant code:**\n- Deleted `require_run_tool_actor` helper in `principal_middleware.rs` — was a trivial alias for `require_run_management_actor`; `RequiredRunToolActor::from_request_parts` now calls the real function directly.\n- Removed stale `#[allow(dead_code, reason = \"...\")]` on `AppState::materialize_automation_run` in `server.rs` (now called from `create_automation_run`).\n\n**Inlined single-use helpers in `automations.rs`:**\n- `automation_api_trigger_disabled_error` + the matching const `AUTOMATION_API_TRIGGER_DISABLED_CODE` — only used by one `else` branch.\n- `automation_materialization_temp_root` — one-line `Storage::new(...).scratch_dir().join(\"automations\")`.\n- `automation_materialize_error` — one-line wrapper around `ApiError::new(UNPROCESSABLE_ENTITY, …)`.\n\n**Moved logic to its proper home:**\n- Promoted `enabled_api_trigger` from a private function in the HTTP handler to `Automation::enabled_api_trigger()` in the `fabro-automation` crate, where future trigger consumers (scheduler) can reuse it.\n\n**Quality tweaks:**\n- `chrono::Utc::now()` → `Utc::now()` via `use chrono::Utc;` to match the idiom in `runs.rs`.\n- Paginate before decorate in `list_automation_runs` so `decorate_run_summaries` only runs over the returned page instead of all filtered runs.\n\n**Considered and intentionally skipped:**\n- Adding `automation_id` to `fabro_store::ListRunsQuery` to push filtering into the store (would scale better but is explicitly out of scope — issue spec mandates in-memory filtering).\n- Sharing a list-runs envelope/sort helper between `list_runs` and `list_automation_runs` (would require exposing `RunsSortKey`/`RunsSortDirection` and a generic helper; the 4-line inline sort is short enough).\n- Collapsing `RequiredRunToolActor` into `RequiredRunManagementActor` (issue explicitly mandates the `RequiredRunToolActor` name).\n\n**Verification:** `cargo check --workspace --all-targets`, `cargo nextest run -p fabro-automation -p fabro-api` (176/176 pass), `cargo nextest run -p fabro-server --test it --features test-support` (171/171 pass, including all 24 automation API tests), `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` clean, `cargo +nightly-2026-04-14 fmt --check --all` clean. Net change: 37 inserts / 54 deletes across 4 files; `apps/fabro-web` and `lib/crates/fabro-cli` untouched.", - "internal.retry_count.simplify_gpt": 0, - "thread.toolchain.current_node": "preflight_compile", - "internal.fidelity": "compact", - "internal.node_visit_count": 1, - "last_stage": "simplify_gpt", - "internal.retry_count.verify": 0, - "internal.thread_id": "simplify_gpt", - "internal.retry_count.implement": 0, - "failure_class": "", + "response.implement": "Implemented Issue #399 automation run endpoints.\n\nSummary:\n- Added `GET /automations/{id}/runs`\n - Requires automation existence.\n - Lists cached persisted runs only.\n - Filters by `Run.automation.id`.\n - Sorts newest-first.\n - Uses existing `page[limit]` / `page[offset]` pagination envelope.\n\n- Added `POST /automations/{id}/runs`\n - Uses new `RequiredRunToolActor`.\n - Requires automation to exist, be enabled, and have an enabled API trigger.\n - Returns `409` with code `automation_api_trigger_disabled` for disabled automation/API trigger or missing API trigger.\n - Uses configured `AutomationRunMaterializer`.\n - Calls the shared run creation helper with populated `AutomationRef`.\n\n- Added integration coverage using the fake materializer for:\n - Disabled automation.\n - Disabled API trigger.\n - Missing API trigger.\n - Missing automation 404s.\n - Successful API-triggered run creation.\n - Persisted/retrievable `Run.automation`.\n - Automation run listing filtering.\n - Newest-first sorting and pagination.\n\nChanged files:\n- `lib/crates/fabro-server/src/server/handler/automations.rs`\n- `lib/crates/fabro-server/src/principal_middleware.rs`\n- `lib/crates/fabro-server/tests/it/api/automations.rs`\n\nVerification run:\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo nextest run -p fabro-server --features test-support automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo check -p fabro-server` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ empty\n\nNote: the `fabro-server` integration test binary is gated behind the crate’s `test-support` feature, so I used `--features test-support` for the focused `automations` and `openapi_conformance` integration runs.", + "internal.run_id": "01KSNZ26XXV9Y70FDJQHBQ0CC4", + "thread.simplify_gpt.current_node": "verify", "command.output": "blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da", - "graph.model_stylesheet": "\n * { model: claude-opus-4-7; }\n ", - "graph.rankdir": "LR", + "internal.retry_count.simplify_gpt": 0, + "internal.fidelity": "compact", "internal.retry_count.toolchain": 0, - "internal.retry_count.preflight_lint": 0, - "thread.preflight_lint.current_node": "implement", - "failure_signature": "", + "graph.model_stylesheet": "\n * { model: claude-opus-4-7; }\n ", "internal.retry_count.start": 0, - "thread.simplify_opus.current_node": "simplify_gpt", - "current_node": "verify", + "thread.start.current_node": "toolchain", + "failure_signature": "", + "graph.goal": "# Issue #399: Add automation run endpoints\n\n- URL: https://github.com/fabro-sh/fabro/issues/399\n- State: OPEN\n- Author: Bryan Helmkamp (@brynary)\n- Created: 2026-05-25T15:06:27Z\n- Updated: 2026-05-25T15:06:27Z\n- Labels: None\n- Assignees: None\n- Milestone: None\n- Comments: 0\n\n---\n\n## Goal\n\nExpose API endpoints for listing runs associated with an automation and starting a run through an enabled API trigger.\n\n## Scope\n\nImplement these endpoints:\n\n```http\nGET /automations/{id}/runs\nPOST /automations/{id}/runs\n```\n\n`GET /automations/{id}/runs` behavior:\n\n- Require the automation definition to exist; return 404 when it does not.\n- List cached runs from the existing run store.\n- Filter by `run.automation.as_ref().is_some_and(|a| a.id == id)`.\n- Sort newest first.\n- Support `page[limit]` and `page[offset]` using existing pagination behavior.\n- Return the existing paginated run list envelope:\n\n```json\n{\n \"data\": [],\n \"meta\": { \"has_more\": false, \"total\": 0 }\n}\n```\n\n`POST /automations/{id}/runs` behavior:\n\n- Use `RequiredRunToolActor`.\n- Require the automation to exist and be enabled.\n- Find an enabled trigger where `type = \"api\"`.\n- Return 409 with API error code `automation_api_trigger_disabled` when the automation is disabled or no enabled API trigger is available.\n- Materialize the run manifest using the configured `AutomationRunMaterializer`.\n- Call the shared create-run helper with:\n\n```rust\nAutomationRef {\n id: automation.id.to_string(),\n name: Some(automation.name.clone()),\n trigger_id: Some(api_trigger.id.to_string()),\n}\n```\n\n- Return 201 and the normal `Run` response shape with automation metadata populated.\n\nFinal integration expectations:\n\n- Automation-created runs are visible through normal run APIs.\n- Automation-created runs are visible through `GET /automations/{id}/runs`.\n- Run history is derived from persisted/cached runs; no runtime automation state store is introduced.\n- Schedule trigger expressions are stored and validated by earlier phases but are not scheduled by this endpoint work.\n\n## Files\n\nModify:\n\n- `lib/crates/fabro-server/src/server/handler/automations.rs`\n- `lib/crates/fabro-server/src/server/handler/runs.rs`, only if additional helper exposure is needed from the previous phase\n- `lib/crates/fabro-server/tests/it/api/automations.rs`\n- `lib/crates/fabro-server/tests/it/api/mod.rs`\n\n## Acceptance Criteria\n\n- Disabled automations cannot start runs through the automation run endpoint.\n- Automations without an enabled API trigger cannot start runs through the automation run endpoint.\n- A successful API-triggered automation run returns a normal `Run` response with `automation.id`, `automation.name`, and `automation.trigger_id`.\n- The automation run listing endpoint returns only runs linked to that automation.\n- Automation run listings are newest-first and paginate correctly.\n- No scheduler, web UI route/component, or CLI command is added.\n\n## Verification\n\nAdd integration tests using the fake materializer for:\n\n- Disabled automation returns 409.\n- Disabled API trigger returns 409.\n- Missing API trigger returns 409.\n- Successful run creation returns 201.\n- Created run persists `Run.automation`.\n- Associated run listing includes the run.\n- Run listing excludes runs from other automations.\n- Run listing pagination and newest-first sorting.\n\nRun:\n\n```bash\ncargo nextest run -p fabro-automation\ncargo nextest run -p fabro-api\ncargo nextest run -p fabro-server automations\ncargo nextest run -p fabro-server openapi_conformance\ncargo +nightly-2026-04-14 fmt --check --all\ncargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings\ngit diff -- apps/fabro-web lib/crates/fabro-cli\n```\n\nExpected: focused tests and checks pass; web UI and CLI command modules remain unchanged.\n", "internal.retry_count.simplify_opus": 0, "internal.retry_count.preflight_compile": 0, - "internal.run_id": "01KSNZ26XXV9Y70FDJQHBQ0CC4", - "outcome": "succeeded", + "internal.retry_count.preflight_lint": 0, + "internal.thread_id": "simplify_gpt", "thread.preflight_compile.current_node": "preflight_lint", - "thread.implement.current_node": "simplify_opus", - "internal.work_dir": "/home/daytona/workspace/fabro", - "response.simplify_gpt": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionable findings, so I also performed a direct reuse/quality/efficiency review of the diff.\n- No cleanup changes were needed; implementation looked focused and consistent.\n\nVerification run:\n\n- `cargo nextest run -p fabro-server --features test-support api::automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ no output\n- Final `git status --short` ✅ clean\n\nNote: Running `cargo nextest run -p fabro-server automations` without `--features test-support` found no integration tests because the `it` test target requires the `test-support` feature, so I reran with the required feature enabled.", - "thread.start.current_node": "toolchain", - "graph.goal": "# Issue #399: Add automation run endpoints\n\n- URL: https://github.com/fabro-sh/fabro/issues/399\n- State: OPEN\n- Author: Bryan Helmkamp (@brynary)\n- Created: 2026-05-25T15:06:27Z\n- Updated: 2026-05-25T15:06:27Z\n- Labels: None\n- Assignees: None\n- Milestone: None\n- Comments: 0\n\n---\n\n## Goal\n\nExpose API endpoints for listing runs associated with an automation and starting a run through an enabled API trigger.\n\n## Scope\n\nImplement these endpoints:\n\n```http\nGET /automations/{id}/runs\nPOST /automations/{id}/runs\n```\n\n`GET /automations/{id}/runs` behavior:\n\n- Require the automation definition to exist; return 404 when it does not.\n- List cached runs from the existing run store.\n- Filter by `run.automation.as_ref().is_some_and(|a| a.id == id)`.\n- Sort newest first.\n- Support `page[limit]` and `page[offset]` using existing pagination behavior.\n- Return the existing paginated run list envelope:\n\n```json\n{\n \"data\": [],\n \"meta\": { \"has_more\": false, \"total\": 0 }\n}\n```\n\n`POST /automations/{id}/runs` behavior:\n\n- Use `RequiredRunToolActor`.\n- Require the automation to exist and be enabled.\n- Find an enabled trigger where `type = \"api\"`.\n- Return 409 with API error code `automation_api_trigger_disabled` when the automation is disabled or no enabled API trigger is available.\n- Materialize the run manifest using the configured `AutomationRunMaterializer`.\n- Call the shared create-run helper with:\n\n```rust\nAutomationRef {\n id: automation.id.to_string(),\n name: Some(automation.name.clone()),\n trigger_id: Some(api_trigger.id.to_string()),\n}\n```\n\n- Return 201 and the normal `Run` response shape with automation metadata populated.\n\nFinal integration expectations:\n\n- Automation-created runs are visible through normal run APIs.\n- Automation-created runs are visible through `GET /automations/{id}/runs`.\n- Run history is derived from persisted/cached runs; no runtime automation state store is introduced.\n- Schedule trigger expressions are stored and validated by earlier phases but are not scheduled by this endpoint work.\n\n## Files\n\nModify:\n\n- `lib/crates/fabro-server/src/server/handler/automations.rs`\n- `lib/crates/fabro-server/src/server/handler/runs.rs`, only if additional helper exposure is needed from the previous phase\n- `lib/crates/fabro-server/tests/it/api/automations.rs`\n- `lib/crates/fabro-server/tests/it/api/mod.rs`\n\n## Acceptance Criteria\n\n- Disabled automations cannot start runs through the automation run endpoint.\n- Automations without an enabled API trigger cannot start runs through the automation run endpoint.\n- A successful API-triggered automation run returns a normal `Run` response with `automation.id`, `automation.name`, and `automation.trigger_id`.\n- The automation run listing endpoint returns only runs linked to that automation.\n- Automation run listings are newest-first and paginate correctly.\n- No scheduler, web UI route/component, or CLI command is added.\n\n## Verification\n\nAdd integration tests using the fake materializer for:\n\n- Disabled automation returns 409.\n- Disabled API trigger returns 409.\n- Missing API trigger returns 409.\n- Successful run creation returns 201.\n- Created run persists `Run.automation`.\n- Associated run listing includes the run.\n- Run listing excludes runs from other automations.\n- Run listing pagination and newest-first sorting.\n\nRun:\n\n```bash\ncargo nextest run -p fabro-automation\ncargo nextest run -p fabro-api\ncargo nextest run -p fabro-server automations\ncargo nextest run -p fabro-server openapi_conformance\ncargo +nightly-2026-04-14 fmt --check --all\ncargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings\ngit diff -- apps/fabro-web lib/crates/fabro-cli\n```\n\nExpected: focused tests and checks pass; web UI and CLI command modules remain unchanged.\n", - "thread.simplify_gpt.current_node": "verify", + "failure_class": "", + "internal.node_visit_count": 1, "last_response": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionabl", - "response.implement": "Implemented Issue #399 automation run endpoints.\n\nSummary:\n- Added `GET /automations/{id}/runs`\n - Requires automation existence.\n - Lists cached persisted runs only.\n - Filters by `Run.automation.id`.\n - Sorts newest-first.\n - Uses existing `page[limit]` / `page[offset]` pagination envelope.\n\n- Added `POST /automations/{id}/runs`\n - Uses new `RequiredRunToolActor`.\n - Requires automation to exist, be enabled, and have an enabled API trigger.\n - Returns `409` with code `automation_api_trigger_disabled` for disabled automation/API trigger or missing API trigger.\n - Uses configured `AutomationRunMaterializer`.\n - Calls the shared run creation helper with populated `AutomationRef`.\n\n- Added integration coverage using the fake materializer for:\n - Disabled automation.\n - Disabled API trigger.\n - Missing API trigger.\n - Missing automation 404s.\n - Successful API-triggered run creation.\n - Persisted/retrievable `Run.automation`.\n - Automation run listing filtering.\n - Newest-first sorting and pagination.\n\nChanged files:\n- `lib/crates/fabro-server/src/server/handler/automations.rs`\n- `lib/crates/fabro-server/src/principal_middleware.rs`\n- `lib/crates/fabro-server/tests/it/api/automations.rs`\n\nVerification run:\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo nextest run -p fabro-server --features test-support automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo check -p fabro-server` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ empty\n\nNote: the `fabro-server` integration test binary is gated behind the crate’s `test-support` feature, so I used `--features test-support` for the focused `automations` and `openapi_conformance` integration runs." + "last_stage": "simplify_gpt", + "response.simplify_gpt": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionable findings, so I also performed a direct reuse/quality/efficiency review of the diff.\n- No cleanup changes were needed; implementation looked focused and consistent.\n\nVerification run:\n\n- `cargo nextest run -p fabro-server --features test-support api::automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ no output\n- Final `git status --short` ✅ clean\n\nNote: Running `cargo nextest run -p fabro-server automations` without `--features test-support` found no integration tests because the `it` test target requires the `test-support` feature, so I reran with the required feature enabled.", + "graph.rankdir": "LR", + "internal.retry_count.verify": 0, + "current_node": "verify", + "thread.implement.current_node": "simplify_opus", + "thread.simplify_opus.current_node": "simplify_gpt", + "internal.retry_count.implement": 0, + "internal.work_dir": "/home/daytona/workspace/fabro", + "response.simplify_opus": "Web UI and CLI remain untouched.\n\n## Summary\n\nReviewed 3 changed files via parallel reuse/quality/efficiency agents and applied the following cleanups:\n\n**Removed dead/redundant code:**\n- Deleted `require_run_tool_actor` helper in `principal_middleware.rs` — was a trivial alias for `require_run_management_actor`; `RequiredRunToolActor::from_request_parts` now calls the real function directly.\n- Removed stale `#[allow(dead_code, reason = \"...\")]` on `AppState::materialize_automation_run` in `server.rs` (now called from `create_automation_run`).\n\n**Inlined single-use helpers in `automations.rs`:**\n- `automation_api_trigger_disabled_error` + the matching const `AUTOMATION_API_TRIGGER_DISABLED_CODE` — only used by one `else` branch.\n- `automation_materialization_temp_root` — one-line `Storage::new(...).scratch_dir().join(\"automations\")`.\n- `automation_materialize_error` — one-line wrapper around `ApiError::new(UNPROCESSABLE_ENTITY, …)`.\n\n**Moved logic to its proper home:**\n- Promoted `enabled_api_trigger` from a private function in the HTTP handler to `Automation::enabled_api_trigger()` in the `fabro-automation` crate, where future trigger consumers (scheduler) can reuse it.\n\n**Quality tweaks:**\n- `chrono::Utc::now()` → `Utc::now()` via `use chrono::Utc;` to match the idiom in `runs.rs`.\n- Paginate before decorate in `list_automation_runs` so `decorate_run_summaries` only runs over the returned page instead of all filtered runs.\n\n**Considered and intentionally skipped:**\n- Adding `automation_id` to `fabro_store::ListRunsQuery` to push filtering into the store (would scale better but is explicitly out of scope — issue spec mandates in-memory filtering).\n- Sharing a list-runs envelope/sort helper between `list_runs` and `list_automation_runs` (would require exposing `RunsSortKey`/`RunsSortDirection` and a generic helper; the 4-line inline sort is short enough).\n- Collapsing `RequiredRunToolActor` into `RequiredRunManagementActor` (issue explicitly mandates the `RequiredRunToolActor` name).\n\n**Verification:** `cargo check --workspace --all-targets`, `cargo nextest run -p fabro-automation -p fabro-api` (176/176 pass), `cargo nextest run -p fabro-server --test it --features test-support` (171/171 pass, including all 24 automation API tests), `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` clean, `cargo +nightly-2026-04-14 fmt --check --all` clean. Net change: 37 inserts / 54 deletes across 4 files; `apps/fabro-web` and `lib/crates/fabro-cli` untouched.", + "outcome": "succeeded", + "thread.preflight_lint.current_node": "implement", + "thread.toolchain.current_node": "preflight_compile" }, "node_outcomes": { - "toolchain": { + "preflight_compile": { "status": "succeeded", "context_updates": { - "command.output": "blob://sha256/fc14b2ba2d770e5cd3169df7a29525c962adfc4cfa3097b9098c63ebd61a748c" + "command.output": "blob://sha256/12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126" }, - "notes": "Script completed: command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", + "notes": "Script completed: cargo check -q --workspace 2>&1", "usage": null, "timing": { "wall_time_ms": 0, "inference_time_ms": 0, - "tool_time_ms": 1398, - "active_time_ms": 1398 + "tool_time_ms": 128511, + "active_time_ms": 128511 + } + }, + "preflight_lint": { + "status": "succeeded", + "context_updates": { + "command.output": "blob://sha256/12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126" + }, + "notes": "Script completed: cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", + "usage": null, + "timing": { + "wall_time_ms": 0, + "inference_time_ms": 0, + "tool_time_ms": 143345, + "active_time_ms": 143345 } }, "simplify_opus": { @@ -1465,18 +1479,68 @@ "active_time_ms": 664607 } }, - "preflight_lint": { + "simplify_gpt": { "status": "succeeded", "context_updates": { - "command.output": "blob://sha256/12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126" + "response.simplify_gpt": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionable findings, so I also performed a direct reuse/quality/efficiency review of the diff.\n- No cleanup changes were needed; implementation looked focused and consistent.\n\nVerification run:\n\n- `cargo nextest run -p fabro-server --features test-support api::automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ no output\n- Final `git status --short` ✅ clean\n\nNote: Running `cargo nextest run -p fabro-server automations` without `--features test-support` found no integration tests because the `it` test target requires the `test-support` feature, so I reran with the required feature enabled.", + "last_stage": "simplify_gpt", + "last_response": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionabl" }, - "notes": "Script completed: cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", + "notes": "Stage completed: simplify_gpt", + "usage": { + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-5.5" + }, + "tokens": { + "input_tokens": 383859, + "output_tokens": 3670, + "reasoning_tokens": 1261, + "cache_read_tokens": 1061888, + "cache_write_tokens": 0 + } + }, + "facts": { + "algorithm": "openai" + } + }, + "total_usd_micros": 2598169 + }, + "timing": { + "wall_time_ms": 0, + "inference_time_ms": 201837, + "tool_time_ms": 97609, + "active_time_ms": 299446 + } + }, + "toolchain": { + "status": "succeeded", + "context_updates": { + "command.output": "blob://sha256/fc14b2ba2d770e5cd3169df7a29525c962adfc4cfa3097b9098c63ebd61a748c" + }, + "notes": "Script completed: command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", "usage": null, "timing": { "wall_time_ms": 0, "inference_time_ms": 0, - "tool_time_ms": 143345, - "active_time_ms": 143345 + "tool_time_ms": 1398, + "active_time_ms": 1398 + } + }, + "verify": { + "status": "succeeded", + "context_updates": { + "command.output": "blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da" + }, + "notes": "Script completed: git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\\bActorRef\\b|\\bActorKind\\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\\s*==\\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", + "usage": null, + "timing": { + "wall_time_ms": 0, + "inference_time_ms": 0, + "tool_time_ms": 544732, + "active_time_ms": 544732 } }, "implement": { @@ -1515,91 +1579,149 @@ "active_time_ms": 1245980 } }, - "preflight_compile": { - "status": "succeeded", - "context_updates": { - "command.output": "blob://sha256/12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126" - }, - "notes": "Script completed: cargo check -q --workspace 2>&1", - "usage": null, - "timing": { - "wall_time_ms": 0, - "inference_time_ms": 0, - "tool_time_ms": 128511, - "active_time_ms": 128511 - } - }, - "verify": { - "status": "succeeded", - "context_updates": { - "command.output": "blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da" - }, - "notes": "Script completed: git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\\bActorRef\\b|\\bActorKind\\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\\s*==\\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", - "usage": null, - "timing": { - "wall_time_ms": 0, - "inference_time_ms": 0, - "tool_time_ms": 544732, - "active_time_ms": 544732 - } - }, "start": { "status": "succeeded", "usage": null - }, - "simplify_gpt": { - "status": "succeeded", - "context_updates": { - "response.simplify_gpt": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionable findings, so I also performed a direct reuse/quality/efficiency review of the diff.\n- No cleanup changes were needed; implementation looked focused and consistent.\n\nVerification run:\n\n- `cargo nextest run -p fabro-server --features test-support api::automations` ✅\n- `cargo nextest run -p fabro-server --features test-support openapi_conformance` ✅\n- `cargo nextest run -p fabro-automation` ✅\n- `cargo nextest run -p fabro-api` ✅\n- `cargo +nightly-2026-04-14 fmt --check --all` ✅\n- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` ✅\n- `git diff -- apps/fabro-web lib/crates/fabro-cli` ✅ no output\n- Final `git status --short` ✅ clean\n\nNote: Running `cargo nextest run -p fabro-server automations` without `--features test-support` found no integration tests because the `it` test target requires the `test-support` feature, so I reran with the required feature enabled.", - "last_stage": "simplify_gpt", - "last_response": "Review complete.\n\n- Working tree was already clean; reviewed committed changes vs `origin/main`.\n- Launched the three requested review agents. The tool reported completion but did not return actionabl" - }, - "notes": "Stage completed: simplify_gpt", - "usage": { - "input": { - "usage": { - "model": { - "provider": "openai", - "model_id": "gpt-5.5" - }, - "tokens": { - "input_tokens": 383859, - "output_tokens": 3670, - "reasoning_tokens": 1261, - "cache_read_tokens": 1061888, - "cache_write_tokens": 0 - } - }, - "facts": { - "algorithm": "openai" - } - }, - "total_usd_micros": 2598169 - }, - "timing": { - "wall_time_ms": 0, - "inference_time_ms": 201837, - "tool_time_ms": 97609, - "active_time_ms": 299446 - } } }, "next_node_id": "exit", + "git_commit_sha": "2446175b1e9f9e836b6ca93acdb0c53d9279a8cd", "node_visits": { "toolchain": 1, - "simplify_gpt": 1, "implement": 1, - "preflight_lint": 1, + "simplify_gpt": 1, "preflight_compile": 1, "simplify_opus": 1, - "start": 1, - "verify": 1 + "preflight_lint": 1, + "verify": 1, + "start": 1 } }, - "diff": {} + "diff": { + "patch": "diff --git a/Cargo.lock b/Cargo.lock\nindex 0986b55a6..0fdc7afc9 100644\n--- a/Cargo.lock\n+++ b/Cargo.lock\n@@ -1889,6 +1889,7 @@ dependencies = [\n \"tempfile\",\n \"thiserror 2.0.18\",\n \"tokio\",\n+ \"tokio-tungstenite 0.26.2\",\n \"tokio-util\",\n \"toml 0.8.23\",\n \"toml_edit\",\n@@ -2408,6 +2409,7 @@ dependencies = [\n \"thiserror 2.0.18\",\n \"tokio\",\n \"tokio-stream\",\n+ \"tokio-tungstenite 0.26.2\",\n \"tokio-util\",\n \"toml 0.8.23\",\n \"toml_edit\",\ndiff --git a/lib/crates/fabro-cli/Cargo.toml b/lib/crates/fabro-cli/Cargo.toml\nindex a4d15b5b5..d0b8b7373 100644\n--- a/lib/crates/fabro-cli/Cargo.toml\n+++ b/lib/crates/fabro-cli/Cargo.toml\n@@ -61,6 +61,7 @@ anyhow.workspace = true\n miette.workspace = true\n dotenvy.workspace = true\n tokio.workspace = true\n+tokio-tungstenite.workspace = true\n tracing.workspace = true\n tracing-subscriber.workspace = true\n tracing-appender.workspace = true\n@@ -128,3 +129,4 @@ fabro-test = { workspace = true }\n fabro-macros = { path = \"../fabro-macros\" }\n hkdf.workspace = true\n reqwest = { workspace = true, features = [\"cookies\"] }\n+tokio = { workspace = true, features = [\"test-util\", \"macros\"] }\ndiff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs\nindex fb3b31d43..ac1a2bbd0 100644\n--- a/lib/crates/fabro-cli/src/commands/run/runner.rs\n+++ b/lib/crates/fabro-cli/src/commands/run/runner.rs\n@@ -1,10 +1,4 @@\n-#![expect(\n- clippy::disallowed_types,\n- reason = \"sync CLI `run` subprocess wrapper: reads server subprocess stdout line-by-line via \\\n- std::io::BufReader; not on a Tokio path\"\n-)]\n-\n-use std::io::{BufRead as StdBufRead, BufReader as StdBufReader};\n+use std::collections::{HashSet, VecDeque};\n use std::path::{Path, PathBuf};\n use std::sync::Arc;\n use std::time::Duration;\n@@ -12,10 +6,14 @@ use std::time::Duration;\n use anyhow::{Context, Result, anyhow};\n use async_trait::async_trait;\n use fabro_api::types::RunManifest;\n+use fabro_client::ServerTarget;\n use fabro_config::user::active_settings_path;\n use fabro_config::{ServerSettingsBuilder, Storage, load_llm_catalog_settings};\n use fabro_interview::{\n- AnswerSubmission, ControlInterviewer, WorkerControlEnvelope, WorkerControlMessage,\n+ AnswerSubmission, ControlInterviewer, WORKER_CONTROL_INVALID_CURSOR_REASON,\n+ WORKER_CONTROL_PONG_TIMEOUT_REASON, WORKER_CONTROL_WS_LIVENESS_TIMEOUT,\n+ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlDeliveryFrame, WorkerControlEnvelope,\n+ WorkerControlMessage,\n };\n use fabro_model::Catalog;\n use fabro_server::run_tool_manifest;\n@@ -34,11 +32,23 @@ use fabro_workflow::operations::{self, StartServices};\n use fabro_workflow::run_control::RunControlState;\n use fabro_workflow::runtime_store::{RunStoreBackend, RunStoreHandle};\n use fabro_workflow::services::FabroRunToolServices;\n+use futures::{SinkExt, StreamExt};\n use jsonwebtoken::dangerous::insecure_decode;\n+#[cfg(test)]\n+use tokio::io::DuplexStream;\n+use tokio::net::TcpStream;\n+#[cfg(unix)]\n+use tokio::net::UnixStream;\n #[cfg(unix)]\n use tokio::signal::unix::{SignalKind, signal};\n-use tokio::sync::{Mutex, RwLock as AsyncRwLock, mpsc};\n-use tokio::time::sleep;\n+use tokio::sync::{Mutex, RwLock as AsyncRwLock, oneshot};\n+use tokio::task::JoinHandle;\n+use tokio::time::{self, Instant, MissedTickBehavior};\n+use tokio_tungstenite::tungstenite::client::IntoClientRequest;\n+use tokio_tungstenite::tungstenite::http::{HeaderValue, Request as WebSocketRequest, header};\n+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;\n+use tokio_tungstenite::tungstenite::protocol::{self, Message as WebSocketMessage};\n+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async, tungstenite};\n use tokio_util::sync::CancellationToken;\n \n use crate::args::RunWorkerMode;\n@@ -75,7 +85,7 @@ pub(crate) async fn execute(\n let _ = fabro_proc::title_init();\n set_worker_title(&run_id, initial_worker_title_phase(mode));\n \n- let target = server.parse::()?;\n+ let target = server.parse::()?;\n let client = server_client::connect_server_target_with_bearer(&target, worker_token).await?;\n let run_store = HttpRunStore::connect(run_id, client.clone_for_reuse()).await?;\n let run_state = run_store\n@@ -110,13 +120,24 @@ pub(crate) async fn execute(\n let cancel_token = CancellationToken::new();\n let emitter = Arc::new(Emitter::new(run_id));\n let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(Arc::clone(&emitter)));\n- spawn_worker_control_stream(\n- Arc::clone(&interviewer),\n- cancel_token.clone(),\n- Arc::clone(&steering_hub),\n- )?;\n let run_control = RunControlState::new();\n install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?;\n+ let mut control_manager = if run_state.status.is_terminal() {\n+ None\n+ } else {\n+ Some(spawn_worker_control_manager(\n+ target.clone(),\n+ run_id,\n+ worker_token.to_owned(),\n+ Arc::clone(&interviewer),\n+ cancel_token.clone(),\n+ Arc::clone(&steering_hub),\n+ Arc::clone(&run_control),\n+ ))\n+ };\n+ if let Some(control_manager) = &mut control_manager {\n+ control_manager.wait_for_first_connection().await?;\n+ }\n let vault = load_worker_vault(storage_dir.as_deref())?;\n let github_app = {\n let vault_guard = match &vault {\n@@ -158,13 +179,26 @@ pub(crate) async fn execute(\n fabro_run_tools,\n };\n \n- match mode {\n- RunWorkerMode::Start => {\n- operations::start(&run_dir, services).await?;\n+ let execution = async {\n+ match mode {\n+ RunWorkerMode::Start => operations::start(&run_dir, services).await,\n+ RunWorkerMode::Resume => operations::resume(&run_dir, services).await,\n }\n- RunWorkerMode::Resume => {\n- operations::resume(&run_dir, services).await?;\n+ };\n+\n+ if let Some(mut control_manager) = control_manager {\n+ tokio::select! {\n+ result = execution => {\n+ control_manager.finish();\n+ result?;\n+ }\n+ fatal = control_manager.fatal_control_loss() => {\n+ control_manager.finish();\n+ return Err(fatal);\n+ }\n }\n+ } else {\n+ execution.await?;\n }\n \n Ok(())\n@@ -254,96 +288,510 @@ fn load_worker_vault(storage_dir: Option<&Path>) -> Result,\n- cancel_token: CancellationToken,\n- steering_hub: Arc,\n-) -> Result<()> {\n- let (event_tx, event_rx) = mpsc::unbounded_channel();\n- tokio::spawn(handle_worker_control_stream_events(\n- interviewer,\n- cancel_token,\n- steering_hub,\n- event_rx,\n- ));\n- std::thread::Builder::new()\n- .name(\"fabro-worker-control\".to_string())\n- .spawn(move || {\n- read_worker_control_stream_blocking(StdBufReader::new(std::io::stdin()), &event_tx);\n- })\n- .context(\"failed to spawn worker control reader thread\")?;\n- Ok(())\n+#[derive(Default)]\n+struct AppliedWorkerControlDeliveryIds {\n+ last: Option,\n+ order: VecDeque,\n+ recent: HashSet,\n }\n \n-fn read_worker_control_stream_blocking(\n- mut reader: R,\n- event_tx: &mpsc::UnboundedSender,\n-) where\n- R: StdBufRead,\n-{\n- let mut line = String::new();\n- loop {\n- line.clear();\n- match reader.read_line(&mut line) {\n- Ok(0) | Err(_) => {\n- let _ = event_tx.send(WorkerControlStreamEvent::Eof);\n- break;\n- }\n- Ok(_) => {\n- let line = line.trim_end_matches(['\\r', '\\n']).to_string();\n- if event_tx.send(WorkerControlStreamEvent::Line(line)).is_err() {\n- break;\n- }\n+impl AppliedWorkerControlDeliveryIds {\n+ fn last_applied_id(&self) -> Option<&str> {\n+ self.last.as_deref()\n+ }\n+\n+ fn contains(&self, id: &str) -> bool {\n+ self.recent.contains(id)\n+ }\n+\n+ fn record(&mut self, id: String) {\n+ if !self.recent.insert(id.clone()) {\n+ self.last = Some(id);\n+ return;\n+ }\n+ self.order.push_back(id.clone());\n+ self.last = Some(id);\n+ while self.order.len() > WORKER_CONTROL_APPLIED_ID_DEDUPE_CAPACITY {\n+ if let Some(evicted) = self.order.pop_front() {\n+ self.recent.remove(&evicted);\n }\n }\n }\n }\n \n-async fn handle_worker_control_stream_events(\n+struct WorkerControlManagerHandle {\n+ first_connection: Option>>,\n+ fatal: Option>,\n+ done: CancellationToken,\n+ task: JoinHandle<()>,\n+}\n+\n+impl WorkerControlManagerHandle {\n+ async fn wait_for_first_connection(&mut self) -> Result<()> {\n+ let receiver = self\n+ .first_connection\n+ .take()\n+ .context(\"worker control first-connection receiver missing\")?;\n+ receiver\n+ .await\n+ .context(\"worker control manager stopped before first connection\")?\n+ }\n+\n+ async fn fatal_control_loss(&mut self) -> anyhow::Error {\n+ let Some(receiver) = self.fatal.take() else {\n+ return anyhow!(\"worker control fatal receiver missing\");\n+ };\n+ receiver\n+ .await\n+ .unwrap_or_else(|_| anyhow!(\"worker control manager stopped before workflow completed\"))\n+ }\n+\n+ fn finish(&self) {\n+ self.done.cancel();\n+ self.task.abort();\n+ }\n+}\n+\n+#[derive(Debug)]\n+struct WorkerControlStreamConnectRequest {\n+ request: WebSocketRequest<()>,\n+ unix_socket_path: Option,\n+}\n+\n+impl WorkerControlStreamConnectRequest {\n+ fn request_for_tungstenite(&self) -> WebSocketRequest<()> {\n+ self.request.clone()\n+ }\n+\n+ #[cfg(test)]\n+ fn uri(&self) -> String {\n+ self.request.uri().to_string()\n+ }\n+\n+ #[cfg(test)]\n+ fn authorization(&self) -> Option<&str> {\n+ self.request\n+ .headers()\n+ .get(header::AUTHORIZATION)\n+ .and_then(|value| value.to_str().ok())\n+ }\n+}\n+\n+enum WorkerControlSocket {\n+ Tcp(Box>>),\n+ #[cfg(unix)]\n+ Unix(Box>),\n+ #[cfg(test)]\n+ Test(Box>),\n+}\n+\n+impl WorkerControlSocket {\n+ async fn send(&mut self, message: WebSocketMessage) -> Result<(), tungstenite::Error> {\n+ match self {\n+ Self::Tcp(socket) => socket.send(message).await,\n+ #[cfg(unix)]\n+ Self::Unix(socket) => socket.send(message).await,\n+ #[cfg(test)]\n+ Self::Test(socket) => socket.send(message).await,\n+ }\n+ }\n+\n+ async fn next(&mut self) -> Option> {\n+ match self {\n+ Self::Tcp(socket) => socket.next().await,\n+ #[cfg(unix)]\n+ Self::Unix(socket) => socket.next().await,\n+ #[cfg(test)]\n+ Self::Test(socket) => socket.next().await,\n+ }\n+ }\n+}\n+\n+#[derive(Debug)]\n+enum WorkerControlConnectError {\n+ InvalidCursor,\n+ Other(anyhow::Error),\n+}\n+\n+fn spawn_worker_control_manager(\n+ target: ServerTarget,\n+ run_id: RunId,\n+ worker_token: String,\n+ interviewer: Arc,\n+ cancel_token: CancellationToken,\n+ steering_hub: Arc,\n+ run_control: Arc,\n+) -> WorkerControlManagerHandle {\n+ let (first_tx, first_rx) = oneshot::channel();\n+ let (fatal_tx, fatal_rx) = oneshot::channel();\n+ let done = CancellationToken::new();\n+ let task_done = done.clone();\n+ let task = tokio::spawn(async move {\n+ run_worker_control_manager(\n+ target,\n+ run_id,\n+ worker_token,\n+ interviewer,\n+ cancel_token,\n+ steering_hub,\n+ run_control,\n+ task_done,\n+ first_tx,\n+ fatal_tx,\n+ )\n+ .await;\n+ });\n+ WorkerControlManagerHandle {\n+ first_connection: Some(first_rx),\n+ fatal: Some(fatal_rx),\n+ done,\n+ task,\n+ }\n+}\n+\n+#[allow(\n+ clippy::too_many_arguments,\n+ reason = \"Worker control manager owns the worker-side control dependencies.\"\n+)]\n+async fn run_worker_control_manager(\n+ target: ServerTarget,\n+ run_id: RunId,\n+ worker_token: String,\n interviewer: Arc,\n cancel_token: CancellationToken,\n steering_hub: Arc,\n- mut event_rx: mpsc::UnboundedReceiver,\n+ run_control: Arc,\n+ done: CancellationToken,\n+ first_tx: oneshot::Sender>,\n+ fatal_tx: oneshot::Sender,\n ) {\n- while let Some(event) = event_rx.recv().await {\n- match event {\n- WorkerControlStreamEvent::Line(line) => {\n- apply_worker_control_line(&interviewer, &cancel_token, &steering_hub, &line).await;\n+ let mut first_tx = Some(first_tx);\n+ let mut fatal_tx = Some(fatal_tx);\n+ let mut backoff = WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF;\n+ let mut applied_ids = AppliedWorkerControlDeliveryIds::default();\n+\n+ while !done.is_cancelled() {\n+ let request = match build_worker_control_stream_request(\n+ &target,\n+ &run_id,\n+ &worker_token,\n+ applied_ids.last_applied_id(),\n+ ) {\n+ Ok(request) => request,\n+ Err(err) => {\n+ report_fatal_control_loss(\n+ &interviewer,\n+ &cancel_token,\n+ &mut first_tx,\n+ &mut fatal_tx,\n+ format!(\"failed to build worker control stream request: {err:#}\"),\n+ )\n+ .await;\n+ return;\n+ }\n+ };\n+\n+ match connect_worker_control_stream(request).await {\n+ Ok(mut socket) => {\n+ if let Some(first_tx) = first_tx.take() {\n+ let _ = first_tx.send(Ok(()));\n+ }\n+ backoff = WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF;\n+ match handle_worker_control_socket(\n+ &mut socket,\n+ &interviewer,\n+ &cancel_token,\n+ &steering_hub,\n+ &run_control,\n+ &mut applied_ids,\n+ &done,\n+ )\n+ .await\n+ {\n+ Ok(()) => {}\n+ Err(WorkerControlConnectError::InvalidCursor) => {\n+ report_fatal_control_loss(\n+ &interviewer,\n+ &cancel_token,\n+ &mut first_tx,\n+ &mut fatal_tx,\n+ \"worker control stream replay cursor is invalid\".to_string(),\n+ )\n+ .await;\n+ return;\n+ }\n+ Err(WorkerControlConnectError::Other(err)) => {\n+ tracing::debug!(error = %err, \"Worker control stream disconnected\");\n+ }\n+ }\n }\n- WorkerControlStreamEvent::Eof => {\n- interviewer.interrupt_all().await;\n+ Err(WorkerControlConnectError::InvalidCursor) => {\n+ report_fatal_control_loss(\n+ &interviewer,\n+ &cancel_token,\n+ &mut first_tx,\n+ &mut fatal_tx,\n+ \"worker control stream replay cursor is invalid\".to_string(),\n+ )\n+ .await;\n return;\n }\n+ Err(WorkerControlConnectError::Other(err)) => {\n+ tracing::debug!(error = %err, \"Worker control stream connection failed\");\n+ }\n }\n+\n+ sleep_or_done(&done, backoff).await;\n+ backoff = next_worker_control_reconnect_backoff(backoff);\n }\n+}\n \n+async fn report_fatal_control_loss(\n+ interviewer: &ControlInterviewer,\n+ cancel_token: &CancellationToken,\n+ first_tx: &mut Option>>,\n+ fatal_tx: &mut Option>,\n+ detail: String,\n+) {\n+ let message = format!(\"worker control channel lost: {detail}\");\n interviewer.interrupt_all().await;\n+ if let Some(first_tx) = first_tx.take() {\n+ let _ = first_tx.send(Err(anyhow!(message.clone())));\n+ }\n+ if let Some(fatal_tx) = fatal_tx.take() {\n+ let _ = fatal_tx.send(anyhow!(message));\n+ }\n+ cancel_token.cancel();\n+}\n+\n+async fn sleep_or_done(done: &CancellationToken, delay: Duration) {\n+ tokio::select! {\n+ () = done.cancelled() => {}\n+ () = time::sleep(delay) => {}\n+ }\n+}\n+\n+fn next_worker_control_reconnect_backoff(current: Duration) -> Duration {\n+ current\n+ .saturating_mul(2)\n+ .min(WORKER_CONTROL_RECONNECT_MAX_BACKOFF)\n+}\n+\n+fn build_worker_control_stream_request(\n+ target: &ServerTarget,\n+ run_id: &RunId,\n+ worker_token: &str,\n+ after: Option<&str>,\n+) -> Result {\n+ let (url, unix_socket_path) = match target {\n+ ServerTarget::HttpUrl(_) => {\n+ let base = target\n+ .as_http_url()\n+ .context(\"HTTP server target missing URL\")?;\n+ let websocket_base = if let Some(rest) = base.strip_prefix(\"http://\") {\n+ format!(\"ws://{rest}\")\n+ } else if let Some(rest) = base.strip_prefix(\"https://\") {\n+ format!(\"wss://{rest}\")\n+ } else {\n+ anyhow::bail!(\"unsupported server URL scheme\");\n+ };\n+ (\n+ worker_control_stream_url(&websocket_base, run_id, after),\n+ None,\n+ )\n+ }\n+ ServerTarget::UnixSocket(path) => {\n+ let url = worker_control_stream_url(\"ws://fabro\", run_id, after);\n+ (url, Some(path.as_path().to_path_buf()))\n+ }\n+ };\n+ let mut request = url\n+ .as_str()\n+ .into_client_request()\n+ .context(\"failed to build worker control stream request\")?;\n+ request.headers_mut().insert(\n+ header::AUTHORIZATION,\n+ HeaderValue::from_str(&format!(\"Bearer {worker_token}\"))\n+ .context(\"failed to build worker control stream authorization header\")?,\n+ );\n+ Ok(WorkerControlStreamConnectRequest {\n+ request,\n+ unix_socket_path,\n+ })\n }\n \n-async fn apply_worker_control_line(\n+fn worker_control_stream_url(base: &str, run_id: &RunId, after: Option<&str>) -> String {\n+ let mut url = format!(\"{base}/api/v1/runs/{run_id}/worker/control-stream\");\n+ if let Some(after) = after {\n+ url.push_str(\"?after=\");\n+ url.push_str(after);\n+ }\n+ url\n+}\n+\n+async fn connect_worker_control_stream(\n+ request: WorkerControlStreamConnectRequest,\n+) -> Result {\n+ if let Some(path) = request.unix_socket_path.as_ref() {\n+ #[cfg(unix)]\n+ {\n+ let ws_request = request.request_for_tungstenite();\n+ let stream = UnixStream::connect(path)\n+ .await\n+ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?;\n+ let (socket, _) = tokio_tungstenite::client_async(ws_request, stream)\n+ .await\n+ .map_err(classify_tungstenite_error)?;\n+ Ok(WorkerControlSocket::Unix(Box::new(socket)))\n+ }\n+ #[cfg(not(unix))]\n+ {\n+ let _ = path;\n+ Err(WorkerControlConnectError::Other(anyhow!(\n+ \"Unix-socket worker control stream is not supported on this platform\"\n+ )))\n+ }\n+ } else {\n+ let (socket, _) = connect_async(request.request)\n+ .await\n+ .map_err(classify_tungstenite_error)?;\n+ Ok(WorkerControlSocket::Tcp(Box::new(socket)))\n+ }\n+}\n+\n+fn classify_tungstenite_error(error: tungstenite::Error) -> WorkerControlConnectError {\n+ if let tungstenite::Error::Http(response) = &error {\n+ if response.status().as_u16() == 410 {\n+ return WorkerControlConnectError::InvalidCursor;\n+ }\n+ }\n+ WorkerControlConnectError::Other(anyhow::Error::new(error))\n+}\n+\n+async fn handle_worker_control_socket(\n+ socket: &mut WorkerControlSocket,\n interviewer: &ControlInterviewer,\n cancel_token: &CancellationToken,\n steering_hub: &fabro_workflow::SteeringHub,\n- line: &str,\n-) {\n- if line.trim().is_empty() {\n- return;\n+ run_control: &RunControlState,\n+ applied_ids: &mut AppliedWorkerControlDeliveryIds,\n+ done: &CancellationToken,\n+) -> Result<(), WorkerControlConnectError> {\n+ let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL);\n+ ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);\n+ let mut last_liveness = Instant::now();\n+ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);\n+ tokio::pin!(liveness_timeout);\n+\n+ loop {\n+ liveness_timeout\n+ .as_mut()\n+ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);\n+\n+ tokio::select! {\n+ () = done.cancelled() => return Ok(()),\n+ _ = ping_interval.tick() => {\n+ socket\n+ .send(WebSocketMessage::Ping(Vec::new().into()))\n+ .await\n+ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?;\n+ }\n+ () = &mut liveness_timeout => {\n+ let _ = socket\n+ .send(WebSocketMessage::Close(Some(protocol::CloseFrame {\n+ code: CloseCode::Away,\n+ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(),\n+ })))\n+ .await;\n+ return Err(WorkerControlConnectError::Other(anyhow!(\n+ \"worker control WebSocket liveness timed out\"\n+ )));\n+ }\n+ message = socket.next() => {\n+ let Some(message) = message else {\n+ return Ok(());\n+ };\n+ match message {\n+ Ok(WebSocketMessage::Text(text)) => {\n+ last_liveness = Instant::now();\n+ let frame = serde_json::from_str::(text.as_str())\n+ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?;\n+ apply_worker_control_delivery_frame(\n+ interviewer,\n+ cancel_token,\n+ steering_hub,\n+ run_control,\n+ applied_ids,\n+ frame,\n+ )\n+ .await;\n+ }\n+ Ok(WebSocketMessage::Ping(payload)) => {\n+ last_liveness = Instant::now();\n+ socket\n+ .send(WebSocketMessage::Pong(payload))\n+ .await\n+ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?;\n+ }\n+ Ok(WebSocketMessage::Pong(_) | WebSocketMessage::Binary(_)) => {\n+ last_liveness = Instant::now();\n+ }\n+ Ok(WebSocketMessage::Close(frame)) => {\n+ if frame.as_ref().is_some_and(|frame| {\n+ frame.reason.as_str() == WORKER_CONTROL_INVALID_CURSOR_REASON\n+ }) {\n+ return Err(WorkerControlConnectError::InvalidCursor);\n+ }\n+ return Ok(());\n+ }\n+ Ok(WebSocketMessage::Frame(_)) => {}\n+ Err(err) => {\n+ return Err(WorkerControlConnectError::Other(anyhow::Error::new(err)));\n+ }\n+ }\n+ }\n+ }\n }\n+}\n \n- let Ok(message) = serde_json::from_str::(line) else {\n- return;\n- };\n+async fn apply_worker_control_delivery_frame(\n+ interviewer: &ControlInterviewer,\n+ cancel_token: &CancellationToken,\n+ steering_hub: &fabro_workflow::SteeringHub,\n+ run_control: &RunControlState,\n+ applied_ids: &mut AppliedWorkerControlDeliveryIds,\n+ frame: WorkerControlDeliveryFrame,\n+) -> bool {\n+ // Duplicate ids cannot reach us under normal operation: the server replays\n+ // strictly after the last applied id. Guard against a server-side bug or\n+ // reconnect race by ignoring recently-applied delivery ids.\n+ if applied_ids.contains(&frame.id) {\n+ return false;\n+ }\n+ let frame_id = frame.id;\n+ apply_worker_control_message(\n+ interviewer,\n+ cancel_token,\n+ steering_hub,\n+ run_control,\n+ frame.envelope,\n+ )\n+ .await;\n+ applied_ids.record(frame_id);\n+ true\n+}\n \n+async fn apply_worker_control_message(\n+ interviewer: &ControlInterviewer,\n+ cancel_token: &CancellationToken,\n+ steering_hub: &fabro_workflow::SteeringHub,\n+ run_control: &RunControlState,\n+ message: WorkerControlEnvelope,\n+) {\n match message.message {\n WorkerControlMessage::InterviewAnswer { qid, answer, actor } => {\n let _ = interviewer\n@@ -354,6 +802,12 @@ async fn apply_worker_control_line(\n cancel_token.cancel();\n interviewer.interrupt_all().await;\n }\n+ WorkerControlMessage::RunPause => {\n+ run_control.request_pause();\n+ }\n+ WorkerControlMessage::RunUnpause => {\n+ run_control.request_unpause();\n+ }\n WorkerControlMessage::Steer { text, actor } => {\n steering_hub.deliver_steer(text, Some(actor));\n }\n@@ -485,7 +939,7 @@ impl HttpRunStore {\n Err(err) => last_error = Some(err),\n }\n if let Some(delay) = RUN_STORE_RETRY_DELAYS.get(attempt) {\n- sleep(*delay).await;\n+ time::sleep(*delay).await;\n }\n }\n Err(last_error\n@@ -757,10 +1211,14 @@ fn install_signal_handlers(\n )]\n mod tests {\n use std::sync::Arc;\n+ use std::time::Duration;\n \n use chrono::Utc;\n+ use fabro_client::ServerTarget;\n use fabro_config::Storage;\n- use fabro_interview::{AnswerValue, ControlInterviewer, Interviewer, Question};\n+ use fabro_interview::{\n+ AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlEnvelope,\n+ };\n use fabro_types::run_event::{\n InterviewCompletedProps, InterviewStartedProps, RunCompletedProps, RunControlEffectProps,\n RunFailedProps, RunStatusTransitionProps,\n@@ -771,12 +1229,17 @@ mod tests {\n };\n use fabro_vault::{SecretType, Vault};\n use fabro_workflow::event::RunEventSink;\n+ use fabro_workflow::run_control::RunControlState;\n+ use tokio::time;\n+ use tokio_tungstenite::tungstenite::protocol::{Message as TestWebSocketMessage, Role};\n use tokio_util::sync::CancellationToken;\n \n use super::{\n- WorkerControlStreamEvent, WorkerTitlePhase, apply_worker_control_line,\n- handle_worker_control_stream_events, initial_worker_title_phase, load_worker_vault,\n- read_worker_control_stream_blocking, stamp_system_worker, worker_title,\n+ AppliedWorkerControlDeliveryIds, WorkerControlConnectError, WorkerControlSocket,\n+ WorkerTitlePhase, apply_worker_control_delivery_frame, apply_worker_control_message,\n+ build_worker_control_stream_request, connect_worker_control_stream,\n+ handle_worker_control_socket, initial_worker_title_phase, load_worker_vault,\n+ next_worker_control_reconnect_backoff, stamp_system_worker, worker_title,\n worker_title_phase_for_event,\n };\n use crate::args::RunWorkerMode;\n@@ -1018,20 +1481,28 @@ mod tests {\n }\n \n #[tokio::test]\n- async fn worker_control_line_routes_answer_by_question_id() {\n+ async fn worker_control_routes_answer_by_question_id() {\n let interviewer = Arc::new(ControlInterviewer::new());\n let cancel_token = CancellationToken::new();\n+ let run_control = RunControlState::new();\n let mut question = Question::new(\"Approve?\", QuestionType::YesNo);\n question.id = \"q-1\".to_string();\n let ask_interviewer = Arc::clone(&interviewer);\n let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await });\n \n let hub = test_steering_hub();\n- apply_worker_control_line(\n+ apply_worker_control_message(\n &interviewer,\n &cancel_token,\n &hub,\n- r#\"{\"v\":1,\"type\":\"interview.answer\",\"qid\":\"q-1\",\"answer\":{\"kind\":\"yes\"},\"actor\":{\"kind\":\"system\",\"system_kind\":\"engine\"}}\"#,\n+ &run_control,\n+ WorkerControlEnvelope::interview_answer(\n+ \"q-1\",\n+ fabro_interview::AnswerSubmission::system(\n+ fabro_interview::Answer::yes(),\n+ fabro_types::SystemActorKind::Engine,\n+ ),\n+ ),\n )\n .await;\n \n@@ -1041,9 +1512,10 @@ mod tests {\n }\n \n #[tokio::test]\n- async fn worker_control_line_cancel_sets_cancel_token_and_interrupts_pending_interviews() {\n+ async fn worker_control_cancel_sets_cancel_token_and_interrupts_pending_interviews() {\n let interviewer = Arc::new(ControlInterviewer::new());\n let cancel_token = CancellationToken::new();\n+ let run_control = RunControlState::new();\n let mut question = Question::new(\"Approve?\", QuestionType::YesNo);\n question.id = \"q-1\".to_string();\n let ask_interviewer = Arc::clone(&interviewer);\n@@ -1051,11 +1523,12 @@ mod tests {\n tokio::task::yield_now().await;\n \n let hub = test_steering_hub();\n- apply_worker_control_line(\n+ apply_worker_control_message(\n &interviewer,\n &cancel_token,\n &hub,\n- r#\"{\"v\":1,\"type\":\"run.cancel\"}\"#,\n+ &run_control,\n+ WorkerControlEnvelope::cancel_run(),\n )\n .await;\n \n@@ -1065,57 +1538,206 @@ mod tests {\n }\n \n #[tokio::test]\n- async fn blocking_worker_control_stream_emits_lines_and_eof() {\n- let (event_tx, mut event_rx) = tokio::sync::mpsc::unbounded_channel();\n+ async fn worker_control_pause_and_unpause_route_to_run_control() {\n+ let interviewer = Arc::new(ControlInterviewer::new());\n+ let cancel_token = CancellationToken::new();\n+ let run_control = RunControlState::new();\n+ let hub = test_steering_hub();\n \n- read_worker_control_stream_blocking(\n- std::io::Cursor::new(\n- b\"{\\\"v\\\":1,\\\"type\\\":\\\"run.cancel\\\"}\\n{\\\"v\\\":1,\\\"type\\\":\\\"interview.answer\\\",\\\"qid\\\":\\\"q-1\\\",\\\"answer\\\":{\\\"kind\\\":\\\"yes\\\"},\\\"actor\\\":{\\\"kind\\\":\\\"system\\\",\\\"system_kind\\\":\\\"engine\\\"}}\\n\",\n- ),\n- &event_tx,\n+ apply_worker_control_message(\n+ &interviewer,\n+ &cancel_token,\n+ &hub,\n+ &run_control,\n+ WorkerControlEnvelope::pause_run(),\n+ )\n+ .await;\n+ assert!(run_control.pause_requested());\n+\n+ apply_worker_control_message(\n+ &interviewer,\n+ &cancel_token,\n+ &hub,\n+ &run_control,\n+ WorkerControlEnvelope::unpause_run(),\n+ )\n+ .await;\n+ assert!(!run_control.pause_requested());\n+ }\n+\n+ #[tokio::test]\n+ async fn duplicate_delivery_ids_are_not_applied_twice() {\n+ let interviewer = Arc::new(ControlInterviewer::new());\n+ let cancel_token = CancellationToken::new();\n+ let run_control = RunControlState::new();\n+ let hub = test_steering_hub();\n+ let mut applied_ids = AppliedWorkerControlDeliveryIds::default();\n+ let frame = fabro_interview::WorkerControlDeliveryFrame {\n+ id: \"local:1\".to_string(),\n+ envelope: WorkerControlEnvelope::pause_run(),\n+ };\n+\n+ assert!(\n+ apply_worker_control_delivery_frame(\n+ &interviewer,\n+ &cancel_token,\n+ &hub,\n+ &run_control,\n+ &mut applied_ids,\n+ frame.clone(),\n+ )\n+ .await\n+ );\n+ assert!(\n+ !apply_worker_control_delivery_frame(\n+ &interviewer,\n+ &cancel_token,\n+ &hub,\n+ &run_control,\n+ &mut applied_ids,\n+ frame,\n+ )\n+ .await\n+ );\n+\n+ assert_eq!(applied_ids.last_applied_id(), Some(\"local:1\"));\n+ }\n+\n+ #[test]\n+ fn worker_control_request_construction_for_http_targets() {\n+ let run_id = fixtures::RUN_1;\n+ let request = build_worker_control_stream_request(\n+ &ServerTarget::http_url(\"http://example.com:3000\").unwrap(),\n+ &run_id,\n+ \"worker-token\",\n+ None,\n+ )\n+ .unwrap();\n+ assert_eq!(\n+ request.uri(),\n+ format!(\"ws://example.com:3000/api/v1/runs/{run_id}/worker/control-stream\")\n+ );\n+ assert_eq!(request.authorization(), Some(\"Bearer worker-token\"));\n+\n+ let reconnect = build_worker_control_stream_request(\n+ &ServerTarget::http_url(\"https://example.com\").unwrap(),\n+ &run_id,\n+ \"worker-token\",\n+ Some(\"local:42\"),\n+ )\n+ .unwrap();\n+ assert_eq!(\n+ reconnect.uri(),\n+ format!(\"wss://example.com/api/v1/runs/{run_id}/worker/control-stream?after=local:42\")\n );\n+ }\n+\n+ #[cfg(unix)]\n+ #[test]\n+ fn worker_control_request_construction_for_unix_socket_targets() {\n+ let run_id = fixtures::RUN_1;\n+ let request = build_worker_control_stream_request(\n+ &ServerTarget::unix_socket_path(\"/tmp/fabro.sock\").unwrap(),\n+ &run_id,\n+ \"worker-token\",\n+ Some(\"local:42\"),\n+ )\n+ .unwrap();\n \n assert_eq!(\n- event_rx.try_recv(),\n- Ok(WorkerControlStreamEvent::Line(\n- r#\"{\"v\":1,\"type\":\"run.cancel\"}\"#.to_string()\n- ))\n+ request.uri(),\n+ format!(\"ws://fabro/api/v1/runs/{run_id}/worker/control-stream?after=local:42\")\n );\n assert_eq!(\n- event_rx.try_recv(),\n- Ok(WorkerControlStreamEvent::Line(\n- r#\"{\"v\":1,\"type\":\"interview.answer\",\"qid\":\"q-1\",\"answer\":{\"kind\":\"yes\"},\"actor\":{\"kind\":\"system\",\"system_kind\":\"engine\"}}\"#\n- .to_string()\n- ))\n+ request.unix_socket_path.as_deref(),\n+ Some(std::path::Path::new(\"/tmp/fabro.sock\"))\n );\n- assert_eq!(event_rx.try_recv(), Ok(WorkerControlStreamEvent::Eof));\n }\n \n- #[tokio::test]\n- async fn worker_control_event_loop_eof_interrupts_pending_interviews() {\n+ #[test]\n+ fn worker_control_reconnect_backoff_is_bounded() {\n+ assert_eq!(\n+ next_worker_control_reconnect_backoff(Duration::from_millis(100)),\n+ Duration::from_millis(200)\n+ );\n+ assert_eq!(\n+ next_worker_control_reconnect_backoff(Duration::from_secs(4)),\n+ Duration::from_secs(5)\n+ );\n+ assert_eq!(\n+ next_worker_control_reconnect_backoff(Duration::from_secs(5)),\n+ Duration::from_secs(5)\n+ );\n+ }\n+\n+ #[tokio::test(start_paused = true)]\n+ async fn worker_control_socket_times_out_without_liveness() {\n+ let (worker_io, _server_io) = tokio::io::duplex(1024);\n+ let worker_ws =\n+ tokio_tungstenite::WebSocketStream::from_raw_socket(worker_io, Role::Client, None)\n+ .await;\n+ let mut socket = WorkerControlSocket::Test(Box::new(worker_ws));\n let interviewer = Arc::new(ControlInterviewer::new());\n let cancel_token = CancellationToken::new();\n- let mut question = Question::new(\"Approve?\", QuestionType::YesNo);\n- question.id = \"q-1\".to_string();\n- let ask_interviewer = Arc::clone(&interviewer);\n- let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await });\n- let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel();\n+ let hub = test_steering_hub();\n+ let run_control = RunControlState::new();\n+ let mut applied_ids = AppliedWorkerControlDeliveryIds::default();\n+ let done = CancellationToken::new();\n+\n+ let task = tokio::spawn(async move {\n+ handle_worker_control_socket(\n+ &mut socket,\n+ &interviewer,\n+ &cancel_token,\n+ &hub,\n+ &run_control,\n+ &mut applied_ids,\n+ &done,\n+ )\n+ .await\n+ });\n \n- event_tx.send(WorkerControlStreamEvent::Eof).unwrap();\n- drop(event_tx);\n+ tokio::task::yield_now().await;\n+ time::advance(Duration::from_secs(46)).await;\n+ let result = task.await.unwrap();\n+ assert!(matches!(result, Err(WorkerControlConnectError::Other(_))));\n+ }\n \n- let hub = test_steering_hub();\n- handle_worker_control_stream_events(\n- Arc::clone(&interviewer),\n- cancel_token.clone(),\n- hub,\n- event_rx,\n+ #[cfg(unix)]\n+ #[tokio::test]\n+ async fn worker_control_unix_socket_handshake_completes() {\n+ let temp = tempfile::tempdir().unwrap();\n+ let socket_path = temp.path().join(\"fabro.sock\");\n+ let listener = tokio::net::UnixListener::bind(&socket_path).unwrap();\n+ let server = tokio::spawn(async move {\n+ let (stream, _) = listener.accept().await.unwrap();\n+ let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap();\n+ while let Some(message) = futures::StreamExt::next(&mut socket).await {\n+ match message.unwrap() {\n+ TestWebSocketMessage::Close(_) => break,\n+ TestWebSocketMessage::Ping(payload) => {\n+ futures::SinkExt::send(&mut socket, TestWebSocketMessage::Pong(payload))\n+ .await\n+ .unwrap();\n+ }\n+ _ => {}\n+ }\n+ }\n+ });\n+ let request = build_worker_control_stream_request(\n+ &ServerTarget::unix_socket_path(&socket_path).unwrap(),\n+ &fixtures::RUN_1,\n+ \"worker-token\",\n+ None,\n )\n- .await;\n+ .unwrap();\n \n- let answer = answer_task.await.unwrap().answer;\n- assert_eq!(answer.value, AnswerValue::Interrupted);\n- assert!(!cancel_token.is_cancelled());\n+ let mut socket = connect_worker_control_stream(request).await.unwrap();\n+ socket\n+ .send(TestWebSocketMessage::Close(None))\n+ .await\n+ .unwrap();\n+ server.await.unwrap();\n }\n \n #[tokio::test]\ndiff --git a/lib/crates/fabro-cli/tests/it/cmd/runner.rs b/lib/crates/fabro-cli/tests/it/cmd/runner.rs\nindex d7d87fb93..d5b307d35 100644\n--- a/lib/crates/fabro-cli/tests/it/cmd/runner.rs\n+++ b/lib/crates/fabro-cli/tests/it/cmd/runner.rs\n@@ -794,6 +794,66 @@ fn detached_run_answers_pending_question_without_interview_scratch_files() {\n )));\n }\n \n+#[test]\n+fn detached_run_cancel_reaches_worker_over_control_websocket() {\n+ let context = auth_context();\n+ let run_id = unique_run_id();\n+ let workflow_path = context.temp_dir.join(\"cancel-over-control-websocket.fabro\");\n+ let _gate = write_gated_workflow(\n+ &workflow_path,\n+ \"cancel_over_control_websocket\",\n+ \"Wait for cancellation\",\n+ );\n+\n+ let output = context\n+ .command()\n+ .args([\n+ \"run\",\n+ \"--detach\",\n+ \"--run-id\",\n+ run_id.as_str(),\n+ \"--environment\",\n+ \"local\",\n+ workflow_path.to_str().unwrap(),\n+ ])\n+ .timeout(SHARED_DAEMON_TIMEOUT)\n+ .output()\n+ .expect(\"detached run should execute\");\n+ assert!(\n+ output.status.success(),\n+ \"detached run failed:\\nstdout:\\n{}\\nstderr:\\n{}\",\n+ String::from_utf8_lossy(&output.stdout),\n+ String::from_utf8_lossy(&output.stderr)\n+ );\n+\n+ let run_dir = context.find_run_dir(&run_id);\n+ wait_for_event_names(&run_dir, &[\"run.running\"]);\n+ tokio::runtime::Runtime::new()\n+ .expect(\"test runtime should build\")\n+ .block_on(async {\n+ let (client, base_url) =\n+ server_endpoint(&context.storage_dir).expect(\"server endpoint should exist\");\n+ let response = client\n+ .post(format!(\"{base_url}/api/v1/runs/{run_id}/cancel\"))\n+ .send()\n+ .await\n+ .expect(\"cancel request should succeed\");\n+ assert_reqwest_status(\n+ response,\n+ fabro_http::StatusCode::OK,\n+ format!(\"POST /api/v1/runs/{run_id}/cancel\"),\n+ )\n+ .await;\n+ });\n+\n+ wait_for_status(&run_dir, &[\"failed\"]);\n+ let events = stored_worker_events(&run_dir);\n+ assert!(events.iter().any(|event| matches!(\n+ &event.body,\n+ EventBody::RunFailed(props) if props.failure.reason == FailureReason::Cancelled\n+ )));\n+}\n+\n #[cfg(unix)]\n #[test]\n fn worker_exits_after_sigterm_cancel_even_when_stdin_stays_open() {\ndiff --git a/lib/crates/fabro-interview/src/control_protocol.rs b/lib/crates/fabro-interview/src/control_protocol.rs\nindex cdc820216..f76b61121 100644\n--- a/lib/crates/fabro-interview/src/control_protocol.rs\n+++ b/lib/crates/fabro-interview/src/control_protocol.rs\n@@ -1,3 +1,5 @@\n+use std::time::Duration;\n+\n use fabro_types::{PairId, PairMessageId, PairTarget, Principal, RunId};\n use serde::{Deserialize, Serialize};\n \n@@ -5,6 +7,25 @@ use crate::{Answer, AnswerSubmission, AnswerValue};\n \n pub const WORKER_CONTROL_PROTOCOL_VERSION: u8 = 1;\n \n+/// Interval between worker-control WebSocket ping frames.\n+///\n+/// Server and worker both initiate pings at this cadence; either side that\n+/// fails to observe inbound traffic for [`WORKER_CONTROL_WS_LIVENESS_TIMEOUT`]\n+/// closes the WebSocket.\n+pub const WORKER_CONTROL_WS_PING_INTERVAL: Duration = Duration::from_secs(15);\n+\n+/// Maximum quiet time allowed on a worker-control WebSocket before either side\n+/// declares the connection dead.\n+pub const WORKER_CONTROL_WS_LIVENESS_TIMEOUT: Duration = Duration::from_secs(45);\n+\n+/// WebSocket close-frame reason used when the server can no longer prove\n+/// replay correctness for the requested cursor. Workers must treat this as\n+/// fatal control-channel loss.\n+pub const WORKER_CONTROL_INVALID_CURSOR_REASON: &str = \"invalid_cursor\";\n+\n+/// WebSocket close-frame reason used when the ping/pong watchdog fires.\n+pub const WORKER_CONTROL_PONG_TIMEOUT_REASON: &str = \"pong_timeout\";\n+\n #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n pub struct WorkerControlEnvelope {\n pub v: u8,\n@@ -33,6 +54,22 @@ impl WorkerControlEnvelope {\n }\n }\n \n+ #[must_use]\n+ pub fn pause_run() -> Self {\n+ Self {\n+ v: WORKER_CONTROL_PROTOCOL_VERSION,\n+ message: WorkerControlMessage::RunPause,\n+ }\n+ }\n+\n+ #[must_use]\n+ pub fn unpause_run() -> Self {\n+ Self {\n+ v: WORKER_CONTROL_PROTOCOL_VERSION,\n+ message: WorkerControlMessage::RunUnpause,\n+ }\n+ }\n+\n #[must_use]\n pub fn steer(text: impl Into, actor: Principal) -> Self {\n Self {\n@@ -121,6 +158,10 @@ pub enum WorkerControlMessage {\n },\n #[serde(rename = \"run.cancel\")]\n RunCancel,\n+ #[serde(rename = \"run.pause\")]\n+ RunPause,\n+ #[serde(rename = \"run.unpause\")]\n+ RunUnpause,\n #[serde(rename = \"run.steer\")]\n Steer { text: String, actor: Principal },\n #[serde(rename = \"run.interrupt\")]\n@@ -147,6 +188,12 @@ pub enum WorkerControlMessage {\n PairEnd { pair_id: PairId, actor: Principal },\n }\n \n+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n+pub struct WorkerControlDeliveryFrame {\n+ pub id: String,\n+ pub envelope: WorkerControlEnvelope,\n+}\n+\n #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n #[serde(tag = \"kind\", rename_all = \"snake_case\")]\n pub enum WorkerControlAnswer {\n@@ -229,6 +276,26 @@ mod tests {\n assert_eq!(parsed, envelope);\n }\n \n+ #[test]\n+ fn pause_run_round_trips_through_json() {\n+ let envelope = WorkerControlEnvelope::pause_run();\n+ let json = serde_json::to_string(&envelope).unwrap();\n+ assert_eq!(json, r#\"{\"v\":1,\"type\":\"run.pause\"}\"#);\n+\n+ let parsed: WorkerControlEnvelope = serde_json::from_str(&json).unwrap();\n+ assert_eq!(parsed, envelope);\n+ }\n+\n+ #[test]\n+ fn unpause_run_round_trips_through_json() {\n+ let envelope = WorkerControlEnvelope::unpause_run();\n+ let json = serde_json::to_string(&envelope).unwrap();\n+ assert_eq!(json, r#\"{\"v\":1,\"type\":\"run.unpause\"}\"#);\n+\n+ let parsed: WorkerControlEnvelope = serde_json::from_str(&json).unwrap();\n+ assert_eq!(parsed, envelope);\n+ }\n+\n #[test]\n fn steer_append_round_trips_through_json() {\n let envelope = WorkerControlEnvelope::steer(\"try again\", Principal::System {\n@@ -326,4 +393,20 @@ mod tests {\n serde_json::from_str(&serde_json::to_string(&end).unwrap()).unwrap();\n assert_eq!(parsed, end);\n }\n+\n+ #[test]\n+ fn delivery_frame_round_trips_through_json() {\n+ let frame = WorkerControlDeliveryFrame {\n+ id: \"local:42\".to_string(),\n+ envelope: WorkerControlEnvelope::cancel_run(),\n+ };\n+ let json = serde_json::to_string(&frame).unwrap();\n+ assert_eq!(\n+ json,\n+ r#\"{\"id\":\"local:42\",\"envelope\":{\"v\":1,\"type\":\"run.cancel\"}}\"#\n+ );\n+\n+ let parsed: WorkerControlDeliveryFrame = serde_json::from_str(&json).unwrap();\n+ assert_eq!(parsed, frame);\n+ }\n }\ndiff --git a/lib/crates/fabro-interview/src/lib.rs b/lib/crates/fabro-interview/src/lib.rs\nindex 4c050c4f7..6d8c414a1 100644\n--- a/lib/crates/fabro-interview/src/lib.rs\n+++ b/lib/crates/fabro-interview/src/lib.rs\n@@ -222,8 +222,10 @@ pub use callback::CallbackInterviewer;\n pub use console::ConsoleInterviewer;\n pub use control::{ControlInterviewer, SubmitError};\n pub use control_protocol::{\n- WORKER_CONTROL_PROTOCOL_VERSION, WorkerControlAnswer, WorkerControlEnvelope,\n- WorkerControlMessage,\n+ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON,\n+ WORKER_CONTROL_PROTOCOL_VERSION, WORKER_CONTROL_WS_LIVENESS_TIMEOUT,\n+ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlAnswer, WorkerControlDeliveryFrame,\n+ WorkerControlEnvelope, WorkerControlMessage,\n };\n pub use queue::QueueInterviewer;\n pub use recording::RecordingInterviewer;\ndiff --git a/lib/crates/fabro-server/Cargo.toml b/lib/crates/fabro-server/Cargo.toml\nindex 2b9568ce1..879cea7ce 100644\n--- a/lib/crates/fabro-server/Cargo.toml\n+++ b/lib/crates/fabro-server/Cargo.toml\n@@ -112,6 +112,7 @@ httpmock = \"0.8\"\n serde_yaml = \"0.9\"\n tracing-subscriber.workspace = true\n tokio-util.workspace = true\n+tokio-tungstenite.workspace = true\n fabro-macros = { path = \"../fabro-macros\" }\n fabro-sandbox = { path = \"../fabro-sandbox\", features = [\"test-support\"] }\n fabro-test = { workspace = true }\ndiff --git a/lib/crates/fabro-server/src/lib.rs b/lib/crates/fabro-server/src/lib.rs\nindex 0e591e26d..d2a449732 100644\n--- a/lib/crates/fabro-server/src/lib.rs\n+++ b/lib/crates/fabro-server/src/lib.rs\n@@ -49,6 +49,7 @@ pub mod static_files;\n #[cfg(any(test, feature = \"test-support\"))]\n pub mod test_support;\n pub mod web_auth;\n+mod worker_control;\n mod worker_token;\n \n pub use error::{ApiError, Error, Result};\ndiff --git a/lib/crates/fabro-server/src/principal_middleware.rs b/lib/crates/fabro-server/src/principal_middleware.rs\nindex 9e28be9e4..f9e8c385c 100644\n--- a/lib/crates/fabro-server/src/principal_middleware.rs\n+++ b/lib/crates/fabro-server/src/principal_middleware.rs\n@@ -59,6 +59,7 @@ pub(crate) struct RequiredUser(pub(crate) UserPrincipal);\n pub(crate) struct RequiredRunManagementActor(pub(crate) Principal);\n pub(crate) struct RequiredRunToolActor(pub(crate) Principal);\n pub(crate) struct RequireRunScoped(pub(crate) RunId);\n+pub(crate) struct RequireWorkerRunScoped(pub(crate) RunId);\n pub(crate) struct RequireRunManagementTarget(pub(crate) RunId, pub(crate) Principal);\n pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId);\n pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String);\n@@ -259,6 +260,23 @@ impl FromRequestParts> for RequireRunScoped {\n }\n }\n \n+impl FromRequestParts> for RequireWorkerRunScoped {\n+ type Rejection = Response;\n+\n+ async fn from_request_parts(\n+ parts: &mut Parts,\n+ state: &Arc,\n+ ) -> Result {\n+ let Path(id): Path = Path::from_request_parts(parts, state)\n+ .await\n+ .map_err(IntoResponse::into_response)?;\n+ let run_id = parse_run_id_path(&id)?;\n+ require_worker_for_run(&auth_slot_from_parts(parts), &run_id)\n+ .map_err(IntoResponse::into_response)?;\n+ Ok(Self(run_id))\n+ }\n+}\n+\n impl FromRequestParts> for RequireRunManagementTarget {\n type Rejection = Response;\n \n@@ -432,6 +450,15 @@ fn require_worker_or_user_for_run(\n }\n }\n \n+fn require_worker_for_run(slot: &AuthContextSlot, route_run_id: &RunId) -> Result<(), ApiError> {\n+ let context = slot.0.lock().expect(\"auth context lock poisoned\");\n+ match &context.principal {\n+ Principal::Worker { run_id } if run_id == route_run_id => Ok(()),\n+ Principal::Worker { .. } | Principal::User(_) => Err(ApiError::forbidden()),\n+ _ => Err(auth_rejection(context.auth_status, context.auth_error_code)),\n+ }\n+}\n+\n fn require_run_management_target(\n slot: &AuthContextSlot,\n route_run_id: &RunId,\ndiff --git a/lib/crates/fabro-server/src/serve.rs b/lib/crates/fabro-server/src/serve.rs\nindex b2eb6d91b..422906eeb 100644\n--- a/lib/crates/fabro-server/src/serve.rs\n+++ b/lib/crates/fabro-server/src/serve.rs\n@@ -814,6 +814,8 @@ where\n http_client: None,\n sandbox_provider_registry: None,\n shutdown: shutdown.clone(),\n+ #[cfg(test)]\n+ worker_control_bus: None,\n #[cfg(any(test, feature = \"test-support\"))]\n automation_materializer_override: None,\n })?;\ndiff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs\nindex 4f0ff4dd5..c43ef1138 100644\n--- a/lib/crates/fabro-server/src/server.rs\n+++ b/lib/crates/fabro-server/src/server.rs\n@@ -124,7 +124,7 @@ use sha2::{Digest, Sha256};\n use tempfile::NamedTempFile;\n use tokio::fs;\n use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};\n-use tokio::process::{ChildStderr, ChildStdin, Command};\n+use tokio::process::{ChildStderr, Command};\n use tokio::sync::broadcast::error::RecvError;\n use tokio::sync::{\n Mutex as AsyncMutex, Notify, OwnedMutexGuard, RwLock as AsyncRwLock, Semaphore, broadcast,\n@@ -153,14 +153,15 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware};\n use crate::jwt_auth::{self, AuthMode};\n use crate::principal_middleware::{\n AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunManagementTarget,\n- RequireRunScoped, RequireRunStageScoped, RequireStageArtifact, RequiredUser,\n- principal_middleware,\n+ RequireRunScoped, RequireRunStageScoped, RequireStageArtifact, RequireWorkerRunScoped,\n+ RequiredUser, principal_middleware,\n };\n use crate::request_id::{self, RequestId};\n use crate::run_files::{FilesInFlight, new_files_in_flight};\n use crate::server_secrets::{LlmClientResult, ServerSecrets};\n use crate::spawn_env::{apply_render_graph_env, apply_worker_env};\n use crate::startup::load_startup_vault;\n+use crate::worker_control::{LocalWorkerControlBus, WorkerControlBus, WorkerControlBusError};\n use crate::worker_token::{WorkerScopeSet, WorkerTokenKeys, issue_worker_token_with_scopes};\n use crate::{\n canonical_host, demo, diagnostics, run_manifest, security_headers, static_files, web_auth,\n@@ -272,7 +273,6 @@ enum ExecutionResult {\n \n const WORKER_CANCEL_GRACE: Duration = Duration::from_secs(5);\n const TERMINAL_DELETE_WORKER_GRACE: Duration = Duration::from_millis(50);\n-const WORKER_CONTROL_QUEUE_CAPACITY: usize = 8;\n const WORKER_CONTROL_ENQUEUE_TIMEOUT: Duration = Duration::from_secs(1);\n /// Per-model billing totals.\n #[derive(Default)]\n@@ -294,8 +294,9 @@ pub(crate) type RegistryFactoryOverride =\n \n #[derive(Clone)]\n enum RunAnswerTransport {\n- Subprocess {\n- control_tx: mpsc::Sender,\n+ Worker {\n+ run_id: RunId,\n+ bus: Arc,\n },\n InProcess {\n interviewer: Arc,\n@@ -317,18 +318,46 @@ enum PairTransportError {\n }\n \n impl RunAnswerTransport {\n+ async fn publish_worker_control(\n+ run_id: RunId,\n+ bus: &Arc,\n+ message: WorkerControlEnvelope,\n+ ) -> Result<(), WorkerControlBusError> {\n+ timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, bus.publish(run_id, message))\n+ .await\n+ .map_err(|_| WorkerControlBusError::PublishTimeout)?\n+ .map(|_| ())\n+ }\n+\n+ fn answer_error_from_bus(error: &WorkerControlBusError) -> AnswerTransportError {\n+ match error {\n+ WorkerControlBusError::PublishTimeout => AnswerTransportError::Timeout,\n+ WorkerControlBusError::Closed\n+ | WorkerControlBusError::Unavailable\n+ | WorkerControlBusError::InvalidCursor { .. } => AnswerTransportError::Closed,\n+ }\n+ }\n+\n+ fn pair_error_from_bus(error: &WorkerControlBusError) -> PairTransportError {\n+ match error {\n+ WorkerControlBusError::PublishTimeout => PairTransportError::Timeout,\n+ WorkerControlBusError::Closed\n+ | WorkerControlBusError::Unavailable\n+ | WorkerControlBusError::InvalidCursor { .. } => PairTransportError::Closed,\n+ }\n+ }\n+\n async fn submit(\n &self,\n qid: &str,\n submission: AnswerSubmission,\n ) -> Result<(), AnswerTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::interview_answer(qid.to_string(), submission);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| AnswerTransportError::Timeout)?\n- .map_err(|_| AnswerTransportError::Closed)\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n }\n Self::InProcess { interviewer, .. } => interviewer\n .submit(qid, submission)\n@@ -339,12 +368,11 @@ impl RunAnswerTransport {\n \n async fn cancel_run(&self) -> Result<(), AnswerTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::cancel_run();\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| AnswerTransportError::Timeout)?\n- .map_err(|_| AnswerTransportError::Closed)\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n }\n Self::InProcess { interviewer, .. } => {\n interviewer.cancel_all().await;\n@@ -357,12 +385,11 @@ impl RunAnswerTransport {\n /// in-process steering hub.\n async fn steer(&self, text: String, actor: Principal) -> Result<(), AnswerTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::steer(text, actor);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| AnswerTransportError::Timeout)?\n- .map_err(|_| AnswerTransportError::Closed)\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => {\n steering_hub.deliver_steer(text, Some(actor));\n@@ -373,12 +400,11 @@ impl RunAnswerTransport {\n \n async fn interrupt(&self, actor: Principal) -> Result<(), AnswerTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::interrupt(actor);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| AnswerTransportError::Timeout)?\n- .map_err(|_| AnswerTransportError::Closed)\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => {\n steering_hub.interrupt(Some(&actor));\n@@ -393,12 +419,11 @@ impl RunAnswerTransport {\n actor: Principal,\n ) -> Result<(), AnswerTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::interrupt_then_steer(text, actor);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| AnswerTransportError::Timeout)?\n- .map_err(|_| AnswerTransportError::Closed)\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => {\n steering_hub.interrupt_then_steer(&text, Some(&actor));\n@@ -415,12 +440,14 @@ impl RunAnswerTransport {\n actor: Principal,\n ) -> Result<(), PairTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker {\n+ run_id: worker_run_id,\n+ bus,\n+ } => {\n let message = WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*worker_run_id, bus, message)\n .await\n- .map_err(|_| PairTransportError::Timeout)?\n- .map_err(|_| PairTransportError::Closed)\n+ .map_err(|err| Self::pair_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => steering_hub\n .start_pair(run_id, pair_id, target, Some(actor))\n@@ -438,7 +465,7 @@ impl RunAnswerTransport {\n actor: Principal,\n ) -> Result<(), PairTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::pair_message(\n pair_id,\n message_id,\n@@ -446,10 +473,9 @@ impl RunAnswerTransport {\n client_message_id.clone(),\n actor,\n );\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| PairTransportError::Timeout)?\n- .map_err(|_| PairTransportError::Closed)\n+ .map_err(|err| Self::pair_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => steering_hub\n .send_pair_message(pair_id, message_id, text, client_message_id, Some(actor))\n@@ -460,12 +486,11 @@ impl RunAnswerTransport {\n \n async fn end_pair(&self, pair_id: PairId, actor: Principal) -> Result<(), PairTransportError> {\n match self {\n- Self::Subprocess { control_tx } => {\n+ Self::Worker { run_id, bus } => {\n let message = WorkerControlEnvelope::end_pair(pair_id, actor);\n- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message))\n+ Self::publish_worker_control(*run_id, bus, message)\n .await\n- .map_err(|_| PairTransportError::Timeout)?\n- .map_err(|_| PairTransportError::Closed)\n+ .map_err(|err| Self::pair_error_from_bus(&err))\n }\n Self::InProcess { steering_hub, .. } => steering_hub\n .end_pair(pair_id, Some(actor))\n@@ -473,6 +498,30 @@ impl RunAnswerTransport {\n .map_err(PairTransportError::Control),\n }\n }\n+\n+ async fn pause_run(&self) -> Result<(), AnswerTransportError> {\n+ match self {\n+ Self::Worker { run_id, bus } => {\n+ let message = WorkerControlEnvelope::pause_run();\n+ Self::publish_worker_control(*run_id, bus, message)\n+ .await\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n+ }\n+ Self::InProcess { .. } => Err(AnswerTransportError::Closed),\n+ }\n+ }\n+\n+ async fn unpause_run(&self) -> Result<(), AnswerTransportError> {\n+ match self {\n+ Self::Worker { run_id, bus } => {\n+ let message = WorkerControlEnvelope::unpause_run();\n+ Self::publish_worker_control(*run_id, bus, message)\n+ .await\n+ .map_err(|err| Self::answer_error_from_bus(&err))\n+ }\n+ Self::InProcess { .. } => Err(AnswerTransportError::Closed),\n+ }\n+ }\n }\n \n #[derive(Debug, Clone)]\n@@ -1019,6 +1068,7 @@ pub struct AppState {\n started_at: Instant,\n resource_sampler: resource_sampler::ResourceSampler,\n max_concurrent_runs: usize,\n+ pub(crate) worker_control_bus: Arc,\n scheduler_notify: Notify,\n global_event_tx: broadcast::Sender,\n /// Per-run coalescing registry for `GET /runs/{id}/files`. Concurrent\n@@ -1174,6 +1224,8 @@ pub(crate) struct AppStateConfig {\n pub(crate) http_client: Option,\n pub(crate) sandbox_provider_registry: Option,\n pub(crate) shutdown: CancellationToken,\n+ #[cfg(test)]\n+ pub(crate) worker_control_bus: Option>,\n #[cfg(any(test, feature = \"test-support\"))]\n pub(crate) automation_materializer_override: Option>,\n }\n@@ -2226,6 +2278,8 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result anyhow::Result = {\n+ #[cfg(test)]\n+ {\n+ worker_control_bus.unwrap_or_else(|| Arc::new(LocalWorkerControlBus::new()))\n+ }\n+ #[cfg(not(test))]\n+ {\n+ Arc::new(LocalWorkerControlBus::new())\n+ }\n+ };\n Ok(Arc::new(AppState {\n runs: Mutex::new(HashMap::new()),\n aggregate_billing: Mutex::new(BillingAccumulator::default()),\n@@ -2324,6 +2388,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result {\n@@ -3019,6 +3091,7 @@ async fn finish_cancelled_run_before_execution(state: &Arc, run_id: Ru\n clear_live_run_state(managed_run);\n }\n drop(runs);\n+ cleanup_worker_control_bus_for_run(state.as_ref(), run_id);\n state.scheduler_notify.notify_one();\n }\n \n@@ -3154,6 +3227,7 @@ fn fail_managed_run(state: &Arc, run_id: RunId, reason: FailureReason,\n managed_run.error = Some(message);\n clear_live_run_state(managed_run);\n }\n+ cleanup_worker_control_bus_for_run(state.as_ref(), run_id);\n }\n \n fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) {\n@@ -3229,6 +3303,7 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)\n managed_run.active_api_targets.clear();\n managed_run.active_steerable_stages.clear();\n managed_run.active_non_steerable_stages.clear();\n+ cleanup_worker_control_bus_for_run(state, run_id);\n }\n EventBody::RunFailed(props) => {\n managed_run.status = RunStatus::Failed {\n@@ -3241,6 +3316,7 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)\n managed_run.active_api_targets.clear();\n managed_run.active_steerable_stages.clear();\n managed_run.active_non_steerable_stages.clear();\n+ cleanup_worker_control_bus_for_run(state, run_id);\n }\n // Track active agent sessions by steerability. Activated/deactivated\n // are leased by session id so stale deactivations cannot clear a newer\n@@ -3325,20 +3401,6 @@ async fn drain_worker_stderr(run_id: RunId, stderr: ChildStderr) -> anyhow::Resu\n Ok(())\n }\n \n-async fn pump_worker_control_jsonl(\n- mut stdin: ChildStdin,\n- mut control_rx: mpsc::Receiver,\n-) -> anyhow::Result<()> {\n- while let Some(message) = control_rx.recv().await {\n- let mut line = serde_json::to_vec(&message)?;\n- line.push(b'\\n');\n- stdin.write_all(&line).await?;\n- stdin.flush().await?;\n- }\n-\n- Ok(())\n-}\n-\n async fn append_worker_exit_failure(\n run_store: &fabro_store::RunDatabase,\n run_id: RunId,\n@@ -3422,7 +3484,7 @@ fn worker_command(\n .arg(run_id.to_string())\n .arg(\"--mode\")\n .arg(worker_mode_arg(mode))\n- .stdin(Stdio::piped())\n+ .stdin(Stdio::null())\n .stdout(worker_stdout)\n .stderr(Stdio::piped());\n \n@@ -4101,25 +4163,6 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) {\n }\n }\n \n- let Some(stdin) = child.stdin.take() else {\n- let message = \"Worker stdin pipe was unavailable\".to_string();\n- tracing::error!(run_id = %run_id, \"{message}\");\n- let _ = child.start_kill();\n- let failure_event = workflow_event::Event::workflow_run_failed_from_error(\n- &WorkflowError::engine(message.clone()),\n- fabro_types::RunTiming::default(),\n- FailureReason::LaunchFailed,\n- None,\n- None,\n- None,\n- None,\n- );\n- let _ = workflow_event::append_event(&run_store, &run_id, &failure_event).await;\n- fail_managed_run(&state, run_id, FailureReason::LaunchFailed, message);\n- state.scheduler_notify.notify_one();\n- return;\n- };\n-\n let Some(stderr) = child.stderr.take() else {\n let message = \"Worker stderr pipe was unavailable\".to_string();\n tracing::error!(run_id = %run_id, \"{message}\");\n@@ -4139,15 +4182,16 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) {\n return;\n };\n \n- let (control_tx, control_rx) = mpsc::channel(WORKER_CONTROL_QUEUE_CAPACITY);\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n if let Some(managed_run) = runs.get_mut(&run_id) {\n- managed_run.answer_transport = Some(RunAnswerTransport::Subprocess { control_tx });\n+ managed_run.answer_transport = Some(RunAnswerTransport::Worker {\n+ run_id,\n+ bus: Arc::clone(&state.worker_control_bus),\n+ });\n }\n }\n \n- let control_task = tokio::spawn(pump_worker_control_jsonl(stdin, control_rx));\n let stderr_task = tokio::spawn(drain_worker_stderr(run_id, stderr));\n \n let wait_status = match child.wait().await {\n@@ -4172,9 +4216,6 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) {\n }\n };\n \n- control_task.abort();\n- let _ = control_task.await;\n-\n match stderr_task.await {\n Ok(Ok(())) => {}\n Ok(Err(err)) => {\ndiff --git a/lib/crates/fabro-server/src/server/handler/lifecycle.rs b/lib/crates/fabro-server/src/server/handler/lifecycle.rs\nindex 0c2697133..d8a2589ac 100644\n--- a/lib/crates/fabro-server/src/server/handler/lifecycle.rs\n+++ b/lib/crates/fabro-server/src/server/handler/lifecycle.rs\n@@ -387,10 +387,6 @@ async fn cancel_run(\n | RunStatus::Running\n | RunStatus::Blocked { .. }\n | RunStatus::Paused { .. } => {\n- let use_cancel_signal = !matches!(\n- managed_run.answer_transport,\n- Some(RunAnswerTransport::InProcess { .. })\n- );\n let persist_cancelled_status = matches!(\n managed_run.status,\n RunStatus::Submitted | RunStatus::Pending { .. } | RunStatus::Runnable\n@@ -404,9 +400,7 @@ async fn cancel_run(\n persist_cancelled_status,\n managed_run.answer_transport.clone(),\n managed_run.cancel_token.clone(),\n- use_cancel_signal\n- .then(|| managed_run.cancel_tx.take())\n- .flatten(),\n+ managed_run.cancel_tx.take(),\n managed_run.worker_pid,\n ))\n }\n@@ -436,22 +430,29 @@ async fn cancel_run(\n if let Some(token) = &cancel_token {\n token.cancel();\n }\n- let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx {\n+ let sent_in_process_cancel = if let Some(cancel_tx) = cancel_tx {\n let _ = cancel_tx.send(());\n true\n } else {\n false\n };\n- if let Some(answer_transport) = answer_transport {\n- if !(sent_cancel_signal && matches!(answer_transport, RunAnswerTransport::InProcess { .. }))\n+ let delivered_control = if let Some(answer_transport) = answer_transport {\n+ if sent_in_process_cancel\n+ && matches!(answer_transport, RunAnswerTransport::InProcess { .. })\n {\n- let _ = answer_transport.cancel_run().await;\n+ true\n+ } else {\n+ answer_transport.cancel_run().await.is_ok()\n+ }\n+ } else {\n+ false\n+ };\n+ if !delivered_control {\n+ if let Some(worker_pid) = worker_pid {\n+ #[cfg(unix)]\n+ fabro_proc::sigterm(worker_pid);\n+ schedule_worker_kill(Arc::clone(&state), id, worker_pid);\n }\n- }\n- if let Some(worker_pid) = worker_pid {\n- #[cfg(unix)]\n- fabro_proc::sigterm(worker_pid);\n- schedule_worker_kill(Arc::clone(&state), id, worker_pid);\n }\n \n if persist_cancelled_status {\n@@ -504,9 +505,9 @@ async fn unmanaged_cancel_response(\n /// How `pause_run` should enact the transition, chosen from the current run\n /// status.\n enum PauseMode {\n- /// Worker is running; ask it to pause via SIGUSR1. Status flips to\n- /// `Paused` once the worker acknowledges.\n- Signal { worker_pid: u32 },\n+ /// Worker is running; ask it to pause via the worker control bus. Status\n+ /// flips to `Paused` once the worker acknowledges.\n+ Transport { transport: RunAnswerTransport },\n /// Worker is blocked on a human gate; flip to `Paused` directly by\n /// appending `RunPaused` ourselves.\n AppendEvent,\n@@ -514,8 +515,9 @@ enum PauseMode {\n \n /// How `unpause_run` should enact the transition.\n enum UnpauseMode {\n- /// No outstanding block; ask the worker to resume via SIGUSR2.\n- Signal { worker_pid: u32 },\n+ /// No outstanding block; ask the worker to resume via the worker control\n+ /// bus.\n+ Transport { transport: RunAnswerTransport },\n /// Was paused while blocked; append `RunUnpaused` and let the reducer\n /// restore the underlying blocked state from `Paused { prior_block }`.\n AppendEvent,\n@@ -544,11 +546,11 @@ async fn pause_run(\n let runs = state.runs.lock().expect(\"runs lock poisoned\");\n match runs.get(&id) {\n Some(managed_run) if managed_run.status == RunStatus::Running => {\n- let Some(worker_pid) = managed_run.worker_pid else {\n+ let Some(transport) = managed_run.answer_transport.clone() else {\n return ApiError::new(StatusCode::CONFLICT, \"Run worker is not available.\")\n .into_response();\n };\n- PauseMode::Signal { worker_pid }\n+ PauseMode::Transport { transport }\n }\n Some(managed_run) if matches!(managed_run.status, RunStatus::Blocked { .. }) => {\n PauseMode::AppendEvent\n@@ -578,11 +580,14 @@ async fn pause_run(\n return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response();\n }\n match mode {\n- PauseMode::Signal { worker_pid } => {\n- #[cfg(unix)]\n- fabro_proc::sigusr1(worker_pid);\n- #[cfg(not(unix))]\n- let _ = worker_pid;\n+ PauseMode::Transport { transport } => {\n+ if transport.pause_run().await.is_err() {\n+ return ApiError::new(\n+ StatusCode::SERVICE_UNAVAILABLE,\n+ \"Failed to deliver pause request to the active run.\",\n+ )\n+ .into_response();\n+ }\n }\n PauseMode::AppendEvent => {\n if let Some(response) = synchronous_transition(state.as_ref(), id, |events| {\n@@ -625,11 +630,11 @@ async fn unpause_run(\n prior_block: Some(_),\n } => UnpauseMode::AppendEvent,\n RunStatus::Paused { prior_block: None } => {\n- let Some(worker_pid) = managed_run.worker_pid else {\n+ let Some(transport) = managed_run.answer_transport.clone() else {\n return ApiError::new(StatusCode::CONFLICT, \"Run worker is not available.\")\n .into_response();\n };\n- UnpauseMode::Signal { worker_pid }\n+ UnpauseMode::Transport { transport }\n }\n _ => {\n return ApiError::new(StatusCode::CONFLICT, \"Run is not paused.\")\n@@ -658,11 +663,14 @@ async fn unpause_run(\n return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response();\n }\n match mode {\n- UnpauseMode::Signal { worker_pid } => {\n- #[cfg(unix)]\n- fabro_proc::sigusr2(worker_pid);\n- #[cfg(not(unix))]\n- let _ = worker_pid;\n+ UnpauseMode::Transport { transport } => {\n+ if transport.unpause_run().await.is_err() {\n+ return ApiError::new(\n+ StatusCode::SERVICE_UNAVAILABLE,\n+ \"Failed to deliver unpause request to the active run.\",\n+ )\n+ .into_response();\n+ }\n }\n UnpauseMode::AppendEvent => {\n if let Some(response) = synchronous_transition(state.as_ref(), id, |events| {\ndiff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs\nindex 3b7cc1ec1..33c6aa8eb 100644\n--- a/lib/crates/fabro-server/src/server/handler/mod.rs\n+++ b/lib/crates/fabro-server/src/server/handler/mod.rs\n@@ -23,6 +23,7 @@ mod sessions;\n mod steer;\n pub(in crate::server) mod system;\n mod variables;\n+mod worker_control;\n \n pub(super) use system::{health, openapi_spec};\n \n@@ -167,6 +168,7 @@ pub(super) fn real_routes() -> Router> {\n .merge(models::routes())\n .merge(secrets::routes())\n .merge(variables::routes())\n+ .merge(worker_control::routes())\n .merge(sessions::routes())\n .merge(system::routes())\n .merge(completions::routes())\ndiff --git a/lib/crates/fabro-server/src/server/handler/worker_control.rs b/lib/crates/fabro-server/src/server/handler/worker_control.rs\nnew file mode 100644\nindex 000000000..d643a904e\n--- /dev/null\n+++ b/lib/crates/fabro-server/src/server/handler/worker_control.rs\n@@ -0,0 +1,166 @@\n+use std::sync::Arc;\n+\n+use axum::extract::ws::{\n+ CloseFrame, Message as WsMessage, WebSocket, WebSocketUpgrade, close_code,\n+};\n+use fabro_interview::{\n+ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON,\n+ WORKER_CONTROL_WS_LIVENESS_TIMEOUT, WORKER_CONTROL_WS_PING_INTERVAL,\n+ WorkerControlDeliveryFrame,\n+};\n+use futures_util::{SinkExt, StreamExt};\n+use tokio::time::{self, Instant, MissedTickBehavior};\n+\n+use super::super::{\n+ ApiError, AppState, IntoResponse, Query, RequireWorkerRunScoped, Response, Router, State,\n+ StatusCode, get,\n+};\n+use crate::worker_control::{WorkerControlBusError, WorkerControlCursor, WorkerControlReceiver};\n+\n+#[derive(Debug, serde::Deserialize)]\n+struct WorkerControlStreamQuery {\n+ after: Option,\n+}\n+\n+pub(super) fn routes() -> Router> {\n+ Router::new().route(\n+ \"/runs/{id}/worker/control-stream\",\n+ get(worker_control_stream),\n+ )\n+}\n+\n+async fn worker_control_stream(\n+ RequireWorkerRunScoped(id): RequireWorkerRunScoped,\n+ State(state): State>,\n+ Query(query): Query,\n+ ws: WebSocketUpgrade,\n+) -> Response {\n+ let cached = match state.store.get_cached_run(&id).await {\n+ Ok(Some(cached)) => cached,\n+ Ok(None) => return ApiError::not_found(\"Run not found.\").into_response(),\n+ Err(err) => {\n+ return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())\n+ .into_response();\n+ }\n+ };\n+ if cached.projection.archived_at.is_some() {\n+ return ApiError::new(StatusCode::CONFLICT, \"Run is archived.\").into_response();\n+ }\n+ if cached.projection.status.is_terminal() {\n+ return ApiError::new(\n+ StatusCode::CONFLICT,\n+ \"Run is terminal and cannot accept worker control streams.\",\n+ )\n+ .into_response();\n+ }\n+\n+ let cursor = match WorkerControlCursor::from_after_query(query.after.as_deref()) {\n+ Ok(cursor) => cursor,\n+ Err(err @ WorkerControlBusError::InvalidCursor { .. }) => {\n+ return ApiError::new(StatusCode::GONE, err.to_string()).into_response();\n+ }\n+ Err(err) => return worker_control_bus_error_response(&err),\n+ };\n+ let receiver = match state.worker_control_bus.subscribe(id, cursor).await {\n+ Ok(receiver) => receiver,\n+ Err(err @ WorkerControlBusError::InvalidCursor { .. }) => {\n+ return ApiError::new(StatusCode::GONE, err.to_string()).into_response();\n+ }\n+ Err(err) => return worker_control_bus_error_response(&err),\n+ };\n+\n+ ws.on_upgrade(move |socket| worker_control_websocket(socket, receiver))\n+}\n+\n+fn worker_control_bus_error_response(err: &WorkerControlBusError) -> Response {\n+ let status = match err {\n+ WorkerControlBusError::Closed | WorkerControlBusError::Unavailable => {\n+ StatusCode::SERVICE_UNAVAILABLE\n+ }\n+ WorkerControlBusError::PublishTimeout => StatusCode::SERVICE_UNAVAILABLE,\n+ WorkerControlBusError::InvalidCursor { .. } => StatusCode::GONE,\n+ };\n+ ApiError::new(status, err.to_string()).into_response()\n+}\n+\n+async fn worker_control_websocket(socket: WebSocket, mut receiver: WorkerControlReceiver) {\n+ let (mut sender, mut receiver_ws) = socket.split();\n+ let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL);\n+ ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);\n+ let mut last_liveness = Instant::now();\n+ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);\n+ tokio::pin!(liveness_timeout);\n+\n+ loop {\n+ liveness_timeout\n+ .as_mut()\n+ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);\n+\n+ tokio::select! {\n+ delivery = receiver.recv() => {\n+ let Some(delivery) = delivery else {\n+ let _ = sender.send(WsMessage::Close(None)).await;\n+ return;\n+ };\n+ let delivery = match delivery {\n+ Ok(delivery) => delivery,\n+ Err(WorkerControlBusError::InvalidCursor { .. }) => {\n+ let _ = sender.send(invalid_cursor_close_message()).await;\n+ return;\n+ }\n+ Err(_) => {\n+ let _ = sender.send(WsMessage::Close(None)).await;\n+ return;\n+ }\n+ };\n+ let frame = WorkerControlDeliveryFrame {\n+ id: delivery.id.to_string(),\n+ envelope: delivery.envelope,\n+ };\n+ let Ok(text) = serde_json::to_string(&frame) else {\n+ let _ = sender.send(WsMessage::Close(None)).await;\n+ return;\n+ };\n+ if sender.send(WsMessage::Text(text.into())).await.is_err() {\n+ return;\n+ }\n+ }\n+ message = receiver_ws.next() => {\n+ let Some(message) = message else {\n+ return;\n+ };\n+ match message {\n+ Ok(WsMessage::Ping(payload)) => {\n+ last_liveness = Instant::now();\n+ if sender.send(WsMessage::Pong(payload)).await.is_err() {\n+ return;\n+ }\n+ }\n+ Ok(WsMessage::Pong(_) | WsMessage::Text(_) | WsMessage::Binary(_)) => {\n+ last_liveness = Instant::now();\n+ }\n+ Ok(WsMessage::Close(_)) | Err(_) => return,\n+ }\n+ }\n+ _ = ping_interval.tick() => {\n+ if sender.send(WsMessage::Ping(Vec::new().into())).await.is_err() {\n+ return;\n+ }\n+ }\n+ () = &mut liveness_timeout => {\n+ let _ = sender.send(WsMessage::Close(Some(CloseFrame {\n+ code: close_code::AWAY,\n+ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(),\n+ }))).await;\n+ return;\n+ }\n+ }\n+ }\n+}\n+\n+fn invalid_cursor_close_message() -> WsMessage {\n+ WsMessage::Close(Some(CloseFrame {\n+ code: close_code::POLICY,\n+ reason: WORKER_CONTROL_INVALID_CURSOR_REASON.into(),\n+ }))\n+}\ndiff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs\nindex a422fa05b..fbb10bbe2 100644\n--- a/lib/crates/fabro-server/src/server/tests.rs\n+++ b/lib/crates/fabro-server/src/server/tests.rs\n@@ -13,7 +13,8 @@ use fabro_automation::{AutomationId, AutomationTarget};\n use fabro_config::ServerSettingsBuilder;\n use fabro_config::bind::Bind;\n use fabro_interview::{\n- AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlMessage,\n+ AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlDeliveryFrame,\n+ WorkerControlEnvelope, WorkerControlMessage,\n };\n use fabro_llm::types::{Message as LlmMessage, Request as LlmRequest, TokenCounts};\n use fabro_model::catalog::LlmCatalogSettings;\n@@ -33,6 +34,8 @@ use httpmock::Method::{GET, POST};\n use httpmock::MockServer;\n use serde_json::json;\n use tokio_stream::StreamExt as _;\n+use tokio_tungstenite::tungstenite::client::IntoClientRequest;\n+use tokio_tungstenite::tungstenite::protocol::Message as WebSocketMessage;\n use tower::ServiceExt;\n use tracing::field::{Field, Visit};\n use tracing::{Event as TracingEvent, Subscriber, subscriber};\n@@ -45,6 +48,9 @@ use crate::automation_materializer::AutomationRunMaterializeInput;\n use crate::github_webhooks::compute_signature;\n use crate::jwt_auth::{AuthMode, ConfiguredAuth};\n use crate::test_support::*;\n+use crate::worker_control::{\n+ LocalWorkerControlBus, WorkerControlBus, WorkerControlCursor, WorkerControlReceiver,\n+};\n \n const MINIMAL_DOT: &str = r#\"digraph Test {\n graph [goal=\"Test\"]\n@@ -669,6 +675,292 @@ fn bearer_request(method: Method, path: &str, bearer: &str, body: Body) -> Reque\n .unwrap()\n }\n \n+struct WorkerControlWsTestServer {\n+ base_url: String,\n+ task: tokio::task::JoinHandle<()>,\n+}\n+\n+impl WorkerControlWsTestServer {\n+ async fn spawn(app: Router) -> Self {\n+ let listener = tokio::net::TcpListener::bind(\"127.0.0.1:0\")\n+ .await\n+ .expect(\"test WebSocket listener should bind\");\n+ let addr = listener\n+ .local_addr()\n+ .expect(\"test WebSocket listener should have a local address\");\n+ let task = tokio::spawn(async move {\n+ let result = axum::serve(\n+ listener,\n+ app.into_make_service_with_connect_info::(),\n+ )\n+ .await;\n+ if let Err(err) = result {\n+ tracing::debug!(error = %err, \"test WebSocket server stopped\");\n+ }\n+ });\n+ Self {\n+ base_url: format!(\"ws://{addr}\"),\n+ task,\n+ }\n+ }\n+\n+ fn worker_control_url(&self, run_id: RunId, after: Option<&str>) -> String {\n+ let mut url = format!(\n+ \"{}/api/v1/runs/{run_id}/worker/control-stream\",\n+ self.base_url\n+ );\n+ if let Some(after) = after {\n+ url.push_str(\"?after=\");\n+ url.push_str(after);\n+ }\n+ url\n+ }\n+}\n+\n+impl Drop for WorkerControlWsTestServer {\n+ fn drop(&mut self) {\n+ self.task.abort();\n+ }\n+}\n+\n+fn worker_control_ws_request(\n+ server: &WorkerControlWsTestServer,\n+ run_id: RunId,\n+ bearer: Option<&str>,\n+ after: Option<&str>,\n+) -> Request<()> {\n+ let mut request = server\n+ .worker_control_url(run_id, after)\n+ .into_client_request()\n+ .expect(\"test worker-control WebSocket request should build\");\n+ if let Some(bearer) = bearer {\n+ request.headers_mut().insert(\n+ header::AUTHORIZATION,\n+ format!(\"Bearer {bearer}\")\n+ .parse()\n+ .expect(\"test bearer header should parse\"),\n+ );\n+ }\n+ request\n+}\n+\n+async fn connect_worker_control_ws(\n+ server: &WorkerControlWsTestServer,\n+ run_id: RunId,\n+ bearer: &str,\n+ after: Option<&str>,\n+) -> tokio_tungstenite::WebSocketStream> {\n+ let request = worker_control_ws_request(server, run_id, Some(bearer), after);\n+ let (socket, _) = tokio_tungstenite::connect_async(request)\n+ .await\n+ .expect(\"worker-control WebSocket should connect\");\n+ socket\n+}\n+\n+async fn assert_worker_control_ws_rejected(\n+ server: &WorkerControlWsTestServer,\n+ run_id: RunId,\n+ bearer: Option<&str>,\n+ after: Option<&str>,\n+ expected: StatusCode,\n+) {\n+ let request = worker_control_ws_request(server, run_id, bearer, after);\n+ let error = tokio_tungstenite::connect_async(request)\n+ .await\n+ .expect_err(\"worker-control WebSocket should be rejected\");\n+ match error {\n+ tokio_tungstenite::tungstenite::Error::Http(response) => {\n+ assert_eq!(response.status(), expected);\n+ }\n+ other => panic!(\"expected HTTP rejection {expected}, got {other:#}\"),\n+ }\n+}\n+\n+async fn next_worker_control_frame(\n+ socket: &mut tokio_tungstenite::WebSocketStream<\n+ tokio_tungstenite::MaybeTlsStream,\n+ >,\n+) -> WorkerControlDeliveryFrame {\n+ let message = tokio::time::timeout(std::time::Duration::from_secs(2), async {\n+ loop {\n+ let message = futures_util::StreamExt::next(socket)\n+ .await\n+ .expect(\"worker-control WebSocket should remain open\")\n+ .expect(\"worker-control WebSocket frame should be ok\");\n+ match message {\n+ WebSocketMessage::Text(text) => return text,\n+ WebSocketMessage::Ping(payload) => {\n+ futures_util::SinkExt::send(socket, WebSocketMessage::Pong(payload))\n+ .await\n+ .expect(\"test worker-control pong should send\");\n+ }\n+ WebSocketMessage::Pong(_)\n+ | WebSocketMessage::Binary(_)\n+ | WebSocketMessage::Frame(_) => {}\n+ WebSocketMessage::Close(frame) => {\n+ panic!(\"worker-control WebSocket closed before text frame: {frame:?}\");\n+ }\n+ }\n+ }\n+ })\n+ .await\n+ .expect(\"worker-control frame should arrive\");\n+ serde_json::from_str(message.as_str()).expect(\"worker-control delivery frame should parse\")\n+}\n+\n+#[tokio::test(flavor = \"current_thread\")]\n+async fn worker_control_stream_rejects_missing_user_and_cross_run_auth() {\n+ let (_state, app) = jwt_auth_app();\n+ let user_bearer = issue_test_user_jwt();\n+ let run_id = create_run_with_bearer(&app, &user_bearer).await;\n+ let worker_bearer = issue_test_worker_token(&run_id);\n+ let other_run_id = create_run_with_bearer(&app, &user_bearer).await;\n+ let other_worker_bearer = issue_test_worker_token(&other_run_id);\n+ let server = WorkerControlWsTestServer::spawn(app).await;\n+\n+ assert_worker_control_ws_rejected(&server, run_id, None, None, StatusCode::UNAUTHORIZED).await;\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ run_id,\n+ Some(&user_bearer),\n+ None,\n+ StatusCode::FORBIDDEN,\n+ )\n+ .await;\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ run_id,\n+ Some(&other_worker_bearer),\n+ None,\n+ StatusCode::FORBIDDEN,\n+ )\n+ .await;\n+\n+ let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await;\n+ futures_util::SinkExt::send(&mut socket, WebSocketMessage::Close(None))\n+ .await\n+ .unwrap();\n+}\n+\n+#[tokio::test(flavor = \"current_thread\")]\n+async fn worker_control_stream_start_subscription_delivers_frames() {\n+ let (state, app) = jwt_auth_app();\n+ let user_bearer = issue_test_user_jwt();\n+ let run_id = create_run_with_bearer(&app, &user_bearer).await;\n+ let worker_bearer = issue_test_worker_token(&run_id);\n+ let server = WorkerControlWsTestServer::spawn(app).await;\n+ let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await;\n+\n+ let expected = WorkerControlEnvelope::cancel_run();\n+ let id = state\n+ .worker_control_bus\n+ .publish(run_id, expected.clone())\n+ .await\n+ .unwrap();\n+ let frame = next_worker_control_frame(&mut socket).await;\n+\n+ assert_eq!(frame.id, id.to_string());\n+ assert_eq!(frame.envelope, expected);\n+}\n+\n+#[tokio::test(flavor = \"current_thread\")]\n+async fn worker_control_stream_after_subscription_delivers_only_later_frames() {\n+ let (state, app) = jwt_auth_app();\n+ let user_bearer = issue_test_user_jwt();\n+ let run_id = create_run_with_bearer(&app, &user_bearer).await;\n+ let worker_bearer = issue_test_worker_token(&run_id);\n+ let first = state\n+ .worker_control_bus\n+ .publish(run_id, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+ let expected = WorkerControlEnvelope::pause_run();\n+ let second = state\n+ .worker_control_bus\n+ .publish(run_id, expected.clone())\n+ .await\n+ .unwrap();\n+ let server = WorkerControlWsTestServer::spawn(app).await;\n+\n+ let mut socket =\n+ connect_worker_control_ws(&server, run_id, &worker_bearer, Some(first.as_str())).await;\n+ let frame = next_worker_control_frame(&mut socket).await;\n+\n+ assert_eq!(frame.id, second.to_string());\n+ assert_eq!(frame.envelope, expected);\n+}\n+\n+#[tokio::test(flavor = \"current_thread\")]\n+async fn worker_control_stream_invalid_cursor_is_http_gone_before_upgrade() {\n+ let (_state, app) = jwt_auth_app();\n+ let user_bearer = issue_test_user_jwt();\n+ let run_id = create_run_with_bearer(&app, &user_bearer).await;\n+ let worker_bearer = issue_test_worker_token(&run_id);\n+ let server = WorkerControlWsTestServer::spawn(app).await;\n+\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ run_id,\n+ Some(&worker_bearer),\n+ Some(\"local:999\"),\n+ StatusCode::GONE,\n+ )\n+ .await;\n+}\n+\n+#[tokio::test(flavor = \"current_thread\")]\n+async fn worker_control_stream_rejects_missing_terminal_and_archived_runs() {\n+ let (state, app) = jwt_auth_app();\n+ let user_bearer = issue_test_user_jwt();\n+ let missing_run_id = RunId::new();\n+ let missing_worker_bearer = issue_test_worker_token(&missing_run_id);\n+ let terminal_run_id = RunId::new();\n+ create_succeeded_run(&state, terminal_run_id).await;\n+ let terminal_worker_bearer = issue_test_worker_token(&terminal_run_id);\n+ let archived_run_id = RunId::new();\n+ create_succeeded_run(&state, archived_run_id).await;\n+ let archived_worker_bearer = issue_test_worker_token(&archived_run_id);\n+ let archive_response = app\n+ .clone()\n+ .oneshot(\n+ Request::builder()\n+ .method(\"POST\")\n+ .uri(api(&format!(\"/runs/{archived_run_id}/archive\")))\n+ .header(header::AUTHORIZATION, format!(\"Bearer {user_bearer}\"))\n+ .body(Body::empty())\n+ .unwrap(),\n+ )\n+ .await\n+ .unwrap();\n+ response_json!(archive_response, StatusCode::OK).await;\n+ let server = WorkerControlWsTestServer::spawn(app).await;\n+\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ missing_run_id,\n+ Some(&missing_worker_bearer),\n+ None,\n+ StatusCode::NOT_FOUND,\n+ )\n+ .await;\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ terminal_run_id,\n+ Some(&terminal_worker_bearer),\n+ None,\n+ StatusCode::CONFLICT,\n+ )\n+ .await;\n+ assert_worker_control_ws_rejected(\n+ &server,\n+ archived_run_id,\n+ Some(&archived_worker_bearer),\n+ None,\n+ StatusCode::CONFLICT,\n+ )\n+ .await;\n+}\n+\n fn json_bearer_request(\n method: Method,\n path: &str,\n@@ -1664,6 +1956,7 @@ fn slack_app_state_with_secret_sources(\n http_client: Some(fabro_http::test_http_client().expect(\"test HTTP client should build\")),\n sandbox_provider_registry: None,\n shutdown: tokio_util::sync::CancellationToken::new(),\n+ worker_control_bus: None,\n automation_materializer_override: None,\n })\n .expect(\"slack test app state should build\")\n@@ -1764,6 +2057,7 @@ fn slack_service_respects_disabled_server_config_even_with_vault_tokens() {\n http_client: Some(fabro_http::test_http_client().expect(\"test HTTP client should build\")),\n sandbox_provider_registry: None,\n shutdown: tokio_util::sync::CancellationToken::new(),\n+ worker_control_bus: None,\n automation_materializer_override: None,\n })\n .expect(\"slack disabled test app state should build\");\n@@ -1771,6 +2065,23 @@ fn slack_service_respects_disabled_server_config_even_with_vault_tokens() {\n assert!(state.slack_service.is_none());\n }\n \n+#[cfg(unix)]\n+#[test]\n+fn worker_command_uses_null_stdin_and_token_env() {\n+ let storage_dir = tempfile::tempdir().unwrap();\n+ let state = worker_command_test_state(storage_dir.path(), &[\"dev-token\"], Some(TEST_DEV_TOKEN));\n+ let cmd = worker_command(\n+ state.as_ref(),\n+ RunId::new(),\n+ RunExecutionMode::Start,\n+ storage_dir.path(),\n+ false,\n+ )\n+ .unwrap();\n+\n+ assert_worker_command_passes_token_only_by_env(&cmd);\n+}\n+\n #[cfg(unix)]\n #[test]\n fn worker_command_default_token_omits_agent_run_tools_scope() {\n@@ -2069,6 +2380,7 @@ methods = [\"dev-token\"]\n http_client: Some(fabro_http::test_http_client().expect(\"test HTTP client should build\")),\n sandbox_provider_registry: None,\n shutdown: tokio_util::sync::CancellationToken::new(),\n+ worker_control_bus: None,\n automation_materializer_override: None,\n }) else {\n panic!(\"build_app_state should require SESSION_SECRET\")\n@@ -2197,6 +2509,7 @@ fn build_test_app_state_with_vault_path(vault_path: &Path) -> anyhow::Result crate::worker_token::\n .claims\n }\n \n+async fn worker_transport_with_receiver(\n+ run_id: RunId,\n+) -> (RunAnswerTransport, WorkerControlReceiver) {\n+ let bus = StdArc::new(LocalWorkerControlBus::new());\n+ let receiver = bus\n+ .subscribe(run_id, WorkerControlCursor::Start)\n+ .await\n+ .expect(\"test worker bus should subscribe\");\n+ // Ensure the subscription task is waiting before the test publishes.\n+ tokio::task::yield_now().await;\n+ let bus: StdArc = bus;\n+ let transport = RunAnswerTransport::Worker { run_id, bus };\n+ (transport, receiver)\n+}\n+\n+async fn recv_worker_control_envelope(\n+ receiver: &mut WorkerControlReceiver,\n+) -> WorkerControlEnvelope {\n+ receiver\n+ .recv()\n+ .await\n+ .expect(\"test worker control receiver should stay open\")\n+ .expect(\"test worker control delivery should succeed\")\n+ .envelope\n+}\n+\n #[tokio::test]\n-async fn subprocess_answer_transport_cancel_run_enqueues_cancel_message() {\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let transport = RunAnswerTransport::Subprocess { control_tx };\n+async fn worker_answer_transport_cancel_run_publishes_cancel_message() {\n+ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;\n \n transport.cancel_run().await.unwrap();\n \n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::cancel_run())\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::cancel_run()\n );\n }\n \n #[tokio::test]\n-async fn subprocess_answer_transport_steer_enqueues_plain_steer_message() {\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let transport = RunAnswerTransport::Subprocess { control_tx };\n+async fn worker_answer_transport_steer_publishes_plain_steer_message() {\n+ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;\n let actor = Principal::System {\n system_kind: SystemActorKind::Engine,\n };\n@@ -2404,15 +2741,14 @@ async fn subprocess_answer_transport_steer_enqueues_plain_steer_message() {\n .unwrap();\n \n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::steer(\"try again\", actor))\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::steer(\"try again\", actor)\n );\n }\n \n #[tokio::test]\n-async fn subprocess_answer_transport_interrupt_enqueues_interrupt_message() {\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let transport = RunAnswerTransport::Subprocess { control_tx };\n+async fn worker_answer_transport_interrupt_publishes_interrupt_message() {\n+ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;\n let actor = Principal::System {\n system_kind: SystemActorKind::Engine,\n };\n@@ -2420,15 +2756,14 @@ async fn subprocess_answer_transport_interrupt_enqueues_interrupt_message() {\n transport.interrupt(actor.clone()).await.unwrap();\n \n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::interrupt(actor))\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::interrupt(actor)\n );\n }\n \n #[tokio::test]\n-async fn subprocess_answer_transport_interrupt_then_steer_enqueues_single_combined_message() {\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let transport = RunAnswerTransport::Subprocess { control_tx };\n+async fn worker_answer_transport_interrupt_then_steer_publishes_single_combined_message() {\n+ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;\n let actor = Principal::System {\n system_kind: SystemActorKind::Engine,\n };\n@@ -2439,19 +2774,15 @@ async fn subprocess_answer_transport_interrupt_then_steer_enqueues_single_combin\n .unwrap();\n \n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::interrupt_then_steer(\n- \"try again\",\n- actor\n- ))\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::interrupt_then_steer(\"try again\", actor)\n );\n }\n \n #[tokio::test]\n-async fn subprocess_answer_transport_pair_commands_enqueue_control_messages() {\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(3);\n- let transport = RunAnswerTransport::Subprocess { control_tx };\n+async fn worker_answer_transport_pair_commands_publish_control_messages() {\n let run_id = fixtures::RUN_1;\n+ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;\n let pair_id = \"01HZX6M29F1CD5YYMHT1F5D7WQ\".parse().unwrap();\n let message_id = \"01HZX6M4D7Y1QW0Q0P6V8Z4DR5\".parse().unwrap();\n let actor = Principal::System {\n@@ -2476,27 +2807,39 @@ async fn subprocess_answer_transport_pair_commands_enqueue_control_messages() {\n transport.end_pair(pair_id, actor.clone()).await.unwrap();\n \n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::start_pair(\n- run_id,\n- pair_id,\n- target,\n- actor.clone()\n- ))\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor.clone())\n );\n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::pair_message(\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::pair_message(\n pair_id,\n message_id,\n \"inspect this\",\n Some(\"client-1\".to_string()),\n actor.clone()\n- ))\n+ )\n );\n assert_eq!(\n- control_rx.recv().await,\n- Some(WorkerControlEnvelope::end_pair(pair_id, actor))\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::end_pair(pair_id, actor)\n+ );\n+}\n+\n+#[tokio::test]\n+async fn worker_answer_transport_pause_and_unpause_publish_control_messages() {\n+ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;\n+\n+ transport.pause_run().await.unwrap();\n+ transport.unpause_run().await.unwrap();\n+\n+ assert_eq!(\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::pause_run()\n+ );\n+ assert_eq!(\n+ recv_worker_control_envelope(&mut control_rx).await,\n+ WorkerControlEnvelope::unpause_run()\n );\n }\n \n@@ -5444,6 +5787,7 @@ fn create_github_token_app_state_with_env_lookup_and_llm_catalog_settings(\n http_client: Some(fabro_http::test_http_client().expect(\"test HTTP client should build\")),\n sandbox_provider_registry: None,\n shutdown: tokio_util::sync::CancellationToken::new(),\n+ worker_control_bus: None,\n automation_materializer_override: None,\n };\n let state = build_app_state(config).expect(\"test app state should build\");\n@@ -10847,12 +11191,8 @@ async fn steer_without_active_steerable_session_forwards_plain_steer_for_bufferi\n let state = test_app_state();\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n \n let req = Request::builder()\n .method(\"POST\")\n@@ -10863,7 +11203,7 @@ async fn steer_without_active_steerable_session_forwards_plain_steer_for_bufferi\n \n let response = app.oneshot(req).await.unwrap();\n assert_status!(response, StatusCode::ACCEPTED).await;\n- let envelope = control_rx.recv().await.unwrap();\n+ let envelope = recv_worker_control_envelope(&mut control_rx).await;\n assert!(matches!(\n envelope.message,\n WorkerControlMessage::Steer { ref text, .. } if text == \"try again\"\n@@ -10876,12 +11216,8 @@ async fn steer_with_active_non_steerable_session_returns_conflict() {\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n let stage_id = StageId::new(\"agent\", 1);\n- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n runs.get_mut(&run_id)\n@@ -10908,12 +11244,8 @@ async fn steer_interrupt_without_active_steerable_session_returns_conflict() {\n let state = test_app_state();\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n \n let req = Request::builder()\n .method(\"POST\")\n@@ -10934,12 +11266,8 @@ async fn interrupt_with_active_steerable_session_forwards_interrupt() {\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n let stage_id = StageId::new(\"agent\", 1);\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n runs.get_mut(&run_id)\n@@ -10956,7 +11284,7 @@ async fn interrupt_with_active_steerable_session_forwards_interrupt() {\n \n let response = app.oneshot(req).await.unwrap();\n assert_status!(response, StatusCode::ACCEPTED).await;\n- let envelope = control_rx.recv().await.unwrap();\n+ let envelope = recv_worker_control_envelope(&mut control_rx).await;\n assert!(matches!(\n envelope.message,\n WorkerControlMessage::Interrupt {\n@@ -10971,12 +11299,8 @@ async fn steer_interrupt_with_active_steerable_session_forwards_combined_control\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n let stage_id = StageId::new(\"agent\", 1);\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n runs.get_mut(&run_id)\n@@ -10994,7 +11318,7 @@ async fn steer_interrupt_with_active_steerable_session_forwards_combined_control\n \n let response = app.oneshot(req).await.unwrap();\n assert_status!(response, StatusCode::ACCEPTED).await;\n- let envelope = control_rx.recv().await.unwrap();\n+ let envelope = recv_worker_control_envelope(&mut control_rx).await;\n assert!(matches!(\n envelope.message,\n WorkerControlMessage::InterruptThenSteer { ref text, .. } if text == \"try again\"\n@@ -11160,12 +11484,8 @@ async fn steer_with_active_acp_session_forwards_to_worker() {\n let state = test_app_state();\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n \n let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted {\n node_id: \"agent\".to_string(),\n@@ -11198,7 +11518,7 @@ async fn steer_with_active_acp_session_forwards_to_worker() {\n \n let response = app.oneshot(req).await.unwrap();\n assert_status!(response, StatusCode::ACCEPTED).await;\n- let envelope = control_rx.recv().await.unwrap();\n+ let envelope = recv_worker_control_envelope(&mut control_rx).await;\n assert!(matches!(\n envelope.message,\n WorkerControlMessage::Steer { ref text, .. } if text == \"try again\"\n@@ -11265,12 +11585,8 @@ async fn active_acp_steerable_marker_clears_on_terminal_paths() {\n let state = test_app_state();\n let app = crate::test_support::build_test_router(Arc::clone(&state));\n let run_id = fixtures::RUN_1;\n- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1);\n- let _temp_dir = insert_running_control_run(\n- &state,\n- run_id,\n- Some(RunAnswerTransport::Subprocess { control_tx }),\n- );\n+ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;\n+ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));\n let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted {\n node_id: \"agent\".to_string(),\n visit: 1,\n@@ -13200,11 +13516,13 @@ async fn pause_run_sets_pending_control_on_board_response() {\n let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;\n let run_id = run_id_str.parse::().unwrap();\n \n+ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n let managed_run = runs.get_mut(&run_id).expect(\"run should exist\");\n managed_run.status = RunStatus::Running;\n managed_run.worker_pid = Some(u32::MAX);\n+ managed_run.answer_transport = Some(transport);\n }\n \n let req = Request::builder()\n@@ -13320,11 +13638,13 @@ async fn unpause_run_sets_pending_control() {\n let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;\n let run_id = run_id_str.parse::().unwrap();\n \n+ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;\n {\n let mut runs = state.runs.lock().expect(\"runs lock poisoned\");\n let managed_run = runs.get_mut(&run_id).expect(\"run should exist\");\n managed_run.status = RunStatus::Paused { prior_block: None };\n managed_run.worker_pid = Some(u32::MAX);\n+ managed_run.answer_transport = Some(transport);\n }\n \n let req = Request::builder()\ndiff --git a/lib/crates/fabro-server/src/test_support.rs b/lib/crates/fabro-server/src/test_support.rs\nindex 11f80e0e7..585aa36e9 100644\n--- a/lib/crates/fabro-server/src/test_support.rs\n+++ b/lib/crates/fabro-server/src/test_support.rs\n@@ -249,6 +249,8 @@ impl TestAppStateBuilder {\n ),\n sandbox_provider_registry: self.sandbox_provider_registry,\n shutdown: CancellationToken::new(),\n+ #[cfg(test)]\n+ worker_control_bus: None,\n automation_materializer_override: self.automation_materializer,\n })\n }\ndiff --git a/lib/crates/fabro-server/src/worker_control/bus.rs b/lib/crates/fabro-server/src/worker_control/bus.rs\nnew file mode 100644\nindex 000000000..b6fadedb5\n--- /dev/null\n+++ b/lib/crates/fabro-server/src/worker_control/bus.rs\n@@ -0,0 +1,155 @@\n+use std::fmt;\n+\n+use fabro_interview::WorkerControlEnvelope;\n+use fabro_types::RunId;\n+use futures_util::future::BoxFuture;\n+use tokio::sync::mpsc;\n+\n+#[derive(Clone, PartialEq, Eq)]\n+pub(crate) struct WorkerControlMessageId(String);\n+\n+impl WorkerControlMessageId {\n+ #[must_use]\n+ pub(crate) fn new(value: impl Into) -> Self {\n+ Self(value.into())\n+ }\n+\n+ #[must_use]\n+ pub(crate) fn as_str(&self) -> &str {\n+ &self.0\n+ }\n+}\n+\n+impl fmt::Debug for WorkerControlMessageId {\n+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n+ f.debug_tuple(\"WorkerControlMessageId\")\n+ .field(&self.0)\n+ .finish()\n+ }\n+}\n+\n+impl fmt::Display for WorkerControlMessageId {\n+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n+ f.write_str(&self.0)\n+ }\n+}\n+\n+impl From for WorkerControlMessageId {\n+ fn from(value: String) -> Self {\n+ Self::new(value)\n+ }\n+}\n+\n+impl From<&str> for WorkerControlMessageId {\n+ fn from(value: &str) -> Self {\n+ Self::new(value)\n+ }\n+}\n+\n+/// Cursor for replaying a run's worker-control stream.\n+///\n+/// Future Redis Streams mapping:\n+/// - `Start` maps to `XREAD ... STREAMS fabro:run:{run_id}:control 0-0`.\n+/// - `After(id)` maps to `XREAD ... STREAMS fabro:run:{run_id}:control {id}`.\n+#[derive(Debug, Clone, PartialEq, Eq)]\n+pub(crate) enum WorkerControlCursor {\n+ Start,\n+ After(WorkerControlMessageId),\n+}\n+\n+impl WorkerControlCursor {\n+ pub(crate) fn from_after_query(after: Option<&str>) -> Result {\n+ match after {\n+ None => Ok(Self::Start),\n+ Some(value) if value.trim().is_empty() => Err(WorkerControlBusError::InvalidCursor {\n+ cursor: value.to_string(),\n+ reason: \"cursor id must not be empty\".to_string(),\n+ }),\n+ Some(value) => Ok(Self::After(WorkerControlMessageId::from(value))),\n+ }\n+ }\n+}\n+\n+#[derive(Debug, Clone, PartialEq, Eq)]\n+pub(crate) struct WorkerControlDelivery {\n+ pub(crate) id: WorkerControlMessageId,\n+ pub(crate) envelope: WorkerControlEnvelope,\n+}\n+\n+#[allow(\n+ dead_code,\n+ reason = \"The local backend does not construct every cross-backend error variant.\"\n+)]\n+#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]\n+pub(crate) enum WorkerControlBusError {\n+ #[error(\"worker control backend is closed\")]\n+ Closed,\n+ #[error(\"worker control backend is unavailable\")]\n+ Unavailable,\n+ #[error(\"worker control cursor `{cursor}` is invalid: {reason}\")]\n+ InvalidCursor { cursor: String, reason: String },\n+ #[error(\"timed out publishing worker control message\")]\n+ PublishTimeout,\n+}\n+\n+impl WorkerControlBusError {\n+ #[must_use]\n+ pub(crate) fn invalid_cursor(cursor: impl Into, reason: impl Into) -> Self {\n+ Self::InvalidCursor {\n+ cursor: cursor.into(),\n+ reason: reason.into(),\n+ }\n+ }\n+}\n+\n+pub(crate) type WorkerControlReceiver =\n+ mpsc::Receiver>;\n+\n+pub(crate) trait WorkerControlBus: Send + Sync {\n+ fn publish(\n+ &self,\n+ run_id: RunId,\n+ envelope: WorkerControlEnvelope,\n+ ) -> BoxFuture<'_, Result>;\n+\n+ fn subscribe(\n+ &self,\n+ run_id: RunId,\n+ cursor: WorkerControlCursor,\n+ ) -> BoxFuture<'_, Result>;\n+\n+ fn cleanup_run(&self, run_id: RunId) -> BoxFuture<'_, ()>;\n+}\n+\n+#[cfg(test)]\n+mod tests {\n+ use super::*;\n+\n+ #[test]\n+ fn message_id_equality_and_debug_are_opaque() {\n+ let first = WorkerControlMessageId::from(\"local:42\");\n+ let same = WorkerControlMessageId::from(\"local:42\");\n+ let different = WorkerControlMessageId::from(\"local:43\");\n+\n+ assert_eq!(first, same);\n+ assert_ne!(first, different);\n+ assert_eq!(format!(\"{first}\"), \"local:42\");\n+ assert_eq!(format!(\"{first:?}\"), \"WorkerControlMessageId(\\\"local:42\\\")\");\n+ }\n+\n+ #[test]\n+ fn absent_after_query_parses_as_start_cursor() {\n+ assert_eq!(\n+ WorkerControlCursor::from_after_query(None).unwrap(),\n+ WorkerControlCursor::Start\n+ );\n+ }\n+\n+ #[test]\n+ fn present_after_query_parses_as_after_cursor() {\n+ assert_eq!(\n+ WorkerControlCursor::from_after_query(Some(\"local:42\")).unwrap(),\n+ WorkerControlCursor::After(WorkerControlMessageId::from(\"local:42\"))\n+ );\n+ }\n+}\ndiff --git a/lib/crates/fabro-server/src/worker_control/local.rs b/lib/crates/fabro-server/src/worker_control/local.rs\nnew file mode 100644\nindex 000000000..219ff18ec\n--- /dev/null\n+++ b/lib/crates/fabro-server/src/worker_control/local.rs\n@@ -0,0 +1,458 @@\n+use std::collections::{HashMap, VecDeque};\n+use std::sync::atomic::{AtomicU64, Ordering};\n+use std::sync::{Arc, Mutex};\n+\n+use fabro_interview::WorkerControlEnvelope;\n+use fabro_types::RunId;\n+use futures_util::FutureExt;\n+use futures_util::future::BoxFuture;\n+use tokio::sync::{Notify, mpsc};\n+\n+use super::{\n+ WorkerControlBus, WorkerControlBusError, WorkerControlCursor, WorkerControlDelivery,\n+ WorkerControlMessageId, WorkerControlReceiver,\n+};\n+\n+pub(crate) const LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN: usize = 1024;\n+const LOCAL_WORKER_CONTROL_SUBSCRIBER_BUFFER: usize = 64;\n+\n+#[derive(Default)]\n+pub(crate) struct LocalWorkerControlBus {\n+ streams: Arc>>,\n+ next_sequence: Arc,\n+}\n+\n+struct LocalRunControlStream {\n+ messages: VecDeque,\n+ notify: Arc,\n+ has_trimmed: bool,\n+}\n+\n+#[derive(Clone)]\n+struct LocalMessage {\n+ sequence: u64,\n+ delivery: WorkerControlDelivery,\n+}\n+\n+impl LocalWorkerControlBus {\n+ #[must_use]\n+ pub(crate) fn new() -> Self {\n+ Self {\n+ streams: Arc::new(Mutex::new(HashMap::new())),\n+ next_sequence: Arc::new(AtomicU64::new(1)),\n+ }\n+ }\n+\n+ fn next_message_id(&self) -> (u64, WorkerControlMessageId) {\n+ let sequence = self.next_sequence.fetch_add(1, Ordering::Relaxed);\n+ (\n+ sequence,\n+ WorkerControlMessageId::new(format!(\"local:{sequence}\")),\n+ )\n+ }\n+\n+ fn subscribe_inner(\n+ &self,\n+ run_id: RunId,\n+ cursor: &WorkerControlCursor,\n+ ) -> Result {\n+ let (next_sequence, notify) = {\n+ let mut streams = self\n+ .streams\n+ .lock()\n+ .expect(\"worker control streams poisoned\");\n+ let stream = match cursor {\n+ WorkerControlCursor::Start => streams\n+ .entry(run_id)\n+ .or_insert_with(LocalRunControlStream::new),\n+ WorkerControlCursor::After(id) => streams.get_mut(&run_id).ok_or_else(|| {\n+ WorkerControlBusError::invalid_cursor(\n+ id.as_str(),\n+ \"message id is not retained for this run\",\n+ )\n+ })?,\n+ };\n+ let next_sequence = stream.next_sequence_for_cursor(cursor)?;\n+ (next_sequence, Arc::clone(&stream.notify))\n+ };\n+\n+ let (tx, rx) = mpsc::channel(LOCAL_WORKER_CONTROL_SUBSCRIBER_BUFFER);\n+ let streams = Arc::clone(&self.streams);\n+ tokio::spawn(async move {\n+ local_subscription_task(streams, run_id, notify, next_sequence, tx).await;\n+ });\n+ Ok(rx)\n+ }\n+\n+ #[cfg(test)]\n+ pub(crate) fn retained_len(&self, run_id: &RunId) -> usize {\n+ let streams = self\n+ .streams\n+ .lock()\n+ .expect(\"worker control streams poisoned\");\n+ streams\n+ .get(run_id)\n+ .map_or(0, |stream| stream.messages.len())\n+ }\n+}\n+\n+impl LocalRunControlStream {\n+ fn new() -> Self {\n+ Self {\n+ messages: VecDeque::new(),\n+ notify: Arc::new(Notify::new()),\n+ has_trimmed: false,\n+ }\n+ }\n+\n+ fn next_sequence_for_cursor(\n+ &self,\n+ cursor: &WorkerControlCursor,\n+ ) -> Result, WorkerControlBusError> {\n+ match cursor {\n+ WorkerControlCursor::Start if self.has_trimmed => Err(\n+ WorkerControlBusError::invalid_cursor(\"start\", \"retained local stream is trimmed\"),\n+ ),\n+ WorkerControlCursor::Start => Ok(self.messages.front().map(|message| message.sequence)),\n+ WorkerControlCursor::After(id) => {\n+ let requested_sequence = parse_local_sequence(id)?;\n+ let Some(position) = self\n+ .messages\n+ .iter()\n+ .position(|message| message.delivery.id == *id)\n+ else {\n+ return Err(WorkerControlBusError::invalid_cursor(\n+ id.as_str(),\n+ \"message id is not retained for this run\",\n+ ));\n+ };\n+ Ok(self\n+ .messages\n+ .get(position + 1)\n+ .map(|message| message.sequence)\n+ .or(Some(requested_sequence.saturating_add(1))))\n+ }\n+ }\n+ }\n+\n+ fn trim_retained(&mut self) {\n+ while self.messages.len() > LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN {\n+ self.messages.pop_front();\n+ self.has_trimmed = true;\n+ }\n+ }\n+}\n+\n+fn parse_local_sequence(id: &WorkerControlMessageId) -> Result {\n+ let Some(raw) = id.as_str().strip_prefix(\"local:\") else {\n+ return Err(WorkerControlBusError::invalid_cursor(\n+ id.as_str(),\n+ \"local backend only understands local message ids\",\n+ ));\n+ };\n+ raw.parse::().map_err(|_| {\n+ WorkerControlBusError::invalid_cursor(id.as_str(), \"local message id sequence is invalid\")\n+ })\n+}\n+\n+async fn local_subscription_task(\n+ streams: Arc>>,\n+ run_id: RunId,\n+ notify: Arc,\n+ mut next_sequence: Option,\n+ tx: mpsc::Sender>,\n+) {\n+ loop {\n+ // Register interest *before* inspecting the stream so that a publish\n+ // racing with this read does not cause a lost wakeup. `notify_waiters`\n+ // does not leave a permit for future `notified()` calls.\n+ let notified = notify.notified();\n+ tokio::pin!(notified);\n+ notified.as_mut().enable();\n+\n+ let collected = {\n+ let streams_guard = streams.lock().expect(\"worker control streams poisoned\");\n+ match streams_guard.get(&run_id) {\n+ None => None,\n+ Some(stream) => {\n+ // A `Start` subscriber that joined before any publish lazily\n+ // adopts the first retained message as its cursor.\n+ if next_sequence.is_none() && stream.has_trimmed {\n+ Some(Err(WorkerControlBusError::invalid_cursor(\n+ \"start\",\n+ \"retained local stream is trimmed\",\n+ )))\n+ } else {\n+ if next_sequence.is_none() {\n+ next_sequence = stream.messages.front().map(|message| message.sequence);\n+ }\n+ match next_sequence {\n+ None => Some(Ok(Vec::new())),\n+ Some(next) => Some(collect_messages_from(&stream.messages, next)),\n+ }\n+ }\n+ }\n+ }\n+ };\n+\n+ let messages = match collected {\n+ None => return,\n+ Some(Err(err)) => {\n+ let _ = tx.send(Err(err)).await;\n+ return;\n+ }\n+ Some(Ok(messages)) => messages,\n+ };\n+\n+ if messages.is_empty() {\n+ notified.await;\n+ continue;\n+ }\n+\n+ for message in messages {\n+ next_sequence = Some(message.sequence.saturating_add(1));\n+ if tx.send(Ok(message.delivery)).await.is_err() {\n+ return;\n+ }\n+ }\n+ }\n+}\n+\n+/// Returns the retained messages with `sequence >= next`. Cheaper than scanning\n+/// the whole deque: `partition_point` is O(log N) and we only clone the tail.\n+fn collect_messages_from(\n+ messages: &VecDeque,\n+ next: u64,\n+) -> Result, WorkerControlBusError> {\n+ let Some(first_sequence) = messages.front().map(|message| message.sequence) else {\n+ return Ok(Vec::new());\n+ };\n+ if next < first_sequence {\n+ return Err(WorkerControlBusError::invalid_cursor(\n+ format!(\"local:{next}\"),\n+ \"subscriber fell behind retained local messages\",\n+ ));\n+ }\n+ let start = messages.partition_point(|message| message.sequence < next);\n+ Ok(messages.iter().skip(start).cloned().collect())\n+}\n+\n+impl WorkerControlBus for LocalWorkerControlBus {\n+ fn publish(\n+ &self,\n+ run_id: RunId,\n+ envelope: WorkerControlEnvelope,\n+ ) -> BoxFuture<'_, Result> {\n+ async move {\n+ let (sequence, id) = self.next_message_id();\n+ let delivery = WorkerControlDelivery {\n+ id: id.clone(),\n+ envelope,\n+ };\n+ let notify = {\n+ let mut streams = self\n+ .streams\n+ .lock()\n+ .expect(\"worker control streams poisoned\");\n+ let stream = streams\n+ .entry(run_id)\n+ .or_insert_with(LocalRunControlStream::new);\n+ stream\n+ .messages\n+ .push_back(LocalMessage { sequence, delivery });\n+ stream.trim_retained();\n+ Arc::clone(&stream.notify)\n+ };\n+ notify.notify_waiters();\n+ Ok(id)\n+ }\n+ .boxed()\n+ }\n+\n+ fn subscribe(\n+ &self,\n+ run_id: RunId,\n+ cursor: WorkerControlCursor,\n+ ) -> BoxFuture<'_, Result> {\n+ async move { self.subscribe_inner(run_id, &cursor) }.boxed()\n+ }\n+\n+ fn cleanup_run(&self, run_id: RunId) -> BoxFuture<'_, ()> {\n+ async move {\n+ let notify = {\n+ let mut streams = self\n+ .streams\n+ .lock()\n+ .expect(\"worker control streams poisoned\");\n+ streams.remove(&run_id).map(|stream| stream.notify)\n+ };\n+ if let Some(notify) = notify {\n+ notify.notify_waiters();\n+ }\n+ }\n+ .boxed()\n+ }\n+}\n+\n+#[cfg(test)]\n+mod tests {\n+ use std::time::Duration;\n+\n+ use fabro_interview::WorkerControlMessage;\n+ use fabro_types::fixtures;\n+\n+ use super::*;\n+\n+ async fn recv_delivery(receiver: &mut WorkerControlReceiver) -> WorkerControlDelivery {\n+ tokio::time::timeout(Duration::from_secs(1), receiver.recv())\n+ .await\n+ .expect(\"delivery should arrive\")\n+ .expect(\"subscription should remain open\")\n+ .expect(\"delivery should be ok\")\n+ }\n+\n+ #[tokio::test]\n+ async fn messages_publish_in_order() {\n+ let bus = LocalWorkerControlBus::new();\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run())\n+ .await\n+ .unwrap();\n+\n+ let mut receiver = bus\n+ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start)\n+ .await\n+ .unwrap();\n+\n+ assert!(matches!(\n+ recv_delivery(&mut receiver).await.envelope.message,\n+ WorkerControlMessage::RunCancel\n+ ));\n+ assert!(matches!(\n+ recv_delivery(&mut receiver).await.envelope.message,\n+ WorkerControlMessage::RunPause\n+ ));\n+ }\n+\n+ #[tokio::test]\n+ async fn active_subscriber_receives_message_published_after_subscription() {\n+ let bus = LocalWorkerControlBus::new();\n+ let mut receiver = bus\n+ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start)\n+ .await\n+ .unwrap();\n+\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+\n+ assert!(matches!(\n+ recv_delivery(&mut receiver).await.envelope.message,\n+ WorkerControlMessage::RunCancel\n+ ));\n+ }\n+\n+ #[tokio::test]\n+ async fn messages_published_before_subscription_replay_from_start() {\n+ let bus = LocalWorkerControlBus::new();\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+\n+ let mut receiver = bus\n+ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start)\n+ .await\n+ .unwrap();\n+\n+ assert!(matches!(\n+ recv_delivery(&mut receiver).await.envelope.message,\n+ WorkerControlMessage::RunCancel\n+ ));\n+ }\n+\n+ #[tokio::test]\n+ async fn after_cursor_receives_only_later_messages() {\n+ let bus = LocalWorkerControlBus::new();\n+ let first_id = bus\n+ .publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run())\n+ .await\n+ .unwrap();\n+\n+ let mut receiver = bus\n+ .subscribe(\n+ fixtures::RUN_1,\n+ WorkerControlCursor::After(first_id.clone()),\n+ )\n+ .await\n+ .unwrap();\n+\n+ assert!(matches!(\n+ recv_delivery(&mut receiver).await.envelope.message,\n+ WorkerControlMessage::RunPause\n+ ));\n+ assert!(receiver.try_recv().is_err());\n+ }\n+\n+ #[tokio::test]\n+ async fn trimming_bounds_retained_messages_and_invalidates_old_cursor() {\n+ let bus = LocalWorkerControlBus::new();\n+ let old_id = bus\n+ .publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+ for _ in 0..LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN {\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run())\n+ .await\n+ .unwrap();\n+ }\n+\n+ assert_eq!(\n+ bus.retained_len(&fixtures::RUN_1),\n+ LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN\n+ );\n+ let err = bus\n+ .subscribe(fixtures::RUN_1, WorkerControlCursor::After(old_id))\n+ .await\n+ .unwrap_err();\n+ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. }));\n+\n+ let err = bus\n+ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start)\n+ .await\n+ .unwrap_err();\n+ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. }));\n+ }\n+\n+ #[tokio::test]\n+ async fn after_cursor_for_unknown_run_does_not_create_stream() {\n+ let bus = LocalWorkerControlBus::new();\n+ let err = bus\n+ .subscribe(\n+ fixtures::RUN_1,\n+ WorkerControlCursor::After(WorkerControlMessageId::new(\"local:1\")),\n+ )\n+ .await\n+ .unwrap_err();\n+\n+ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. }));\n+ assert_eq!(bus.retained_len(&fixtures::RUN_1), 0);\n+ }\n+\n+ #[tokio::test]\n+ async fn cleanup_removes_retained_messages() {\n+ let bus = LocalWorkerControlBus::new();\n+ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run())\n+ .await\n+ .unwrap();\n+ assert_eq!(bus.retained_len(&fixtures::RUN_1), 1);\n+\n+ bus.cleanup_run(fixtures::RUN_1).await;\n+\n+ assert_eq!(bus.retained_len(&fixtures::RUN_1), 0);\n+ }\n+}\ndiff --git a/lib/crates/fabro-server/src/worker_control/mod.rs b/lib/crates/fabro-server/src/worker_control/mod.rs\nnew file mode 100644\nindex 000000000..6b6f70896\n--- /dev/null\n+++ b/lib/crates/fabro-server/src/worker_control/mod.rs\n@@ -0,0 +1,8 @@\n+mod bus;\n+mod local;\n+\n+pub(crate) use bus::{\n+ WorkerControlBus, WorkerControlBusError, WorkerControlCursor, WorkerControlDelivery,\n+ WorkerControlMessageId, WorkerControlReceiver,\n+};\n+pub(crate) use local::LocalWorkerControlBus;\ndiff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs\nindex 9d3f08d1c..23c90f3ff 100644\n--- a/lib/crates/fabro-workflow/src/operations/start.rs\n+++ b/lib/crates/fabro-workflow/src/operations/start.rs\n@@ -26,6 +26,7 @@ use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind};\n use fabro_vault::Vault;\n use tokio::runtime::Handle;\n use tokio::sync::RwLock as AsyncRwLock;\n+use tokio::time;\n use tokio_util::sync::CancellationToken;\n \n use crate::artifact_upload::ArtifactSink;\n@@ -987,6 +988,24 @@ impl Drop for DetachedRunBootstrapGuard {\n \n const POSTRUN_INTERRUPTED_MESSAGE: &str = \"Run interrupted before post-run finalization completed.\";\n const POSTRUN_CANCELLED_MESSAGE: &str = \"Run cancelled before post-run finalization completed.\";\n+const DETACHED_COMPLETION_GUARD_TERMINAL_GRACE: Duration = Duration::from_millis(25);\n+\n+async fn run_store_reaches_terminal(run_store: &RunStoreHandle, timeout: Duration) -> bool {\n+ let start = Instant::now();\n+ loop {\n+ if run_store\n+ .state()\n+ .await\n+ .is_ok_and(|state| state.status.is_terminal())\n+ {\n+ return true;\n+ }\n+ if start.elapsed() >= timeout {\n+ return false;\n+ }\n+ time::sleep(Duration::from_millis(10)).await;\n+ }\n+}\n \n struct DetachedRunCompletionGuard {\n event_sink: RunEventSink,\n@@ -1044,6 +1063,11 @@ impl Drop for DetachedRunCompletionGuard {\n let run_store = self.run_store.clone();\n if let Ok(handle) = Handle::try_current() {\n handle.spawn(async move {\n+ if run_store_reaches_terminal(&run_store, DETACHED_COMPLETION_GUARD_TERMINAL_GRACE)\n+ .await\n+ {\n+ return;\n+ }\n emit_workflow_run_failed(\n run_id,\n &run_store,\n", + "summary": { + "files_changed": 23, + "additions": 2680, + "deletions": 344 + } + } } ], - "conclusion": null, + "conclusion": { + "timestamp": "2026-05-28T01:08:24.004738Z", + "status": "succeeded", + "timing": { + "wall_time_ms": 3065977, + "inference_time_ms": 1384357, + "tool_time_ms": 1643662, + "active_time_ms": 3028019 + }, + "final_git_commit_sha": "2446175b1e9f9e836b6ca93acdb0c53d9279a8cd", + "stages": [ + { + "stage_id": "start", + "stage_label": "start", + "timing": { + "wall_time_ms": 0, + "inference_time_ms": 0, + "tool_time_ms": 0, + "active_time_ms": 0 + }, + "retries": 0 + }, + { + "stage_id": "toolchain", + "stage_label": "toolchain", + "timing": { + "wall_time_ms": 1405, + "inference_time_ms": 0, + "tool_time_ms": 1398, + "active_time_ms": 1398 + }, + "retries": 0 + }, + { + "stage_id": "preflight_compile", + "stage_label": "preflight_compile", + "timing": { + "wall_time_ms": 128521, + "inference_time_ms": 0, + "tool_time_ms": 128511, + "active_time_ms": 128511 + }, + "retries": 0 + }, + { + "stage_id": "preflight_lint", + "stage_label": "preflight_lint", + "timing": { + "wall_time_ms": 143356, + "inference_time_ms": 0, + "tool_time_ms": 143345, + "active_time_ms": 143345 + }, + "retries": 0 + }, + { + "stage_id": "implement", + "stage_label": "implement", + "timing": { + "wall_time_ms": 1252463, + "inference_time_ms": 818239, + "tool_time_ms": 427741, + "active_time_ms": 1245980 + }, + "billing_usd_micros": 13047333, + "retries": 0 + }, + { + "stage_id": "simplify_opus", + "stage_label": "simplify_opus", + "timing": { + "wall_time_ms": 665383, + "inference_time_ms": 364281, + "tool_time_ms": 300326, + "active_time_ms": 664607 + }, + "billing_usd_micros": 5696578, + "retries": 0 + }, + { + "stage_id": "simplify_gpt", + "stage_label": "simplify_gpt", + "timing": { + "wall_time_ms": 300123, + "inference_time_ms": 201837, + "tool_time_ms": 97609, + "active_time_ms": 299446 + }, + "billing_usd_micros": 2598169, + "retries": 0 + }, + { + "stage_id": "verify", + "stage_label": "verify", + "timing": { + "wall_time_ms": 544748, + "inference_time_ms": 0, + "tool_time_ms": 544732, + "active_time_ms": 544732 + }, + "retries": 0 + } + ], + "billing": { + "input_tokens": 2120311, + "output_tokens": 37875, + "total_tokens": 15536920, + "reasoning_tokens": 12443, + "cache_read_tokens": 12904958, + "cache_write_tokens": 461333, + "total_usd_micros": 21342080 + }, + "total_retries": 0, + "diff": {} + }, "sandbox": { "kind": "ready", "plan": { @@ -1898,6 +2020,40 @@ }, "state": "succeeded" }, + "exit@1": { + "first_event_seq": 988, + "prompt": null, + "response": null, + "completion": { + "outcome": "succeeded", + "notes": null, + "failure_reason": null, + "timestamp": "2026-05-28T01:08:23.965487Z" + }, + "provider_used": null, + "diff": null, + "script_invocation": null, + "script_timing": null, + "parallel_results": null, + "output": null, + "started_at": "2026-05-28T01:08:23.965462Z", + "handler": "exit", + "timing": { + "wall_time_ms": 0, + "inference_time_ms": 0, + "tool_time_ms": 0, + "active_time_ms": 0 + }, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "state": "succeeded" + }, "preflight_compile@1": { "first_event_seq": 31, "prompt": null, @@ -2528,7 +2684,12 @@ "first_event_seq": 978, "prompt": null, "response": null, - "completion": null, + "completion": { + "outcome": "succeeded", + "notes": "Script completed: git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\\bActorRef\\b|\\bActorKind\\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\\s*==\\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", + "failure_reason": null, + "timestamp": "2026-05-28T01:08:19.664723Z" + }, "provider_used": null, "diff": null, "script_invocation": { @@ -2536,11 +2697,27 @@ "command": "exec 2>&1\ngit fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\\bActorRef\\b|\\bActorKind\\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\\s*==\\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", "language": "shell" }, - "script_timing": null, + "script_timing": { + "output": "blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da", + "exit_code": 0, + "duration_ms": 544732, + "termination": "exited", + "output_bytes": 213463, + "live_streaming": true + }, "parallel_results": null, "output": null, + "output_bytes": 213463, + "live_streaming": true, + "termination": "exited", "started_at": "2026-05-28T00:59:14.909667Z", "handler": "command", + "timing": { + "wall_time_ms": 544748, + "inference_time_ms": 0, + "tool_time_ms": 544732, + "active_time_ms": 544732 + }, "usage": { "input_tokens": 0, "output_tokens": 0, @@ -2549,7 +2726,7 @@ "cache_read_tokens": 0, "cache_write_tokens": 0 }, - "state": "running" + "state": "succeeded" }, "start@1": { "first_event_seq": 17, diff --git a/stages/008-verify@1/diff.patch b/stages/008-verify@1/diff.patch new file mode 100644 index 000000000..0a139bce3 --- /dev/null +++ b/stages/008-verify@1/diff.patch @@ -0,0 +1,3628 @@ +diff --git a/Cargo.lock b/Cargo.lock +index 0986b55a6..0fdc7afc9 100644 +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -1889,6 +1889,7 @@ dependencies = [ + "tempfile", + "thiserror 2.0.18", + "tokio", ++ "tokio-tungstenite 0.26.2", + "tokio-util", + "toml 0.8.23", + "toml_edit", +@@ -2408,6 +2409,7 @@ dependencies = [ + "thiserror 2.0.18", + "tokio", + "tokio-stream", ++ "tokio-tungstenite 0.26.2", + "tokio-util", + "toml 0.8.23", + "toml_edit", +diff --git a/lib/crates/fabro-cli/Cargo.toml b/lib/crates/fabro-cli/Cargo.toml +index a4d15b5b5..d0b8b7373 100644 +--- a/lib/crates/fabro-cli/Cargo.toml ++++ b/lib/crates/fabro-cli/Cargo.toml +@@ -61,6 +61,7 @@ anyhow.workspace = true + miette.workspace = true + dotenvy.workspace = true + tokio.workspace = true ++tokio-tungstenite.workspace = true + tracing.workspace = true + tracing-subscriber.workspace = true + tracing-appender.workspace = true +@@ -128,3 +129,4 @@ fabro-test = { workspace = true } + fabro-macros = { path = "../fabro-macros" } + hkdf.workspace = true + reqwest = { workspace = true, features = ["cookies"] } ++tokio = { workspace = true, features = ["test-util", "macros"] } +diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs +index fb3b31d43..ac1a2bbd0 100644 +--- a/lib/crates/fabro-cli/src/commands/run/runner.rs ++++ b/lib/crates/fabro-cli/src/commands/run/runner.rs +@@ -1,10 +1,4 @@ +-#![expect( +- clippy::disallowed_types, +- reason = "sync CLI `run` subprocess wrapper: reads server subprocess stdout line-by-line via \ +- std::io::BufReader; not on a Tokio path" +-)] +- +-use std::io::{BufRead as StdBufRead, BufReader as StdBufReader}; ++use std::collections::{HashSet, VecDeque}; + use std::path::{Path, PathBuf}; + use std::sync::Arc; + use std::time::Duration; +@@ -12,10 +6,14 @@ use std::time::Duration; + use anyhow::{Context, Result, anyhow}; + use async_trait::async_trait; + use fabro_api::types::RunManifest; ++use fabro_client::ServerTarget; + use fabro_config::user::active_settings_path; + use fabro_config::{ServerSettingsBuilder, Storage, load_llm_catalog_settings}; + use fabro_interview::{ +- AnswerSubmission, ControlInterviewer, WorkerControlEnvelope, WorkerControlMessage, ++ AnswerSubmission, ControlInterviewer, WORKER_CONTROL_INVALID_CURSOR_REASON, ++ WORKER_CONTROL_PONG_TIMEOUT_REASON, WORKER_CONTROL_WS_LIVENESS_TIMEOUT, ++ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlDeliveryFrame, WorkerControlEnvelope, ++ WorkerControlMessage, + }; + use fabro_model::Catalog; + use fabro_server::run_tool_manifest; +@@ -34,11 +32,23 @@ use fabro_workflow::operations::{self, StartServices}; + use fabro_workflow::run_control::RunControlState; + use fabro_workflow::runtime_store::{RunStoreBackend, RunStoreHandle}; + use fabro_workflow::services::FabroRunToolServices; ++use futures::{SinkExt, StreamExt}; + use jsonwebtoken::dangerous::insecure_decode; ++#[cfg(test)] ++use tokio::io::DuplexStream; ++use tokio::net::TcpStream; ++#[cfg(unix)] ++use tokio::net::UnixStream; + #[cfg(unix)] + use tokio::signal::unix::{SignalKind, signal}; +-use tokio::sync::{Mutex, RwLock as AsyncRwLock, mpsc}; +-use tokio::time::sleep; ++use tokio::sync::{Mutex, RwLock as AsyncRwLock, oneshot}; ++use tokio::task::JoinHandle; ++use tokio::time::{self, Instant, MissedTickBehavior}; ++use tokio_tungstenite::tungstenite::client::IntoClientRequest; ++use tokio_tungstenite::tungstenite::http::{HeaderValue, Request as WebSocketRequest, header}; ++use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; ++use tokio_tungstenite::tungstenite::protocol::{self, Message as WebSocketMessage}; ++use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async, tungstenite}; + use tokio_util::sync::CancellationToken; + + use crate::args::RunWorkerMode; +@@ -75,7 +85,7 @@ pub(crate) async fn execute( + let _ = fabro_proc::title_init(); + set_worker_title(&run_id, initial_worker_title_phase(mode)); + +- let target = server.parse::()?; ++ let target = server.parse::()?; + let client = server_client::connect_server_target_with_bearer(&target, worker_token).await?; + let run_store = HttpRunStore::connect(run_id, client.clone_for_reuse()).await?; + let run_state = run_store +@@ -110,13 +120,24 @@ pub(crate) async fn execute( + let cancel_token = CancellationToken::new(); + let emitter = Arc::new(Emitter::new(run_id)); + let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(Arc::clone(&emitter))); +- spawn_worker_control_stream( +- Arc::clone(&interviewer), +- cancel_token.clone(), +- Arc::clone(&steering_hub), +- )?; + let run_control = RunControlState::new(); + install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?; ++ let mut control_manager = if run_state.status.is_terminal() { ++ None ++ } else { ++ Some(spawn_worker_control_manager( ++ target.clone(), ++ run_id, ++ worker_token.to_owned(), ++ Arc::clone(&interviewer), ++ cancel_token.clone(), ++ Arc::clone(&steering_hub), ++ Arc::clone(&run_control), ++ )) ++ }; ++ if let Some(control_manager) = &mut control_manager { ++ control_manager.wait_for_first_connection().await?; ++ } + let vault = load_worker_vault(storage_dir.as_deref())?; + let github_app = { + let vault_guard = match &vault { +@@ -158,13 +179,26 @@ pub(crate) async fn execute( + fabro_run_tools, + }; + +- match mode { +- RunWorkerMode::Start => { +- operations::start(&run_dir, services).await?; ++ let execution = async { ++ match mode { ++ RunWorkerMode::Start => operations::start(&run_dir, services).await, ++ RunWorkerMode::Resume => operations::resume(&run_dir, services).await, + } +- RunWorkerMode::Resume => { +- operations::resume(&run_dir, services).await?; ++ }; ++ ++ if let Some(mut control_manager) = control_manager { ++ tokio::select! { ++ result = execution => { ++ control_manager.finish(); ++ result?; ++ } ++ fatal = control_manager.fatal_control_loss() => { ++ control_manager.finish(); ++ return Err(fatal); ++ } + } ++ } else { ++ execution.await?; + } + + Ok(()) +@@ -254,96 +288,510 @@ fn load_worker_vault(storage_dir: Option<&Path>) -> Result, +- cancel_token: CancellationToken, +- steering_hub: Arc, +-) -> Result<()> { +- let (event_tx, event_rx) = mpsc::unbounded_channel(); +- tokio::spawn(handle_worker_control_stream_events( +- interviewer, +- cancel_token, +- steering_hub, +- event_rx, +- )); +- std::thread::Builder::new() +- .name("fabro-worker-control".to_string()) +- .spawn(move || { +- read_worker_control_stream_blocking(StdBufReader::new(std::io::stdin()), &event_tx); +- }) +- .context("failed to spawn worker control reader thread")?; +- Ok(()) ++#[derive(Default)] ++struct AppliedWorkerControlDeliveryIds { ++ last: Option, ++ order: VecDeque, ++ recent: HashSet, + } + +-fn read_worker_control_stream_blocking( +- mut reader: R, +- event_tx: &mpsc::UnboundedSender, +-) where +- R: StdBufRead, +-{ +- let mut line = String::new(); +- loop { +- line.clear(); +- match reader.read_line(&mut line) { +- Ok(0) | Err(_) => { +- let _ = event_tx.send(WorkerControlStreamEvent::Eof); +- break; +- } +- Ok(_) => { +- let line = line.trim_end_matches(['\r', '\n']).to_string(); +- if event_tx.send(WorkerControlStreamEvent::Line(line)).is_err() { +- break; +- } ++impl AppliedWorkerControlDeliveryIds { ++ fn last_applied_id(&self) -> Option<&str> { ++ self.last.as_deref() ++ } ++ ++ fn contains(&self, id: &str) -> bool { ++ self.recent.contains(id) ++ } ++ ++ fn record(&mut self, id: String) { ++ if !self.recent.insert(id.clone()) { ++ self.last = Some(id); ++ return; ++ } ++ self.order.push_back(id.clone()); ++ self.last = Some(id); ++ while self.order.len() > WORKER_CONTROL_APPLIED_ID_DEDUPE_CAPACITY { ++ if let Some(evicted) = self.order.pop_front() { ++ self.recent.remove(&evicted); + } + } + } + } + +-async fn handle_worker_control_stream_events( ++struct WorkerControlManagerHandle { ++ first_connection: Option>>, ++ fatal: Option>, ++ done: CancellationToken, ++ task: JoinHandle<()>, ++} ++ ++impl WorkerControlManagerHandle { ++ async fn wait_for_first_connection(&mut self) -> Result<()> { ++ let receiver = self ++ .first_connection ++ .take() ++ .context("worker control first-connection receiver missing")?; ++ receiver ++ .await ++ .context("worker control manager stopped before first connection")? ++ } ++ ++ async fn fatal_control_loss(&mut self) -> anyhow::Error { ++ let Some(receiver) = self.fatal.take() else { ++ return anyhow!("worker control fatal receiver missing"); ++ }; ++ receiver ++ .await ++ .unwrap_or_else(|_| anyhow!("worker control manager stopped before workflow completed")) ++ } ++ ++ fn finish(&self) { ++ self.done.cancel(); ++ self.task.abort(); ++ } ++} ++ ++#[derive(Debug)] ++struct WorkerControlStreamConnectRequest { ++ request: WebSocketRequest<()>, ++ unix_socket_path: Option, ++} ++ ++impl WorkerControlStreamConnectRequest { ++ fn request_for_tungstenite(&self) -> WebSocketRequest<()> { ++ self.request.clone() ++ } ++ ++ #[cfg(test)] ++ fn uri(&self) -> String { ++ self.request.uri().to_string() ++ } ++ ++ #[cfg(test)] ++ fn authorization(&self) -> Option<&str> { ++ self.request ++ .headers() ++ .get(header::AUTHORIZATION) ++ .and_then(|value| value.to_str().ok()) ++ } ++} ++ ++enum WorkerControlSocket { ++ Tcp(Box>>), ++ #[cfg(unix)] ++ Unix(Box>), ++ #[cfg(test)] ++ Test(Box>), ++} ++ ++impl WorkerControlSocket { ++ async fn send(&mut self, message: WebSocketMessage) -> Result<(), tungstenite::Error> { ++ match self { ++ Self::Tcp(socket) => socket.send(message).await, ++ #[cfg(unix)] ++ Self::Unix(socket) => socket.send(message).await, ++ #[cfg(test)] ++ Self::Test(socket) => socket.send(message).await, ++ } ++ } ++ ++ async fn next(&mut self) -> Option> { ++ match self { ++ Self::Tcp(socket) => socket.next().await, ++ #[cfg(unix)] ++ Self::Unix(socket) => socket.next().await, ++ #[cfg(test)] ++ Self::Test(socket) => socket.next().await, ++ } ++ } ++} ++ ++#[derive(Debug)] ++enum WorkerControlConnectError { ++ InvalidCursor, ++ Other(anyhow::Error), ++} ++ ++fn spawn_worker_control_manager( ++ target: ServerTarget, ++ run_id: RunId, ++ worker_token: String, ++ interviewer: Arc, ++ cancel_token: CancellationToken, ++ steering_hub: Arc, ++ run_control: Arc, ++) -> WorkerControlManagerHandle { ++ let (first_tx, first_rx) = oneshot::channel(); ++ let (fatal_tx, fatal_rx) = oneshot::channel(); ++ let done = CancellationToken::new(); ++ let task_done = done.clone(); ++ let task = tokio::spawn(async move { ++ run_worker_control_manager( ++ target, ++ run_id, ++ worker_token, ++ interviewer, ++ cancel_token, ++ steering_hub, ++ run_control, ++ task_done, ++ first_tx, ++ fatal_tx, ++ ) ++ .await; ++ }); ++ WorkerControlManagerHandle { ++ first_connection: Some(first_rx), ++ fatal: Some(fatal_rx), ++ done, ++ task, ++ } ++} ++ ++#[allow( ++ clippy::too_many_arguments, ++ reason = "Worker control manager owns the worker-side control dependencies." ++)] ++async fn run_worker_control_manager( ++ target: ServerTarget, ++ run_id: RunId, ++ worker_token: String, + interviewer: Arc, + cancel_token: CancellationToken, + steering_hub: Arc, +- mut event_rx: mpsc::UnboundedReceiver, ++ run_control: Arc, ++ done: CancellationToken, ++ first_tx: oneshot::Sender>, ++ fatal_tx: oneshot::Sender, + ) { +- while let Some(event) = event_rx.recv().await { +- match event { +- WorkerControlStreamEvent::Line(line) => { +- apply_worker_control_line(&interviewer, &cancel_token, &steering_hub, &line).await; ++ let mut first_tx = Some(first_tx); ++ let mut fatal_tx = Some(fatal_tx); ++ let mut backoff = WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF; ++ let mut applied_ids = AppliedWorkerControlDeliveryIds::default(); ++ ++ while !done.is_cancelled() { ++ let request = match build_worker_control_stream_request( ++ &target, ++ &run_id, ++ &worker_token, ++ applied_ids.last_applied_id(), ++ ) { ++ Ok(request) => request, ++ Err(err) => { ++ report_fatal_control_loss( ++ &interviewer, ++ &cancel_token, ++ &mut first_tx, ++ &mut fatal_tx, ++ format!("failed to build worker control stream request: {err:#}"), ++ ) ++ .await; ++ return; ++ } ++ }; ++ ++ match connect_worker_control_stream(request).await { ++ Ok(mut socket) => { ++ if let Some(first_tx) = first_tx.take() { ++ let _ = first_tx.send(Ok(())); ++ } ++ backoff = WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF; ++ match handle_worker_control_socket( ++ &mut socket, ++ &interviewer, ++ &cancel_token, ++ &steering_hub, ++ &run_control, ++ &mut applied_ids, ++ &done, ++ ) ++ .await ++ { ++ Ok(()) => {} ++ Err(WorkerControlConnectError::InvalidCursor) => { ++ report_fatal_control_loss( ++ &interviewer, ++ &cancel_token, ++ &mut first_tx, ++ &mut fatal_tx, ++ "worker control stream replay cursor is invalid".to_string(), ++ ) ++ .await; ++ return; ++ } ++ Err(WorkerControlConnectError::Other(err)) => { ++ tracing::debug!(error = %err, "Worker control stream disconnected"); ++ } ++ } + } +- WorkerControlStreamEvent::Eof => { +- interviewer.interrupt_all().await; ++ Err(WorkerControlConnectError::InvalidCursor) => { ++ report_fatal_control_loss( ++ &interviewer, ++ &cancel_token, ++ &mut first_tx, ++ &mut fatal_tx, ++ "worker control stream replay cursor is invalid".to_string(), ++ ) ++ .await; + return; + } ++ Err(WorkerControlConnectError::Other(err)) => { ++ tracing::debug!(error = %err, "Worker control stream connection failed"); ++ } + } ++ ++ sleep_or_done(&done, backoff).await; ++ backoff = next_worker_control_reconnect_backoff(backoff); + } ++} + ++async fn report_fatal_control_loss( ++ interviewer: &ControlInterviewer, ++ cancel_token: &CancellationToken, ++ first_tx: &mut Option>>, ++ fatal_tx: &mut Option>, ++ detail: String, ++) { ++ let message = format!("worker control channel lost: {detail}"); + interviewer.interrupt_all().await; ++ if let Some(first_tx) = first_tx.take() { ++ let _ = first_tx.send(Err(anyhow!(message.clone()))); ++ } ++ if let Some(fatal_tx) = fatal_tx.take() { ++ let _ = fatal_tx.send(anyhow!(message)); ++ } ++ cancel_token.cancel(); ++} ++ ++async fn sleep_or_done(done: &CancellationToken, delay: Duration) { ++ tokio::select! { ++ () = done.cancelled() => {} ++ () = time::sleep(delay) => {} ++ } ++} ++ ++fn next_worker_control_reconnect_backoff(current: Duration) -> Duration { ++ current ++ .saturating_mul(2) ++ .min(WORKER_CONTROL_RECONNECT_MAX_BACKOFF) ++} ++ ++fn build_worker_control_stream_request( ++ target: &ServerTarget, ++ run_id: &RunId, ++ worker_token: &str, ++ after: Option<&str>, ++) -> Result { ++ let (url, unix_socket_path) = match target { ++ ServerTarget::HttpUrl(_) => { ++ let base = target ++ .as_http_url() ++ .context("HTTP server target missing URL")?; ++ let websocket_base = if let Some(rest) = base.strip_prefix("http://") { ++ format!("ws://{rest}") ++ } else if let Some(rest) = base.strip_prefix("https://") { ++ format!("wss://{rest}") ++ } else { ++ anyhow::bail!("unsupported server URL scheme"); ++ }; ++ ( ++ worker_control_stream_url(&websocket_base, run_id, after), ++ None, ++ ) ++ } ++ ServerTarget::UnixSocket(path) => { ++ let url = worker_control_stream_url("ws://fabro", run_id, after); ++ (url, Some(path.as_path().to_path_buf())) ++ } ++ }; ++ let mut request = url ++ .as_str() ++ .into_client_request() ++ .context("failed to build worker control stream request")?; ++ request.headers_mut().insert( ++ header::AUTHORIZATION, ++ HeaderValue::from_str(&format!("Bearer {worker_token}")) ++ .context("failed to build worker control stream authorization header")?, ++ ); ++ Ok(WorkerControlStreamConnectRequest { ++ request, ++ unix_socket_path, ++ }) + } + +-async fn apply_worker_control_line( ++fn worker_control_stream_url(base: &str, run_id: &RunId, after: Option<&str>) -> String { ++ let mut url = format!("{base}/api/v1/runs/{run_id}/worker/control-stream"); ++ if let Some(after) = after { ++ url.push_str("?after="); ++ url.push_str(after); ++ } ++ url ++} ++ ++async fn connect_worker_control_stream( ++ request: WorkerControlStreamConnectRequest, ++) -> Result { ++ if let Some(path) = request.unix_socket_path.as_ref() { ++ #[cfg(unix)] ++ { ++ let ws_request = request.request_for_tungstenite(); ++ let stream = UnixStream::connect(path) ++ .await ++ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?; ++ let (socket, _) = tokio_tungstenite::client_async(ws_request, stream) ++ .await ++ .map_err(classify_tungstenite_error)?; ++ Ok(WorkerControlSocket::Unix(Box::new(socket))) ++ } ++ #[cfg(not(unix))] ++ { ++ let _ = path; ++ Err(WorkerControlConnectError::Other(anyhow!( ++ "Unix-socket worker control stream is not supported on this platform" ++ ))) ++ } ++ } else { ++ let (socket, _) = connect_async(request.request) ++ .await ++ .map_err(classify_tungstenite_error)?; ++ Ok(WorkerControlSocket::Tcp(Box::new(socket))) ++ } ++} ++ ++fn classify_tungstenite_error(error: tungstenite::Error) -> WorkerControlConnectError { ++ if let tungstenite::Error::Http(response) = &error { ++ if response.status().as_u16() == 410 { ++ return WorkerControlConnectError::InvalidCursor; ++ } ++ } ++ WorkerControlConnectError::Other(anyhow::Error::new(error)) ++} ++ ++async fn handle_worker_control_socket( ++ socket: &mut WorkerControlSocket, + interviewer: &ControlInterviewer, + cancel_token: &CancellationToken, + steering_hub: &fabro_workflow::SteeringHub, +- line: &str, +-) { +- if line.trim().is_empty() { +- return; ++ run_control: &RunControlState, ++ applied_ids: &mut AppliedWorkerControlDeliveryIds, ++ done: &CancellationToken, ++) -> Result<(), WorkerControlConnectError> { ++ let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL); ++ ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); ++ let mut last_liveness = Instant::now(); ++ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT); ++ tokio::pin!(liveness_timeout); ++ ++ loop { ++ liveness_timeout ++ .as_mut() ++ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT); ++ ++ tokio::select! { ++ () = done.cancelled() => return Ok(()), ++ _ = ping_interval.tick() => { ++ socket ++ .send(WebSocketMessage::Ping(Vec::new().into())) ++ .await ++ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?; ++ } ++ () = &mut liveness_timeout => { ++ let _ = socket ++ .send(WebSocketMessage::Close(Some(protocol::CloseFrame { ++ code: CloseCode::Away, ++ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(), ++ }))) ++ .await; ++ return Err(WorkerControlConnectError::Other(anyhow!( ++ "worker control WebSocket liveness timed out" ++ ))); ++ } ++ message = socket.next() => { ++ let Some(message) = message else { ++ return Ok(()); ++ }; ++ match message { ++ Ok(WebSocketMessage::Text(text)) => { ++ last_liveness = Instant::now(); ++ let frame = serde_json::from_str::(text.as_str()) ++ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?; ++ apply_worker_control_delivery_frame( ++ interviewer, ++ cancel_token, ++ steering_hub, ++ run_control, ++ applied_ids, ++ frame, ++ ) ++ .await; ++ } ++ Ok(WebSocketMessage::Ping(payload)) => { ++ last_liveness = Instant::now(); ++ socket ++ .send(WebSocketMessage::Pong(payload)) ++ .await ++ .map_err(|err| WorkerControlConnectError::Other(anyhow::Error::new(err)))?; ++ } ++ Ok(WebSocketMessage::Pong(_) | WebSocketMessage::Binary(_)) => { ++ last_liveness = Instant::now(); ++ } ++ Ok(WebSocketMessage::Close(frame)) => { ++ if frame.as_ref().is_some_and(|frame| { ++ frame.reason.as_str() == WORKER_CONTROL_INVALID_CURSOR_REASON ++ }) { ++ return Err(WorkerControlConnectError::InvalidCursor); ++ } ++ return Ok(()); ++ } ++ Ok(WebSocketMessage::Frame(_)) => {} ++ Err(err) => { ++ return Err(WorkerControlConnectError::Other(anyhow::Error::new(err))); ++ } ++ } ++ } ++ } + } ++} + +- let Ok(message) = serde_json::from_str::(line) else { +- return; +- }; ++async fn apply_worker_control_delivery_frame( ++ interviewer: &ControlInterviewer, ++ cancel_token: &CancellationToken, ++ steering_hub: &fabro_workflow::SteeringHub, ++ run_control: &RunControlState, ++ applied_ids: &mut AppliedWorkerControlDeliveryIds, ++ frame: WorkerControlDeliveryFrame, ++) -> bool { ++ // Duplicate ids cannot reach us under normal operation: the server replays ++ // strictly after the last applied id. Guard against a server-side bug or ++ // reconnect race by ignoring recently-applied delivery ids. ++ if applied_ids.contains(&frame.id) { ++ return false; ++ } ++ let frame_id = frame.id; ++ apply_worker_control_message( ++ interviewer, ++ cancel_token, ++ steering_hub, ++ run_control, ++ frame.envelope, ++ ) ++ .await; ++ applied_ids.record(frame_id); ++ true ++} + ++async fn apply_worker_control_message( ++ interviewer: &ControlInterviewer, ++ cancel_token: &CancellationToken, ++ steering_hub: &fabro_workflow::SteeringHub, ++ run_control: &RunControlState, ++ message: WorkerControlEnvelope, ++) { + match message.message { + WorkerControlMessage::InterviewAnswer { qid, answer, actor } => { + let _ = interviewer +@@ -354,6 +802,12 @@ async fn apply_worker_control_line( + cancel_token.cancel(); + interviewer.interrupt_all().await; + } ++ WorkerControlMessage::RunPause => { ++ run_control.request_pause(); ++ } ++ WorkerControlMessage::RunUnpause => { ++ run_control.request_unpause(); ++ } + WorkerControlMessage::Steer { text, actor } => { + steering_hub.deliver_steer(text, Some(actor)); + } +@@ -485,7 +939,7 @@ impl HttpRunStore { + Err(err) => last_error = Some(err), + } + if let Some(delay) = RUN_STORE_RETRY_DELAYS.get(attempt) { +- sleep(*delay).await; ++ time::sleep(*delay).await; + } + } + Err(last_error +@@ -757,10 +1211,14 @@ fn install_signal_handlers( + )] + mod tests { + use std::sync::Arc; ++ use std::time::Duration; + + use chrono::Utc; ++ use fabro_client::ServerTarget; + use fabro_config::Storage; +- use fabro_interview::{AnswerValue, ControlInterviewer, Interviewer, Question}; ++ use fabro_interview::{ ++ AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlEnvelope, ++ }; + use fabro_types::run_event::{ + InterviewCompletedProps, InterviewStartedProps, RunCompletedProps, RunControlEffectProps, + RunFailedProps, RunStatusTransitionProps, +@@ -771,12 +1229,17 @@ mod tests { + }; + use fabro_vault::{SecretType, Vault}; + use fabro_workflow::event::RunEventSink; ++ use fabro_workflow::run_control::RunControlState; ++ use tokio::time; ++ use tokio_tungstenite::tungstenite::protocol::{Message as TestWebSocketMessage, Role}; + use tokio_util::sync::CancellationToken; + + use super::{ +- WorkerControlStreamEvent, WorkerTitlePhase, apply_worker_control_line, +- handle_worker_control_stream_events, initial_worker_title_phase, load_worker_vault, +- read_worker_control_stream_blocking, stamp_system_worker, worker_title, ++ AppliedWorkerControlDeliveryIds, WorkerControlConnectError, WorkerControlSocket, ++ WorkerTitlePhase, apply_worker_control_delivery_frame, apply_worker_control_message, ++ build_worker_control_stream_request, connect_worker_control_stream, ++ handle_worker_control_socket, initial_worker_title_phase, load_worker_vault, ++ next_worker_control_reconnect_backoff, stamp_system_worker, worker_title, + worker_title_phase_for_event, + }; + use crate::args::RunWorkerMode; +@@ -1018,20 +1481,28 @@ mod tests { + } + + #[tokio::test] +- async fn worker_control_line_routes_answer_by_question_id() { ++ async fn worker_control_routes_answer_by_question_id() { + let interviewer = Arc::new(ControlInterviewer::new()); + let cancel_token = CancellationToken::new(); ++ let run_control = RunControlState::new(); + let mut question = Question::new("Approve?", QuestionType::YesNo); + question.id = "q-1".to_string(); + let ask_interviewer = Arc::clone(&interviewer); + let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await }); + + let hub = test_steering_hub(); +- apply_worker_control_line( ++ apply_worker_control_message( + &interviewer, + &cancel_token, + &hub, +- r#"{"v":1,"type":"interview.answer","qid":"q-1","answer":{"kind":"yes"},"actor":{"kind":"system","system_kind":"engine"}}"#, ++ &run_control, ++ WorkerControlEnvelope::interview_answer( ++ "q-1", ++ fabro_interview::AnswerSubmission::system( ++ fabro_interview::Answer::yes(), ++ fabro_types::SystemActorKind::Engine, ++ ), ++ ), + ) + .await; + +@@ -1041,9 +1512,10 @@ mod tests { + } + + #[tokio::test] +- async fn worker_control_line_cancel_sets_cancel_token_and_interrupts_pending_interviews() { ++ async fn worker_control_cancel_sets_cancel_token_and_interrupts_pending_interviews() { + let interviewer = Arc::new(ControlInterviewer::new()); + let cancel_token = CancellationToken::new(); ++ let run_control = RunControlState::new(); + let mut question = Question::new("Approve?", QuestionType::YesNo); + question.id = "q-1".to_string(); + let ask_interviewer = Arc::clone(&interviewer); +@@ -1051,11 +1523,12 @@ mod tests { + tokio::task::yield_now().await; + + let hub = test_steering_hub(); +- apply_worker_control_line( ++ apply_worker_control_message( + &interviewer, + &cancel_token, + &hub, +- r#"{"v":1,"type":"run.cancel"}"#, ++ &run_control, ++ WorkerControlEnvelope::cancel_run(), + ) + .await; + +@@ -1065,57 +1538,206 @@ mod tests { + } + + #[tokio::test] +- async fn blocking_worker_control_stream_emits_lines_and_eof() { +- let (event_tx, mut event_rx) = tokio::sync::mpsc::unbounded_channel(); ++ async fn worker_control_pause_and_unpause_route_to_run_control() { ++ let interviewer = Arc::new(ControlInterviewer::new()); ++ let cancel_token = CancellationToken::new(); ++ let run_control = RunControlState::new(); ++ let hub = test_steering_hub(); + +- read_worker_control_stream_blocking( +- std::io::Cursor::new( +- b"{\"v\":1,\"type\":\"run.cancel\"}\n{\"v\":1,\"type\":\"interview.answer\",\"qid\":\"q-1\",\"answer\":{\"kind\":\"yes\"},\"actor\":{\"kind\":\"system\",\"system_kind\":\"engine\"}}\n", +- ), +- &event_tx, ++ apply_worker_control_message( ++ &interviewer, ++ &cancel_token, ++ &hub, ++ &run_control, ++ WorkerControlEnvelope::pause_run(), ++ ) ++ .await; ++ assert!(run_control.pause_requested()); ++ ++ apply_worker_control_message( ++ &interviewer, ++ &cancel_token, ++ &hub, ++ &run_control, ++ WorkerControlEnvelope::unpause_run(), ++ ) ++ .await; ++ assert!(!run_control.pause_requested()); ++ } ++ ++ #[tokio::test] ++ async fn duplicate_delivery_ids_are_not_applied_twice() { ++ let interviewer = Arc::new(ControlInterviewer::new()); ++ let cancel_token = CancellationToken::new(); ++ let run_control = RunControlState::new(); ++ let hub = test_steering_hub(); ++ let mut applied_ids = AppliedWorkerControlDeliveryIds::default(); ++ let frame = fabro_interview::WorkerControlDeliveryFrame { ++ id: "local:1".to_string(), ++ envelope: WorkerControlEnvelope::pause_run(), ++ }; ++ ++ assert!( ++ apply_worker_control_delivery_frame( ++ &interviewer, ++ &cancel_token, ++ &hub, ++ &run_control, ++ &mut applied_ids, ++ frame.clone(), ++ ) ++ .await ++ ); ++ assert!( ++ !apply_worker_control_delivery_frame( ++ &interviewer, ++ &cancel_token, ++ &hub, ++ &run_control, ++ &mut applied_ids, ++ frame, ++ ) ++ .await ++ ); ++ ++ assert_eq!(applied_ids.last_applied_id(), Some("local:1")); ++ } ++ ++ #[test] ++ fn worker_control_request_construction_for_http_targets() { ++ let run_id = fixtures::RUN_1; ++ let request = build_worker_control_stream_request( ++ &ServerTarget::http_url("http://example.com:3000").unwrap(), ++ &run_id, ++ "worker-token", ++ None, ++ ) ++ .unwrap(); ++ assert_eq!( ++ request.uri(), ++ format!("ws://example.com:3000/api/v1/runs/{run_id}/worker/control-stream") ++ ); ++ assert_eq!(request.authorization(), Some("Bearer worker-token")); ++ ++ let reconnect = build_worker_control_stream_request( ++ &ServerTarget::http_url("https://example.com").unwrap(), ++ &run_id, ++ "worker-token", ++ Some("local:42"), ++ ) ++ .unwrap(); ++ assert_eq!( ++ reconnect.uri(), ++ format!("wss://example.com/api/v1/runs/{run_id}/worker/control-stream?after=local:42") + ); ++ } ++ ++ #[cfg(unix)] ++ #[test] ++ fn worker_control_request_construction_for_unix_socket_targets() { ++ let run_id = fixtures::RUN_1; ++ let request = build_worker_control_stream_request( ++ &ServerTarget::unix_socket_path("/tmp/fabro.sock").unwrap(), ++ &run_id, ++ "worker-token", ++ Some("local:42"), ++ ) ++ .unwrap(); + + assert_eq!( +- event_rx.try_recv(), +- Ok(WorkerControlStreamEvent::Line( +- r#"{"v":1,"type":"run.cancel"}"#.to_string() +- )) ++ request.uri(), ++ format!("ws://fabro/api/v1/runs/{run_id}/worker/control-stream?after=local:42") + ); + assert_eq!( +- event_rx.try_recv(), +- Ok(WorkerControlStreamEvent::Line( +- r#"{"v":1,"type":"interview.answer","qid":"q-1","answer":{"kind":"yes"},"actor":{"kind":"system","system_kind":"engine"}}"# +- .to_string() +- )) ++ request.unix_socket_path.as_deref(), ++ Some(std::path::Path::new("/tmp/fabro.sock")) + ); +- assert_eq!(event_rx.try_recv(), Ok(WorkerControlStreamEvent::Eof)); + } + +- #[tokio::test] +- async fn worker_control_event_loop_eof_interrupts_pending_interviews() { ++ #[test] ++ fn worker_control_reconnect_backoff_is_bounded() { ++ assert_eq!( ++ next_worker_control_reconnect_backoff(Duration::from_millis(100)), ++ Duration::from_millis(200) ++ ); ++ assert_eq!( ++ next_worker_control_reconnect_backoff(Duration::from_secs(4)), ++ Duration::from_secs(5) ++ ); ++ assert_eq!( ++ next_worker_control_reconnect_backoff(Duration::from_secs(5)), ++ Duration::from_secs(5) ++ ); ++ } ++ ++ #[tokio::test(start_paused = true)] ++ async fn worker_control_socket_times_out_without_liveness() { ++ let (worker_io, _server_io) = tokio::io::duplex(1024); ++ let worker_ws = ++ tokio_tungstenite::WebSocketStream::from_raw_socket(worker_io, Role::Client, None) ++ .await; ++ let mut socket = WorkerControlSocket::Test(Box::new(worker_ws)); + let interviewer = Arc::new(ControlInterviewer::new()); + let cancel_token = CancellationToken::new(); +- let mut question = Question::new("Approve?", QuestionType::YesNo); +- question.id = "q-1".to_string(); +- let ask_interviewer = Arc::clone(&interviewer); +- let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await }); +- let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel(); ++ let hub = test_steering_hub(); ++ let run_control = RunControlState::new(); ++ let mut applied_ids = AppliedWorkerControlDeliveryIds::default(); ++ let done = CancellationToken::new(); ++ ++ let task = tokio::spawn(async move { ++ handle_worker_control_socket( ++ &mut socket, ++ &interviewer, ++ &cancel_token, ++ &hub, ++ &run_control, ++ &mut applied_ids, ++ &done, ++ ) ++ .await ++ }); + +- event_tx.send(WorkerControlStreamEvent::Eof).unwrap(); +- drop(event_tx); ++ tokio::task::yield_now().await; ++ time::advance(Duration::from_secs(46)).await; ++ let result = task.await.unwrap(); ++ assert!(matches!(result, Err(WorkerControlConnectError::Other(_)))); ++ } + +- let hub = test_steering_hub(); +- handle_worker_control_stream_events( +- Arc::clone(&interviewer), +- cancel_token.clone(), +- hub, +- event_rx, ++ #[cfg(unix)] ++ #[tokio::test] ++ async fn worker_control_unix_socket_handshake_completes() { ++ let temp = tempfile::tempdir().unwrap(); ++ let socket_path = temp.path().join("fabro.sock"); ++ let listener = tokio::net::UnixListener::bind(&socket_path).unwrap(); ++ let server = tokio::spawn(async move { ++ let (stream, _) = listener.accept().await.unwrap(); ++ let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); ++ while let Some(message) = futures::StreamExt::next(&mut socket).await { ++ match message.unwrap() { ++ TestWebSocketMessage::Close(_) => break, ++ TestWebSocketMessage::Ping(payload) => { ++ futures::SinkExt::send(&mut socket, TestWebSocketMessage::Pong(payload)) ++ .await ++ .unwrap(); ++ } ++ _ => {} ++ } ++ } ++ }); ++ let request = build_worker_control_stream_request( ++ &ServerTarget::unix_socket_path(&socket_path).unwrap(), ++ &fixtures::RUN_1, ++ "worker-token", ++ None, + ) +- .await; ++ .unwrap(); + +- let answer = answer_task.await.unwrap().answer; +- assert_eq!(answer.value, AnswerValue::Interrupted); +- assert!(!cancel_token.is_cancelled()); ++ let mut socket = connect_worker_control_stream(request).await.unwrap(); ++ socket ++ .send(TestWebSocketMessage::Close(None)) ++ .await ++ .unwrap(); ++ server.await.unwrap(); + } + + #[tokio::test] +diff --git a/lib/crates/fabro-cli/tests/it/cmd/runner.rs b/lib/crates/fabro-cli/tests/it/cmd/runner.rs +index d7d87fb93..d5b307d35 100644 +--- a/lib/crates/fabro-cli/tests/it/cmd/runner.rs ++++ b/lib/crates/fabro-cli/tests/it/cmd/runner.rs +@@ -794,6 +794,66 @@ fn detached_run_answers_pending_question_without_interview_scratch_files() { + ))); + } + ++#[test] ++fn detached_run_cancel_reaches_worker_over_control_websocket() { ++ let context = auth_context(); ++ let run_id = unique_run_id(); ++ let workflow_path = context.temp_dir.join("cancel-over-control-websocket.fabro"); ++ let _gate = write_gated_workflow( ++ &workflow_path, ++ "cancel_over_control_websocket", ++ "Wait for cancellation", ++ ); ++ ++ let output = context ++ .command() ++ .args([ ++ "run", ++ "--detach", ++ "--run-id", ++ run_id.as_str(), ++ "--environment", ++ "local", ++ workflow_path.to_str().unwrap(), ++ ]) ++ .timeout(SHARED_DAEMON_TIMEOUT) ++ .output() ++ .expect("detached run should execute"); ++ assert!( ++ output.status.success(), ++ "detached run failed:\nstdout:\n{}\nstderr:\n{}", ++ String::from_utf8_lossy(&output.stdout), ++ String::from_utf8_lossy(&output.stderr) ++ ); ++ ++ let run_dir = context.find_run_dir(&run_id); ++ wait_for_event_names(&run_dir, &["run.running"]); ++ tokio::runtime::Runtime::new() ++ .expect("test runtime should build") ++ .block_on(async { ++ let (client, base_url) = ++ server_endpoint(&context.storage_dir).expect("server endpoint should exist"); ++ let response = client ++ .post(format!("{base_url}/api/v1/runs/{run_id}/cancel")) ++ .send() ++ .await ++ .expect("cancel request should succeed"); ++ assert_reqwest_status( ++ response, ++ fabro_http::StatusCode::OK, ++ format!("POST /api/v1/runs/{run_id}/cancel"), ++ ) ++ .await; ++ }); ++ ++ wait_for_status(&run_dir, &["failed"]); ++ let events = stored_worker_events(&run_dir); ++ assert!(events.iter().any(|event| matches!( ++ &event.body, ++ EventBody::RunFailed(props) if props.failure.reason == FailureReason::Cancelled ++ ))); ++} ++ + #[cfg(unix)] + #[test] + fn worker_exits_after_sigterm_cancel_even_when_stdin_stays_open() { +diff --git a/lib/crates/fabro-interview/src/control_protocol.rs b/lib/crates/fabro-interview/src/control_protocol.rs +index cdc820216..f76b61121 100644 +--- a/lib/crates/fabro-interview/src/control_protocol.rs ++++ b/lib/crates/fabro-interview/src/control_protocol.rs +@@ -1,3 +1,5 @@ ++use std::time::Duration; ++ + use fabro_types::{PairId, PairMessageId, PairTarget, Principal, RunId}; + use serde::{Deserialize, Serialize}; + +@@ -5,6 +7,25 @@ use crate::{Answer, AnswerSubmission, AnswerValue}; + + pub const WORKER_CONTROL_PROTOCOL_VERSION: u8 = 1; + ++/// Interval between worker-control WebSocket ping frames. ++/// ++/// Server and worker both initiate pings at this cadence; either side that ++/// fails to observe inbound traffic for [`WORKER_CONTROL_WS_LIVENESS_TIMEOUT`] ++/// closes the WebSocket. ++pub const WORKER_CONTROL_WS_PING_INTERVAL: Duration = Duration::from_secs(15); ++ ++/// Maximum quiet time allowed on a worker-control WebSocket before either side ++/// declares the connection dead. ++pub const WORKER_CONTROL_WS_LIVENESS_TIMEOUT: Duration = Duration::from_secs(45); ++ ++/// WebSocket close-frame reason used when the server can no longer prove ++/// replay correctness for the requested cursor. Workers must treat this as ++/// fatal control-channel loss. ++pub const WORKER_CONTROL_INVALID_CURSOR_REASON: &str = "invalid_cursor"; ++ ++/// WebSocket close-frame reason used when the ping/pong watchdog fires. ++pub const WORKER_CONTROL_PONG_TIMEOUT_REASON: &str = "pong_timeout"; ++ + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + pub struct WorkerControlEnvelope { + pub v: u8, +@@ -33,6 +54,22 @@ impl WorkerControlEnvelope { + } + } + ++ #[must_use] ++ pub fn pause_run() -> Self { ++ Self { ++ v: WORKER_CONTROL_PROTOCOL_VERSION, ++ message: WorkerControlMessage::RunPause, ++ } ++ } ++ ++ #[must_use] ++ pub fn unpause_run() -> Self { ++ Self { ++ v: WORKER_CONTROL_PROTOCOL_VERSION, ++ message: WorkerControlMessage::RunUnpause, ++ } ++ } ++ + #[must_use] + pub fn steer(text: impl Into, actor: Principal) -> Self { + Self { +@@ -121,6 +158,10 @@ pub enum WorkerControlMessage { + }, + #[serde(rename = "run.cancel")] + RunCancel, ++ #[serde(rename = "run.pause")] ++ RunPause, ++ #[serde(rename = "run.unpause")] ++ RunUnpause, + #[serde(rename = "run.steer")] + Steer { text: String, actor: Principal }, + #[serde(rename = "run.interrupt")] +@@ -147,6 +188,12 @@ pub enum WorkerControlMessage { + PairEnd { pair_id: PairId, actor: Principal }, + } + ++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] ++pub struct WorkerControlDeliveryFrame { ++ pub id: String, ++ pub envelope: WorkerControlEnvelope, ++} ++ + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + #[serde(tag = "kind", rename_all = "snake_case")] + pub enum WorkerControlAnswer { +@@ -229,6 +276,26 @@ mod tests { + assert_eq!(parsed, envelope); + } + ++ #[test] ++ fn pause_run_round_trips_through_json() { ++ let envelope = WorkerControlEnvelope::pause_run(); ++ let json = serde_json::to_string(&envelope).unwrap(); ++ assert_eq!(json, r#"{"v":1,"type":"run.pause"}"#); ++ ++ let parsed: WorkerControlEnvelope = serde_json::from_str(&json).unwrap(); ++ assert_eq!(parsed, envelope); ++ } ++ ++ #[test] ++ fn unpause_run_round_trips_through_json() { ++ let envelope = WorkerControlEnvelope::unpause_run(); ++ let json = serde_json::to_string(&envelope).unwrap(); ++ assert_eq!(json, r#"{"v":1,"type":"run.unpause"}"#); ++ ++ let parsed: WorkerControlEnvelope = serde_json::from_str(&json).unwrap(); ++ assert_eq!(parsed, envelope); ++ } ++ + #[test] + fn steer_append_round_trips_through_json() { + let envelope = WorkerControlEnvelope::steer("try again", Principal::System { +@@ -326,4 +393,20 @@ mod tests { + serde_json::from_str(&serde_json::to_string(&end).unwrap()).unwrap(); + assert_eq!(parsed, end); + } ++ ++ #[test] ++ fn delivery_frame_round_trips_through_json() { ++ let frame = WorkerControlDeliveryFrame { ++ id: "local:42".to_string(), ++ envelope: WorkerControlEnvelope::cancel_run(), ++ }; ++ let json = serde_json::to_string(&frame).unwrap(); ++ assert_eq!( ++ json, ++ r#"{"id":"local:42","envelope":{"v":1,"type":"run.cancel"}}"# ++ ); ++ ++ let parsed: WorkerControlDeliveryFrame = serde_json::from_str(&json).unwrap(); ++ assert_eq!(parsed, frame); ++ } + } +diff --git a/lib/crates/fabro-interview/src/lib.rs b/lib/crates/fabro-interview/src/lib.rs +index 4c050c4f7..6d8c414a1 100644 +--- a/lib/crates/fabro-interview/src/lib.rs ++++ b/lib/crates/fabro-interview/src/lib.rs +@@ -222,8 +222,10 @@ pub use callback::CallbackInterviewer; + pub use console::ConsoleInterviewer; + pub use control::{ControlInterviewer, SubmitError}; + pub use control_protocol::{ +- WORKER_CONTROL_PROTOCOL_VERSION, WorkerControlAnswer, WorkerControlEnvelope, +- WorkerControlMessage, ++ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON, ++ WORKER_CONTROL_PROTOCOL_VERSION, WORKER_CONTROL_WS_LIVENESS_TIMEOUT, ++ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlAnswer, WorkerControlDeliveryFrame, ++ WorkerControlEnvelope, WorkerControlMessage, + }; + pub use queue::QueueInterviewer; + pub use recording::RecordingInterviewer; +diff --git a/lib/crates/fabro-server/Cargo.toml b/lib/crates/fabro-server/Cargo.toml +index 2b9568ce1..879cea7ce 100644 +--- a/lib/crates/fabro-server/Cargo.toml ++++ b/lib/crates/fabro-server/Cargo.toml +@@ -112,6 +112,7 @@ httpmock = "0.8" + serde_yaml = "0.9" + tracing-subscriber.workspace = true + tokio-util.workspace = true ++tokio-tungstenite.workspace = true + fabro-macros = { path = "../fabro-macros" } + fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] } + fabro-test = { workspace = true } +diff --git a/lib/crates/fabro-server/src/lib.rs b/lib/crates/fabro-server/src/lib.rs +index 0e591e26d..d2a449732 100644 +--- a/lib/crates/fabro-server/src/lib.rs ++++ b/lib/crates/fabro-server/src/lib.rs +@@ -49,6 +49,7 @@ pub mod static_files; + #[cfg(any(test, feature = "test-support"))] + pub mod test_support; + pub mod web_auth; ++mod worker_control; + mod worker_token; + + pub use error::{ApiError, Error, Result}; +diff --git a/lib/crates/fabro-server/src/principal_middleware.rs b/lib/crates/fabro-server/src/principal_middleware.rs +index 9e28be9e4..f9e8c385c 100644 +--- a/lib/crates/fabro-server/src/principal_middleware.rs ++++ b/lib/crates/fabro-server/src/principal_middleware.rs +@@ -59,6 +59,7 @@ pub(crate) struct RequiredUser(pub(crate) UserPrincipal); + pub(crate) struct RequiredRunManagementActor(pub(crate) Principal); + pub(crate) struct RequiredRunToolActor(pub(crate) Principal); + pub(crate) struct RequireRunScoped(pub(crate) RunId); ++pub(crate) struct RequireWorkerRunScoped(pub(crate) RunId); + pub(crate) struct RequireRunManagementTarget(pub(crate) RunId, pub(crate) Principal); + pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId); + pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String); +@@ -259,6 +260,23 @@ impl FromRequestParts> for RequireRunScoped { + } + } + ++impl FromRequestParts> for RequireWorkerRunScoped { ++ type Rejection = Response; ++ ++ async fn from_request_parts( ++ parts: &mut Parts, ++ state: &Arc, ++ ) -> Result { ++ let Path(id): Path = Path::from_request_parts(parts, state) ++ .await ++ .map_err(IntoResponse::into_response)?; ++ let run_id = parse_run_id_path(&id)?; ++ require_worker_for_run(&auth_slot_from_parts(parts), &run_id) ++ .map_err(IntoResponse::into_response)?; ++ Ok(Self(run_id)) ++ } ++} ++ + impl FromRequestParts> for RequireRunManagementTarget { + type Rejection = Response; + +@@ -432,6 +450,15 @@ fn require_worker_or_user_for_run( + } + } + ++fn require_worker_for_run(slot: &AuthContextSlot, route_run_id: &RunId) -> Result<(), ApiError> { ++ let context = slot.0.lock().expect("auth context lock poisoned"); ++ match &context.principal { ++ Principal::Worker { run_id } if run_id == route_run_id => Ok(()), ++ Principal::Worker { .. } | Principal::User(_) => Err(ApiError::forbidden()), ++ _ => Err(auth_rejection(context.auth_status, context.auth_error_code)), ++ } ++} ++ + fn require_run_management_target( + slot: &AuthContextSlot, + route_run_id: &RunId, +diff --git a/lib/crates/fabro-server/src/serve.rs b/lib/crates/fabro-server/src/serve.rs +index b2eb6d91b..422906eeb 100644 +--- a/lib/crates/fabro-server/src/serve.rs ++++ b/lib/crates/fabro-server/src/serve.rs +@@ -814,6 +814,8 @@ where + http_client: None, + sandbox_provider_registry: None, + shutdown: shutdown.clone(), ++ #[cfg(test)] ++ worker_control_bus: None, + #[cfg(any(test, feature = "test-support"))] + automation_materializer_override: None, + })?; +diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs +index 4f0ff4dd5..c43ef1138 100644 +--- a/lib/crates/fabro-server/src/server.rs ++++ b/lib/crates/fabro-server/src/server.rs +@@ -124,7 +124,7 @@ use sha2::{Digest, Sha256}; + use tempfile::NamedTempFile; + use tokio::fs; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +-use tokio::process::{ChildStderr, ChildStdin, Command}; ++use tokio::process::{ChildStderr, Command}; + use tokio::sync::broadcast::error::RecvError; + use tokio::sync::{ + Mutex as AsyncMutex, Notify, OwnedMutexGuard, RwLock as AsyncRwLock, Semaphore, broadcast, +@@ -153,14 +153,15 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware}; + use crate::jwt_auth::{self, AuthMode}; + use crate::principal_middleware::{ + AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunManagementTarget, +- RequireRunScoped, RequireRunStageScoped, RequireStageArtifact, RequiredUser, +- principal_middleware, ++ RequireRunScoped, RequireRunStageScoped, RequireStageArtifact, RequireWorkerRunScoped, ++ RequiredUser, principal_middleware, + }; + use crate::request_id::{self, RequestId}; + use crate::run_files::{FilesInFlight, new_files_in_flight}; + use crate::server_secrets::{LlmClientResult, ServerSecrets}; + use crate::spawn_env::{apply_render_graph_env, apply_worker_env}; + use crate::startup::load_startup_vault; ++use crate::worker_control::{LocalWorkerControlBus, WorkerControlBus, WorkerControlBusError}; + use crate::worker_token::{WorkerScopeSet, WorkerTokenKeys, issue_worker_token_with_scopes}; + use crate::{ + canonical_host, demo, diagnostics, run_manifest, security_headers, static_files, web_auth, +@@ -272,7 +273,6 @@ enum ExecutionResult { + + const WORKER_CANCEL_GRACE: Duration = Duration::from_secs(5); + const TERMINAL_DELETE_WORKER_GRACE: Duration = Duration::from_millis(50); +-const WORKER_CONTROL_QUEUE_CAPACITY: usize = 8; + const WORKER_CONTROL_ENQUEUE_TIMEOUT: Duration = Duration::from_secs(1); + /// Per-model billing totals. + #[derive(Default)] +@@ -294,8 +294,9 @@ pub(crate) type RegistryFactoryOverride = + + #[derive(Clone)] + enum RunAnswerTransport { +- Subprocess { +- control_tx: mpsc::Sender, ++ Worker { ++ run_id: RunId, ++ bus: Arc, + }, + InProcess { + interviewer: Arc, +@@ -317,18 +318,46 @@ enum PairTransportError { + } + + impl RunAnswerTransport { ++ async fn publish_worker_control( ++ run_id: RunId, ++ bus: &Arc, ++ message: WorkerControlEnvelope, ++ ) -> Result<(), WorkerControlBusError> { ++ timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, bus.publish(run_id, message)) ++ .await ++ .map_err(|_| WorkerControlBusError::PublishTimeout)? ++ .map(|_| ()) ++ } ++ ++ fn answer_error_from_bus(error: &WorkerControlBusError) -> AnswerTransportError { ++ match error { ++ WorkerControlBusError::PublishTimeout => AnswerTransportError::Timeout, ++ WorkerControlBusError::Closed ++ | WorkerControlBusError::Unavailable ++ | WorkerControlBusError::InvalidCursor { .. } => AnswerTransportError::Closed, ++ } ++ } ++ ++ fn pair_error_from_bus(error: &WorkerControlBusError) -> PairTransportError { ++ match error { ++ WorkerControlBusError::PublishTimeout => PairTransportError::Timeout, ++ WorkerControlBusError::Closed ++ | WorkerControlBusError::Unavailable ++ | WorkerControlBusError::InvalidCursor { .. } => PairTransportError::Closed, ++ } ++ } ++ + async fn submit( + &self, + qid: &str, + submission: AnswerSubmission, + ) -> Result<(), AnswerTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::interview_answer(qid.to_string(), submission); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| AnswerTransportError::Timeout)? +- .map_err(|_| AnswerTransportError::Closed) ++ .map_err(|err| Self::answer_error_from_bus(&err)) + } + Self::InProcess { interviewer, .. } => interviewer + .submit(qid, submission) +@@ -339,12 +368,11 @@ impl RunAnswerTransport { + + async fn cancel_run(&self) -> Result<(), AnswerTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::cancel_run(); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| AnswerTransportError::Timeout)? +- .map_err(|_| AnswerTransportError::Closed) ++ .map_err(|err| Self::answer_error_from_bus(&err)) + } + Self::InProcess { interviewer, .. } => { + interviewer.cancel_all().await; +@@ -357,12 +385,11 @@ impl RunAnswerTransport { + /// in-process steering hub. + async fn steer(&self, text: String, actor: Principal) -> Result<(), AnswerTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::steer(text, actor); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| AnswerTransportError::Timeout)? +- .map_err(|_| AnswerTransportError::Closed) ++ .map_err(|err| Self::answer_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => { + steering_hub.deliver_steer(text, Some(actor)); +@@ -373,12 +400,11 @@ impl RunAnswerTransport { + + async fn interrupt(&self, actor: Principal) -> Result<(), AnswerTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::interrupt(actor); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| AnswerTransportError::Timeout)? +- .map_err(|_| AnswerTransportError::Closed) ++ .map_err(|err| Self::answer_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => { + steering_hub.interrupt(Some(&actor)); +@@ -393,12 +419,11 @@ impl RunAnswerTransport { + actor: Principal, + ) -> Result<(), AnswerTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::interrupt_then_steer(text, actor); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| AnswerTransportError::Timeout)? +- .map_err(|_| AnswerTransportError::Closed) ++ .map_err(|err| Self::answer_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => { + steering_hub.interrupt_then_steer(&text, Some(&actor)); +@@ -415,12 +440,14 @@ impl RunAnswerTransport { + actor: Principal, + ) -> Result<(), PairTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { ++ run_id: worker_run_id, ++ bus, ++ } => { + let message = WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*worker_run_id, bus, message) + .await +- .map_err(|_| PairTransportError::Timeout)? +- .map_err(|_| PairTransportError::Closed) ++ .map_err(|err| Self::pair_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => steering_hub + .start_pair(run_id, pair_id, target, Some(actor)) +@@ -438,7 +465,7 @@ impl RunAnswerTransport { + actor: Principal, + ) -> Result<(), PairTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::pair_message( + pair_id, + message_id, +@@ -446,10 +473,9 @@ impl RunAnswerTransport { + client_message_id.clone(), + actor, + ); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| PairTransportError::Timeout)? +- .map_err(|_| PairTransportError::Closed) ++ .map_err(|err| Self::pair_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => steering_hub + .send_pair_message(pair_id, message_id, text, client_message_id, Some(actor)) +@@ -460,12 +486,11 @@ impl RunAnswerTransport { + + async fn end_pair(&self, pair_id: PairId, actor: Principal) -> Result<(), PairTransportError> { + match self { +- Self::Subprocess { control_tx } => { ++ Self::Worker { run_id, bus } => { + let message = WorkerControlEnvelope::end_pair(pair_id, actor); +- timeout(WORKER_CONTROL_ENQUEUE_TIMEOUT, control_tx.send(message)) ++ Self::publish_worker_control(*run_id, bus, message) + .await +- .map_err(|_| PairTransportError::Timeout)? +- .map_err(|_| PairTransportError::Closed) ++ .map_err(|err| Self::pair_error_from_bus(&err)) + } + Self::InProcess { steering_hub, .. } => steering_hub + .end_pair(pair_id, Some(actor)) +@@ -473,6 +498,30 @@ impl RunAnswerTransport { + .map_err(PairTransportError::Control), + } + } ++ ++ async fn pause_run(&self) -> Result<(), AnswerTransportError> { ++ match self { ++ Self::Worker { run_id, bus } => { ++ let message = WorkerControlEnvelope::pause_run(); ++ Self::publish_worker_control(*run_id, bus, message) ++ .await ++ .map_err(|err| Self::answer_error_from_bus(&err)) ++ } ++ Self::InProcess { .. } => Err(AnswerTransportError::Closed), ++ } ++ } ++ ++ async fn unpause_run(&self) -> Result<(), AnswerTransportError> { ++ match self { ++ Self::Worker { run_id, bus } => { ++ let message = WorkerControlEnvelope::unpause_run(); ++ Self::publish_worker_control(*run_id, bus, message) ++ .await ++ .map_err(|err| Self::answer_error_from_bus(&err)) ++ } ++ Self::InProcess { .. } => Err(AnswerTransportError::Closed), ++ } ++ } + } + + #[derive(Debug, Clone)] +@@ -1019,6 +1068,7 @@ pub struct AppState { + started_at: Instant, + resource_sampler: resource_sampler::ResourceSampler, + max_concurrent_runs: usize, ++ pub(crate) worker_control_bus: Arc, + scheduler_notify: Notify, + global_event_tx: broadcast::Sender, + /// Per-run coalescing registry for `GET /runs/{id}/files`. Concurrent +@@ -1174,6 +1224,8 @@ pub(crate) struct AppStateConfig { + pub(crate) http_client: Option, + pub(crate) sandbox_provider_registry: Option, + pub(crate) shutdown: CancellationToken, ++ #[cfg(test)] ++ pub(crate) worker_control_bus: Option>, + #[cfg(any(test, feature = "test-support"))] + pub(crate) automation_materializer_override: Option>, + } +@@ -2226,6 +2278,8 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result anyhow::Result = { ++ #[cfg(test)] ++ { ++ worker_control_bus.unwrap_or_else(|| Arc::new(LocalWorkerControlBus::new())) ++ } ++ #[cfg(not(test))] ++ { ++ Arc::new(LocalWorkerControlBus::new()) ++ } ++ }; + Ok(Arc::new(AppState { + runs: Mutex::new(HashMap::new()), + aggregate_billing: Mutex::new(BillingAccumulator::default()), +@@ -2324,6 +2388,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result { +@@ -3019,6 +3091,7 @@ async fn finish_cancelled_run_before_execution(state: &Arc, run_id: Ru + clear_live_run_state(managed_run); + } + drop(runs); ++ cleanup_worker_control_bus_for_run(state.as_ref(), run_id); + state.scheduler_notify.notify_one(); + } + +@@ -3154,6 +3227,7 @@ fn fail_managed_run(state: &Arc, run_id: RunId, reason: FailureReason, + managed_run.error = Some(message); + clear_live_run_state(managed_run); + } ++ cleanup_worker_control_bus_for_run(state.as_ref(), run_id); + } + + fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) { +@@ -3229,6 +3303,7 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) + managed_run.active_api_targets.clear(); + managed_run.active_steerable_stages.clear(); + managed_run.active_non_steerable_stages.clear(); ++ cleanup_worker_control_bus_for_run(state, run_id); + } + EventBody::RunFailed(props) => { + managed_run.status = RunStatus::Failed { +@@ -3241,6 +3316,7 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) + managed_run.active_api_targets.clear(); + managed_run.active_steerable_stages.clear(); + managed_run.active_non_steerable_stages.clear(); ++ cleanup_worker_control_bus_for_run(state, run_id); + } + // Track active agent sessions by steerability. Activated/deactivated + // are leased by session id so stale deactivations cannot clear a newer +@@ -3325,20 +3401,6 @@ async fn drain_worker_stderr(run_id: RunId, stderr: ChildStderr) -> anyhow::Resu + Ok(()) + } + +-async fn pump_worker_control_jsonl( +- mut stdin: ChildStdin, +- mut control_rx: mpsc::Receiver, +-) -> anyhow::Result<()> { +- while let Some(message) = control_rx.recv().await { +- let mut line = serde_json::to_vec(&message)?; +- line.push(b'\n'); +- stdin.write_all(&line).await?; +- stdin.flush().await?; +- } +- +- Ok(()) +-} +- + async fn append_worker_exit_failure( + run_store: &fabro_store::RunDatabase, + run_id: RunId, +@@ -3422,7 +3484,7 @@ fn worker_command( + .arg(run_id.to_string()) + .arg("--mode") + .arg(worker_mode_arg(mode)) +- .stdin(Stdio::piped()) ++ .stdin(Stdio::null()) + .stdout(worker_stdout) + .stderr(Stdio::piped()); + +@@ -4101,25 +4163,6 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { + } + } + +- let Some(stdin) = child.stdin.take() else { +- let message = "Worker stdin pipe was unavailable".to_string(); +- tracing::error!(run_id = %run_id, "{message}"); +- let _ = child.start_kill(); +- let failure_event = workflow_event::Event::workflow_run_failed_from_error( +- &WorkflowError::engine(message.clone()), +- fabro_types::RunTiming::default(), +- FailureReason::LaunchFailed, +- None, +- None, +- None, +- None, +- ); +- let _ = workflow_event::append_event(&run_store, &run_id, &failure_event).await; +- fail_managed_run(&state, run_id, FailureReason::LaunchFailed, message); +- state.scheduler_notify.notify_one(); +- return; +- }; +- + let Some(stderr) = child.stderr.take() else { + let message = "Worker stderr pipe was unavailable".to_string(); + tracing::error!(run_id = %run_id, "{message}"); +@@ -4139,15 +4182,16 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { + return; + }; + +- let (control_tx, control_rx) = mpsc::channel(WORKER_CONTROL_QUEUE_CAPACITY); + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + if let Some(managed_run) = runs.get_mut(&run_id) { +- managed_run.answer_transport = Some(RunAnswerTransport::Subprocess { control_tx }); ++ managed_run.answer_transport = Some(RunAnswerTransport::Worker { ++ run_id, ++ bus: Arc::clone(&state.worker_control_bus), ++ }); + } + } + +- let control_task = tokio::spawn(pump_worker_control_jsonl(stdin, control_rx)); + let stderr_task = tokio::spawn(drain_worker_stderr(run_id, stderr)); + + let wait_status = match child.wait().await { +@@ -4172,9 +4216,6 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { + } + }; + +- control_task.abort(); +- let _ = control_task.await; +- + match stderr_task.await { + Ok(Ok(())) => {} + Ok(Err(err)) => { +diff --git a/lib/crates/fabro-server/src/server/handler/lifecycle.rs b/lib/crates/fabro-server/src/server/handler/lifecycle.rs +index 0c2697133..d8a2589ac 100644 +--- a/lib/crates/fabro-server/src/server/handler/lifecycle.rs ++++ b/lib/crates/fabro-server/src/server/handler/lifecycle.rs +@@ -387,10 +387,6 @@ async fn cancel_run( + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } => { +- let use_cancel_signal = !matches!( +- managed_run.answer_transport, +- Some(RunAnswerTransport::InProcess { .. }) +- ); + let persist_cancelled_status = matches!( + managed_run.status, + RunStatus::Submitted | RunStatus::Pending { .. } | RunStatus::Runnable +@@ -404,9 +400,7 @@ async fn cancel_run( + persist_cancelled_status, + managed_run.answer_transport.clone(), + managed_run.cancel_token.clone(), +- use_cancel_signal +- .then(|| managed_run.cancel_tx.take()) +- .flatten(), ++ managed_run.cancel_tx.take(), + managed_run.worker_pid, + )) + } +@@ -436,22 +430,29 @@ async fn cancel_run( + if let Some(token) = &cancel_token { + token.cancel(); + } +- let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx { ++ let sent_in_process_cancel = if let Some(cancel_tx) = cancel_tx { + let _ = cancel_tx.send(()); + true + } else { + false + }; +- if let Some(answer_transport) = answer_transport { +- if !(sent_cancel_signal && matches!(answer_transport, RunAnswerTransport::InProcess { .. })) ++ let delivered_control = if let Some(answer_transport) = answer_transport { ++ if sent_in_process_cancel ++ && matches!(answer_transport, RunAnswerTransport::InProcess { .. }) + { +- let _ = answer_transport.cancel_run().await; ++ true ++ } else { ++ answer_transport.cancel_run().await.is_ok() ++ } ++ } else { ++ false ++ }; ++ if !delivered_control { ++ if let Some(worker_pid) = worker_pid { ++ #[cfg(unix)] ++ fabro_proc::sigterm(worker_pid); ++ schedule_worker_kill(Arc::clone(&state), id, worker_pid); + } +- } +- if let Some(worker_pid) = worker_pid { +- #[cfg(unix)] +- fabro_proc::sigterm(worker_pid); +- schedule_worker_kill(Arc::clone(&state), id, worker_pid); + } + + if persist_cancelled_status { +@@ -504,9 +505,9 @@ async fn unmanaged_cancel_response( + /// How `pause_run` should enact the transition, chosen from the current run + /// status. + enum PauseMode { +- /// Worker is running; ask it to pause via SIGUSR1. Status flips to +- /// `Paused` once the worker acknowledges. +- Signal { worker_pid: u32 }, ++ /// Worker is running; ask it to pause via the worker control bus. Status ++ /// flips to `Paused` once the worker acknowledges. ++ Transport { transport: RunAnswerTransport }, + /// Worker is blocked on a human gate; flip to `Paused` directly by + /// appending `RunPaused` ourselves. + AppendEvent, +@@ -514,8 +515,9 @@ enum PauseMode { + + /// How `unpause_run` should enact the transition. + enum UnpauseMode { +- /// No outstanding block; ask the worker to resume via SIGUSR2. +- Signal { worker_pid: u32 }, ++ /// No outstanding block; ask the worker to resume via the worker control ++ /// bus. ++ Transport { transport: RunAnswerTransport }, + /// Was paused while blocked; append `RunUnpaused` and let the reducer + /// restore the underlying blocked state from `Paused { prior_block }`. + AppendEvent, +@@ -544,11 +546,11 @@ async fn pause_run( + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) if managed_run.status == RunStatus::Running => { +- let Some(worker_pid) = managed_run.worker_pid else { ++ let Some(transport) = managed_run.answer_transport.clone() else { + return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") + .into_response(); + }; +- PauseMode::Signal { worker_pid } ++ PauseMode::Transport { transport } + } + Some(managed_run) if matches!(managed_run.status, RunStatus::Blocked { .. }) => { + PauseMode::AppendEvent +@@ -578,11 +580,14 @@ async fn pause_run( + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + match mode { +- PauseMode::Signal { worker_pid } => { +- #[cfg(unix)] +- fabro_proc::sigusr1(worker_pid); +- #[cfg(not(unix))] +- let _ = worker_pid; ++ PauseMode::Transport { transport } => { ++ if transport.pause_run().await.is_err() { ++ return ApiError::new( ++ StatusCode::SERVICE_UNAVAILABLE, ++ "Failed to deliver pause request to the active run.", ++ ) ++ .into_response(); ++ } + } + PauseMode::AppendEvent => { + if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { +@@ -625,11 +630,11 @@ async fn unpause_run( + prior_block: Some(_), + } => UnpauseMode::AppendEvent, + RunStatus::Paused { prior_block: None } => { +- let Some(worker_pid) = managed_run.worker_pid else { ++ let Some(transport) = managed_run.answer_transport.clone() else { + return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") + .into_response(); + }; +- UnpauseMode::Signal { worker_pid } ++ UnpauseMode::Transport { transport } + } + _ => { + return ApiError::new(StatusCode::CONFLICT, "Run is not paused.") +@@ -658,11 +663,14 @@ async fn unpause_run( + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + match mode { +- UnpauseMode::Signal { worker_pid } => { +- #[cfg(unix)] +- fabro_proc::sigusr2(worker_pid); +- #[cfg(not(unix))] +- let _ = worker_pid; ++ UnpauseMode::Transport { transport } => { ++ if transport.unpause_run().await.is_err() { ++ return ApiError::new( ++ StatusCode::SERVICE_UNAVAILABLE, ++ "Failed to deliver unpause request to the active run.", ++ ) ++ .into_response(); ++ } + } + UnpauseMode::AppendEvent => { + if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { +diff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs +index 3b7cc1ec1..33c6aa8eb 100644 +--- a/lib/crates/fabro-server/src/server/handler/mod.rs ++++ b/lib/crates/fabro-server/src/server/handler/mod.rs +@@ -23,6 +23,7 @@ mod sessions; + mod steer; + pub(in crate::server) mod system; + mod variables; ++mod worker_control; + + pub(super) use system::{health, openapi_spec}; + +@@ -167,6 +168,7 @@ pub(super) fn real_routes() -> Router> { + .merge(models::routes()) + .merge(secrets::routes()) + .merge(variables::routes()) ++ .merge(worker_control::routes()) + .merge(sessions::routes()) + .merge(system::routes()) + .merge(completions::routes()) +diff --git a/lib/crates/fabro-server/src/server/handler/worker_control.rs b/lib/crates/fabro-server/src/server/handler/worker_control.rs +new file mode 100644 +index 000000000..d643a904e +--- /dev/null ++++ b/lib/crates/fabro-server/src/server/handler/worker_control.rs +@@ -0,0 +1,166 @@ ++use std::sync::Arc; ++ ++use axum::extract::ws::{ ++ CloseFrame, Message as WsMessage, WebSocket, WebSocketUpgrade, close_code, ++}; ++use fabro_interview::{ ++ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON, ++ WORKER_CONTROL_WS_LIVENESS_TIMEOUT, WORKER_CONTROL_WS_PING_INTERVAL, ++ WorkerControlDeliveryFrame, ++}; ++use futures_util::{SinkExt, StreamExt}; ++use tokio::time::{self, Instant, MissedTickBehavior}; ++ ++use super::super::{ ++ ApiError, AppState, IntoResponse, Query, RequireWorkerRunScoped, Response, Router, State, ++ StatusCode, get, ++}; ++use crate::worker_control::{WorkerControlBusError, WorkerControlCursor, WorkerControlReceiver}; ++ ++#[derive(Debug, serde::Deserialize)] ++struct WorkerControlStreamQuery { ++ after: Option, ++} ++ ++pub(super) fn routes() -> Router> { ++ Router::new().route( ++ "/runs/{id}/worker/control-stream", ++ get(worker_control_stream), ++ ) ++} ++ ++async fn worker_control_stream( ++ RequireWorkerRunScoped(id): RequireWorkerRunScoped, ++ State(state): State>, ++ Query(query): Query, ++ ws: WebSocketUpgrade, ++) -> Response { ++ let cached = match state.store.get_cached_run(&id).await { ++ Ok(Some(cached)) => cached, ++ Ok(None) => return ApiError::not_found("Run not found.").into_response(), ++ Err(err) => { ++ return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) ++ .into_response(); ++ } ++ }; ++ if cached.projection.archived_at.is_some() { ++ return ApiError::new(StatusCode::CONFLICT, "Run is archived.").into_response(); ++ } ++ if cached.projection.status.is_terminal() { ++ return ApiError::new( ++ StatusCode::CONFLICT, ++ "Run is terminal and cannot accept worker control streams.", ++ ) ++ .into_response(); ++ } ++ ++ let cursor = match WorkerControlCursor::from_after_query(query.after.as_deref()) { ++ Ok(cursor) => cursor, ++ Err(err @ WorkerControlBusError::InvalidCursor { .. }) => { ++ return ApiError::new(StatusCode::GONE, err.to_string()).into_response(); ++ } ++ Err(err) => return worker_control_bus_error_response(&err), ++ }; ++ let receiver = match state.worker_control_bus.subscribe(id, cursor).await { ++ Ok(receiver) => receiver, ++ Err(err @ WorkerControlBusError::InvalidCursor { .. }) => { ++ return ApiError::new(StatusCode::GONE, err.to_string()).into_response(); ++ } ++ Err(err) => return worker_control_bus_error_response(&err), ++ }; ++ ++ ws.on_upgrade(move |socket| worker_control_websocket(socket, receiver)) ++} ++ ++fn worker_control_bus_error_response(err: &WorkerControlBusError) -> Response { ++ let status = match err { ++ WorkerControlBusError::Closed | WorkerControlBusError::Unavailable => { ++ StatusCode::SERVICE_UNAVAILABLE ++ } ++ WorkerControlBusError::PublishTimeout => StatusCode::SERVICE_UNAVAILABLE, ++ WorkerControlBusError::InvalidCursor { .. } => StatusCode::GONE, ++ }; ++ ApiError::new(status, err.to_string()).into_response() ++} ++ ++async fn worker_control_websocket(socket: WebSocket, mut receiver: WorkerControlReceiver) { ++ let (mut sender, mut receiver_ws) = socket.split(); ++ let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL); ++ ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); ++ let mut last_liveness = Instant::now(); ++ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT); ++ tokio::pin!(liveness_timeout); ++ ++ loop { ++ liveness_timeout ++ .as_mut() ++ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT); ++ ++ tokio::select! { ++ delivery = receiver.recv() => { ++ let Some(delivery) = delivery else { ++ let _ = sender.send(WsMessage::Close(None)).await; ++ return; ++ }; ++ let delivery = match delivery { ++ Ok(delivery) => delivery, ++ Err(WorkerControlBusError::InvalidCursor { .. }) => { ++ let _ = sender.send(invalid_cursor_close_message()).await; ++ return; ++ } ++ Err(_) => { ++ let _ = sender.send(WsMessage::Close(None)).await; ++ return; ++ } ++ }; ++ let frame = WorkerControlDeliveryFrame { ++ id: delivery.id.to_string(), ++ envelope: delivery.envelope, ++ }; ++ let Ok(text) = serde_json::to_string(&frame) else { ++ let _ = sender.send(WsMessage::Close(None)).await; ++ return; ++ }; ++ if sender.send(WsMessage::Text(text.into())).await.is_err() { ++ return; ++ } ++ } ++ message = receiver_ws.next() => { ++ let Some(message) = message else { ++ return; ++ }; ++ match message { ++ Ok(WsMessage::Ping(payload)) => { ++ last_liveness = Instant::now(); ++ if sender.send(WsMessage::Pong(payload)).await.is_err() { ++ return; ++ } ++ } ++ Ok(WsMessage::Pong(_) | WsMessage::Text(_) | WsMessage::Binary(_)) => { ++ last_liveness = Instant::now(); ++ } ++ Ok(WsMessage::Close(_)) | Err(_) => return, ++ } ++ } ++ _ = ping_interval.tick() => { ++ if sender.send(WsMessage::Ping(Vec::new().into())).await.is_err() { ++ return; ++ } ++ } ++ () = &mut liveness_timeout => { ++ let _ = sender.send(WsMessage::Close(Some(CloseFrame { ++ code: close_code::AWAY, ++ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(), ++ }))).await; ++ return; ++ } ++ } ++ } ++} ++ ++fn invalid_cursor_close_message() -> WsMessage { ++ WsMessage::Close(Some(CloseFrame { ++ code: close_code::POLICY, ++ reason: WORKER_CONTROL_INVALID_CURSOR_REASON.into(), ++ })) ++} +diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs +index a422fa05b..fbb10bbe2 100644 +--- a/lib/crates/fabro-server/src/server/tests.rs ++++ b/lib/crates/fabro-server/src/server/tests.rs +@@ -13,7 +13,8 @@ use fabro_automation::{AutomationId, AutomationTarget}; + use fabro_config::ServerSettingsBuilder; + use fabro_config::bind::Bind; + use fabro_interview::{ +- AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlMessage, ++ AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlDeliveryFrame, ++ WorkerControlEnvelope, WorkerControlMessage, + }; + use fabro_llm::types::{Message as LlmMessage, Request as LlmRequest, TokenCounts}; + use fabro_model::catalog::LlmCatalogSettings; +@@ -33,6 +34,8 @@ use httpmock::Method::{GET, POST}; + use httpmock::MockServer; + use serde_json::json; + use tokio_stream::StreamExt as _; ++use tokio_tungstenite::tungstenite::client::IntoClientRequest; ++use tokio_tungstenite::tungstenite::protocol::Message as WebSocketMessage; + use tower::ServiceExt; + use tracing::field::{Field, Visit}; + use tracing::{Event as TracingEvent, Subscriber, subscriber}; +@@ -45,6 +48,9 @@ use crate::automation_materializer::AutomationRunMaterializeInput; + use crate::github_webhooks::compute_signature; + use crate::jwt_auth::{AuthMode, ConfiguredAuth}; + use crate::test_support::*; ++use crate::worker_control::{ ++ LocalWorkerControlBus, WorkerControlBus, WorkerControlCursor, WorkerControlReceiver, ++}; + + const MINIMAL_DOT: &str = r#"digraph Test { + graph [goal="Test"] +@@ -669,6 +675,292 @@ fn bearer_request(method: Method, path: &str, bearer: &str, body: Body) -> Reque + .unwrap() + } + ++struct WorkerControlWsTestServer { ++ base_url: String, ++ task: tokio::task::JoinHandle<()>, ++} ++ ++impl WorkerControlWsTestServer { ++ async fn spawn(app: Router) -> Self { ++ let listener = tokio::net::TcpListener::bind("127.0.0.1:0") ++ .await ++ .expect("test WebSocket listener should bind"); ++ let addr = listener ++ .local_addr() ++ .expect("test WebSocket listener should have a local address"); ++ let task = tokio::spawn(async move { ++ let result = axum::serve( ++ listener, ++ app.into_make_service_with_connect_info::(), ++ ) ++ .await; ++ if let Err(err) = result { ++ tracing::debug!(error = %err, "test WebSocket server stopped"); ++ } ++ }); ++ Self { ++ base_url: format!("ws://{addr}"), ++ task, ++ } ++ } ++ ++ fn worker_control_url(&self, run_id: RunId, after: Option<&str>) -> String { ++ let mut url = format!( ++ "{}/api/v1/runs/{run_id}/worker/control-stream", ++ self.base_url ++ ); ++ if let Some(after) = after { ++ url.push_str("?after="); ++ url.push_str(after); ++ } ++ url ++ } ++} ++ ++impl Drop for WorkerControlWsTestServer { ++ fn drop(&mut self) { ++ self.task.abort(); ++ } ++} ++ ++fn worker_control_ws_request( ++ server: &WorkerControlWsTestServer, ++ run_id: RunId, ++ bearer: Option<&str>, ++ after: Option<&str>, ++) -> Request<()> { ++ let mut request = server ++ .worker_control_url(run_id, after) ++ .into_client_request() ++ .expect("test worker-control WebSocket request should build"); ++ if let Some(bearer) = bearer { ++ request.headers_mut().insert( ++ header::AUTHORIZATION, ++ format!("Bearer {bearer}") ++ .parse() ++ .expect("test bearer header should parse"), ++ ); ++ } ++ request ++} ++ ++async fn connect_worker_control_ws( ++ server: &WorkerControlWsTestServer, ++ run_id: RunId, ++ bearer: &str, ++ after: Option<&str>, ++) -> tokio_tungstenite::WebSocketStream> { ++ let request = worker_control_ws_request(server, run_id, Some(bearer), after); ++ let (socket, _) = tokio_tungstenite::connect_async(request) ++ .await ++ .expect("worker-control WebSocket should connect"); ++ socket ++} ++ ++async fn assert_worker_control_ws_rejected( ++ server: &WorkerControlWsTestServer, ++ run_id: RunId, ++ bearer: Option<&str>, ++ after: Option<&str>, ++ expected: StatusCode, ++) { ++ let request = worker_control_ws_request(server, run_id, bearer, after); ++ let error = tokio_tungstenite::connect_async(request) ++ .await ++ .expect_err("worker-control WebSocket should be rejected"); ++ match error { ++ tokio_tungstenite::tungstenite::Error::Http(response) => { ++ assert_eq!(response.status(), expected); ++ } ++ other => panic!("expected HTTP rejection {expected}, got {other:#}"), ++ } ++} ++ ++async fn next_worker_control_frame( ++ socket: &mut tokio_tungstenite::WebSocketStream< ++ tokio_tungstenite::MaybeTlsStream, ++ >, ++) -> WorkerControlDeliveryFrame { ++ let message = tokio::time::timeout(std::time::Duration::from_secs(2), async { ++ loop { ++ let message = futures_util::StreamExt::next(socket) ++ .await ++ .expect("worker-control WebSocket should remain open") ++ .expect("worker-control WebSocket frame should be ok"); ++ match message { ++ WebSocketMessage::Text(text) => return text, ++ WebSocketMessage::Ping(payload) => { ++ futures_util::SinkExt::send(socket, WebSocketMessage::Pong(payload)) ++ .await ++ .expect("test worker-control pong should send"); ++ } ++ WebSocketMessage::Pong(_) ++ | WebSocketMessage::Binary(_) ++ | WebSocketMessage::Frame(_) => {} ++ WebSocketMessage::Close(frame) => { ++ panic!("worker-control WebSocket closed before text frame: {frame:?}"); ++ } ++ } ++ } ++ }) ++ .await ++ .expect("worker-control frame should arrive"); ++ serde_json::from_str(message.as_str()).expect("worker-control delivery frame should parse") ++} ++ ++#[tokio::test(flavor = "current_thread")] ++async fn worker_control_stream_rejects_missing_user_and_cross_run_auth() { ++ let (_state, app) = jwt_auth_app(); ++ let user_bearer = issue_test_user_jwt(); ++ let run_id = create_run_with_bearer(&app, &user_bearer).await; ++ let worker_bearer = issue_test_worker_token(&run_id); ++ let other_run_id = create_run_with_bearer(&app, &user_bearer).await; ++ let other_worker_bearer = issue_test_worker_token(&other_run_id); ++ let server = WorkerControlWsTestServer::spawn(app).await; ++ ++ assert_worker_control_ws_rejected(&server, run_id, None, None, StatusCode::UNAUTHORIZED).await; ++ assert_worker_control_ws_rejected( ++ &server, ++ run_id, ++ Some(&user_bearer), ++ None, ++ StatusCode::FORBIDDEN, ++ ) ++ .await; ++ assert_worker_control_ws_rejected( ++ &server, ++ run_id, ++ Some(&other_worker_bearer), ++ None, ++ StatusCode::FORBIDDEN, ++ ) ++ .await; ++ ++ let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await; ++ futures_util::SinkExt::send(&mut socket, WebSocketMessage::Close(None)) ++ .await ++ .unwrap(); ++} ++ ++#[tokio::test(flavor = "current_thread")] ++async fn worker_control_stream_start_subscription_delivers_frames() { ++ let (state, app) = jwt_auth_app(); ++ let user_bearer = issue_test_user_jwt(); ++ let run_id = create_run_with_bearer(&app, &user_bearer).await; ++ let worker_bearer = issue_test_worker_token(&run_id); ++ let server = WorkerControlWsTestServer::spawn(app).await; ++ let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await; ++ ++ let expected = WorkerControlEnvelope::cancel_run(); ++ let id = state ++ .worker_control_bus ++ .publish(run_id, expected.clone()) ++ .await ++ .unwrap(); ++ let frame = next_worker_control_frame(&mut socket).await; ++ ++ assert_eq!(frame.id, id.to_string()); ++ assert_eq!(frame.envelope, expected); ++} ++ ++#[tokio::test(flavor = "current_thread")] ++async fn worker_control_stream_after_subscription_delivers_only_later_frames() { ++ let (state, app) = jwt_auth_app(); ++ let user_bearer = issue_test_user_jwt(); ++ let run_id = create_run_with_bearer(&app, &user_bearer).await; ++ let worker_bearer = issue_test_worker_token(&run_id); ++ let first = state ++ .worker_control_bus ++ .publish(run_id, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ let expected = WorkerControlEnvelope::pause_run(); ++ let second = state ++ .worker_control_bus ++ .publish(run_id, expected.clone()) ++ .await ++ .unwrap(); ++ let server = WorkerControlWsTestServer::spawn(app).await; ++ ++ let mut socket = ++ connect_worker_control_ws(&server, run_id, &worker_bearer, Some(first.as_str())).await; ++ let frame = next_worker_control_frame(&mut socket).await; ++ ++ assert_eq!(frame.id, second.to_string()); ++ assert_eq!(frame.envelope, expected); ++} ++ ++#[tokio::test(flavor = "current_thread")] ++async fn worker_control_stream_invalid_cursor_is_http_gone_before_upgrade() { ++ let (_state, app) = jwt_auth_app(); ++ let user_bearer = issue_test_user_jwt(); ++ let run_id = create_run_with_bearer(&app, &user_bearer).await; ++ let worker_bearer = issue_test_worker_token(&run_id); ++ let server = WorkerControlWsTestServer::spawn(app).await; ++ ++ assert_worker_control_ws_rejected( ++ &server, ++ run_id, ++ Some(&worker_bearer), ++ Some("local:999"), ++ StatusCode::GONE, ++ ) ++ .await; ++} ++ ++#[tokio::test(flavor = "current_thread")] ++async fn worker_control_stream_rejects_missing_terminal_and_archived_runs() { ++ let (state, app) = jwt_auth_app(); ++ let user_bearer = issue_test_user_jwt(); ++ let missing_run_id = RunId::new(); ++ let missing_worker_bearer = issue_test_worker_token(&missing_run_id); ++ let terminal_run_id = RunId::new(); ++ create_succeeded_run(&state, terminal_run_id).await; ++ let terminal_worker_bearer = issue_test_worker_token(&terminal_run_id); ++ let archived_run_id = RunId::new(); ++ create_succeeded_run(&state, archived_run_id).await; ++ let archived_worker_bearer = issue_test_worker_token(&archived_run_id); ++ let archive_response = app ++ .clone() ++ .oneshot( ++ Request::builder() ++ .method("POST") ++ .uri(api(&format!("/runs/{archived_run_id}/archive"))) ++ .header(header::AUTHORIZATION, format!("Bearer {user_bearer}")) ++ .body(Body::empty()) ++ .unwrap(), ++ ) ++ .await ++ .unwrap(); ++ response_json!(archive_response, StatusCode::OK).await; ++ let server = WorkerControlWsTestServer::spawn(app).await; ++ ++ assert_worker_control_ws_rejected( ++ &server, ++ missing_run_id, ++ Some(&missing_worker_bearer), ++ None, ++ StatusCode::NOT_FOUND, ++ ) ++ .await; ++ assert_worker_control_ws_rejected( ++ &server, ++ terminal_run_id, ++ Some(&terminal_worker_bearer), ++ None, ++ StatusCode::CONFLICT, ++ ) ++ .await; ++ assert_worker_control_ws_rejected( ++ &server, ++ archived_run_id, ++ Some(&archived_worker_bearer), ++ None, ++ StatusCode::CONFLICT, ++ ) ++ .await; ++} ++ + fn json_bearer_request( + method: Method, + path: &str, +@@ -1664,6 +1956,7 @@ fn slack_app_state_with_secret_sources( + http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), + sandbox_provider_registry: None, + shutdown: tokio_util::sync::CancellationToken::new(), ++ worker_control_bus: None, + automation_materializer_override: None, + }) + .expect("slack test app state should build") +@@ -1764,6 +2057,7 @@ fn slack_service_respects_disabled_server_config_even_with_vault_tokens() { + http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), + sandbox_provider_registry: None, + shutdown: tokio_util::sync::CancellationToken::new(), ++ worker_control_bus: None, + automation_materializer_override: None, + }) + .expect("slack disabled test app state should build"); +@@ -1771,6 +2065,23 @@ fn slack_service_respects_disabled_server_config_even_with_vault_tokens() { + assert!(state.slack_service.is_none()); + } + ++#[cfg(unix)] ++#[test] ++fn worker_command_uses_null_stdin_and_token_env() { ++ let storage_dir = tempfile::tempdir().unwrap(); ++ let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN)); ++ let cmd = worker_command( ++ state.as_ref(), ++ RunId::new(), ++ RunExecutionMode::Start, ++ storage_dir.path(), ++ false, ++ ) ++ .unwrap(); ++ ++ assert_worker_command_passes_token_only_by_env(&cmd); ++} ++ + #[cfg(unix)] + #[test] + fn worker_command_default_token_omits_agent_run_tools_scope() { +@@ -2069,6 +2380,7 @@ methods = ["dev-token"] + http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), + sandbox_provider_registry: None, + shutdown: tokio_util::sync::CancellationToken::new(), ++ worker_control_bus: None, + automation_materializer_override: None, + }) else { + panic!("build_app_state should require SESSION_SECRET") +@@ -2197,6 +2509,7 @@ fn build_test_app_state_with_vault_path(vault_path: &Path) -> anyhow::Result crate::worker_token:: + .claims + } + ++async fn worker_transport_with_receiver( ++ run_id: RunId, ++) -> (RunAnswerTransport, WorkerControlReceiver) { ++ let bus = StdArc::new(LocalWorkerControlBus::new()); ++ let receiver = bus ++ .subscribe(run_id, WorkerControlCursor::Start) ++ .await ++ .expect("test worker bus should subscribe"); ++ // Ensure the subscription task is waiting before the test publishes. ++ tokio::task::yield_now().await; ++ let bus: StdArc = bus; ++ let transport = RunAnswerTransport::Worker { run_id, bus }; ++ (transport, receiver) ++} ++ ++async fn recv_worker_control_envelope( ++ receiver: &mut WorkerControlReceiver, ++) -> WorkerControlEnvelope { ++ receiver ++ .recv() ++ .await ++ .expect("test worker control receiver should stay open") ++ .expect("test worker control delivery should succeed") ++ .envelope ++} ++ + #[tokio::test] +-async fn subprocess_answer_transport_cancel_run_enqueues_cancel_message() { +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let transport = RunAnswerTransport::Subprocess { control_tx }; ++async fn worker_answer_transport_cancel_run_publishes_cancel_message() { ++ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await; + + transport.cancel_run().await.unwrap(); + + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::cancel_run()) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::cancel_run() + ); + } + + #[tokio::test] +-async fn subprocess_answer_transport_steer_enqueues_plain_steer_message() { +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let transport = RunAnswerTransport::Subprocess { control_tx }; ++async fn worker_answer_transport_steer_publishes_plain_steer_message() { ++ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await; + let actor = Principal::System { + system_kind: SystemActorKind::Engine, + }; +@@ -2404,15 +2741,14 @@ async fn subprocess_answer_transport_steer_enqueues_plain_steer_message() { + .unwrap(); + + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::steer("try again", actor)) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::steer("try again", actor) + ); + } + + #[tokio::test] +-async fn subprocess_answer_transport_interrupt_enqueues_interrupt_message() { +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let transport = RunAnswerTransport::Subprocess { control_tx }; ++async fn worker_answer_transport_interrupt_publishes_interrupt_message() { ++ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await; + let actor = Principal::System { + system_kind: SystemActorKind::Engine, + }; +@@ -2420,15 +2756,14 @@ async fn subprocess_answer_transport_interrupt_enqueues_interrupt_message() { + transport.interrupt(actor.clone()).await.unwrap(); + + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::interrupt(actor)) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::interrupt(actor) + ); + } + + #[tokio::test] +-async fn subprocess_answer_transport_interrupt_then_steer_enqueues_single_combined_message() { +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let transport = RunAnswerTransport::Subprocess { control_tx }; ++async fn worker_answer_transport_interrupt_then_steer_publishes_single_combined_message() { ++ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await; + let actor = Principal::System { + system_kind: SystemActorKind::Engine, + }; +@@ -2439,19 +2774,15 @@ async fn subprocess_answer_transport_interrupt_then_steer_enqueues_single_combin + .unwrap(); + + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::interrupt_then_steer( +- "try again", +- actor +- )) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::interrupt_then_steer("try again", actor) + ); + } + + #[tokio::test] +-async fn subprocess_answer_transport_pair_commands_enqueue_control_messages() { +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(3); +- let transport = RunAnswerTransport::Subprocess { control_tx }; ++async fn worker_answer_transport_pair_commands_publish_control_messages() { + let run_id = fixtures::RUN_1; ++ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; + let pair_id = "01HZX6M29F1CD5YYMHT1F5D7WQ".parse().unwrap(); + let message_id = "01HZX6M4D7Y1QW0Q0P6V8Z4DR5".parse().unwrap(); + let actor = Principal::System { +@@ -2476,27 +2807,39 @@ async fn subprocess_answer_transport_pair_commands_enqueue_control_messages() { + transport.end_pair(pair_id, actor.clone()).await.unwrap(); + + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::start_pair( +- run_id, +- pair_id, +- target, +- actor.clone() +- )) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor.clone()) + ); + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::pair_message( ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::pair_message( + pair_id, + message_id, + "inspect this", + Some("client-1".to_string()), + actor.clone() +- )) ++ ) + ); + assert_eq!( +- control_rx.recv().await, +- Some(WorkerControlEnvelope::end_pair(pair_id, actor)) ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::end_pair(pair_id, actor) ++ ); ++} ++ ++#[tokio::test] ++async fn worker_answer_transport_pause_and_unpause_publish_control_messages() { ++ let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await; ++ ++ transport.pause_run().await.unwrap(); ++ transport.unpause_run().await.unwrap(); ++ ++ assert_eq!( ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::pause_run() ++ ); ++ assert_eq!( ++ recv_worker_control_envelope(&mut control_rx).await, ++ WorkerControlEnvelope::unpause_run() + ); + } + +@@ -5444,6 +5787,7 @@ fn create_github_token_app_state_with_env_lookup_and_llm_catalog_settings( + http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), + sandbox_provider_registry: None, + shutdown: tokio_util::sync::CancellationToken::new(), ++ worker_control_bus: None, + automation_materializer_override: None, + }; + let state = build_app_state(config).expect("test app state should build"); +@@ -10847,12 +11191,8 @@ async fn steer_without_active_steerable_session_forwards_plain_steer_for_bufferi + let state = test_app_state(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + + let req = Request::builder() + .method("POST") +@@ -10863,7 +11203,7 @@ async fn steer_without_active_steerable_session_forwards_plain_steer_for_bufferi + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::ACCEPTED).await; +- let envelope = control_rx.recv().await.unwrap(); ++ let envelope = recv_worker_control_envelope(&mut control_rx).await; + assert!(matches!( + envelope.message, + WorkerControlMessage::Steer { ref text, .. } if text == "try again" +@@ -10876,12 +11216,8 @@ async fn steer_with_active_non_steerable_session_returns_conflict() { + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; + let stage_id = StageId::new("agent", 1); +- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + runs.get_mut(&run_id) +@@ -10908,12 +11244,8 @@ async fn steer_interrupt_without_active_steerable_session_returns_conflict() { + let state = test_app_state(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; +- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + + let req = Request::builder() + .method("POST") +@@ -10934,12 +11266,8 @@ async fn interrupt_with_active_steerable_session_forwards_interrupt() { + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; + let stage_id = StageId::new("agent", 1); +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + runs.get_mut(&run_id) +@@ -10956,7 +11284,7 @@ async fn interrupt_with_active_steerable_session_forwards_interrupt() { + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::ACCEPTED).await; +- let envelope = control_rx.recv().await.unwrap(); ++ let envelope = recv_worker_control_envelope(&mut control_rx).await; + assert!(matches!( + envelope.message, + WorkerControlMessage::Interrupt { +@@ -10971,12 +11299,8 @@ async fn steer_interrupt_with_active_steerable_session_forwards_combined_control + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; + let stage_id = StageId::new("agent", 1); +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + runs.get_mut(&run_id) +@@ -10994,7 +11318,7 @@ async fn steer_interrupt_with_active_steerable_session_forwards_combined_control + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::ACCEPTED).await; +- let envelope = control_rx.recv().await.unwrap(); ++ let envelope = recv_worker_control_envelope(&mut control_rx).await; + assert!(matches!( + envelope.message, + WorkerControlMessage::InterruptThenSteer { ref text, .. } if text == "try again" +@@ -11160,12 +11484,8 @@ async fn steer_with_active_acp_session_forwards_to_worker() { + let state = test_app_state(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; +- let (control_tx, mut control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + + let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted { + node_id: "agent".to_string(), +@@ -11198,7 +11518,7 @@ async fn steer_with_active_acp_session_forwards_to_worker() { + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::ACCEPTED).await; +- let envelope = control_rx.recv().await.unwrap(); ++ let envelope = recv_worker_control_envelope(&mut control_rx).await; + assert!(matches!( + envelope.message, + WorkerControlMessage::Steer { ref text, .. } if text == "try again" +@@ -11265,12 +11585,8 @@ async fn active_acp_steerable_marker_clears_on_terminal_paths() { + let state = test_app_state(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = fixtures::RUN_1; +- let (control_tx, _control_rx) = tokio::sync::mpsc::channel(1); +- let _temp_dir = insert_running_control_run( +- &state, +- run_id, +- Some(RunAnswerTransport::Subprocess { control_tx }), +- ); ++ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; ++ let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); + let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted { + node_id: "agent".to_string(), + visit: 1, +@@ -13200,11 +13516,13 @@ async fn pause_run_sets_pending_control_on_board_response() { + let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await; + let run_id = run_id_str.parse::().unwrap(); + ++ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + let managed_run = runs.get_mut(&run_id).expect("run should exist"); + managed_run.status = RunStatus::Running; + managed_run.worker_pid = Some(u32::MAX); ++ managed_run.answer_transport = Some(transport); + } + + let req = Request::builder() +@@ -13320,11 +13638,13 @@ async fn unpause_run_sets_pending_control() { + let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await; + let run_id = run_id_str.parse::().unwrap(); + ++ let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + let managed_run = runs.get_mut(&run_id).expect("run should exist"); + managed_run.status = RunStatus::Paused { prior_block: None }; + managed_run.worker_pid = Some(u32::MAX); ++ managed_run.answer_transport = Some(transport); + } + + let req = Request::builder() +diff --git a/lib/crates/fabro-server/src/test_support.rs b/lib/crates/fabro-server/src/test_support.rs +index 11f80e0e7..585aa36e9 100644 +--- a/lib/crates/fabro-server/src/test_support.rs ++++ b/lib/crates/fabro-server/src/test_support.rs +@@ -249,6 +249,8 @@ impl TestAppStateBuilder { + ), + sandbox_provider_registry: self.sandbox_provider_registry, + shutdown: CancellationToken::new(), ++ #[cfg(test)] ++ worker_control_bus: None, + automation_materializer_override: self.automation_materializer, + }) + } +diff --git a/lib/crates/fabro-server/src/worker_control/bus.rs b/lib/crates/fabro-server/src/worker_control/bus.rs +new file mode 100644 +index 000000000..b6fadedb5 +--- /dev/null ++++ b/lib/crates/fabro-server/src/worker_control/bus.rs +@@ -0,0 +1,155 @@ ++use std::fmt; ++ ++use fabro_interview::WorkerControlEnvelope; ++use fabro_types::RunId; ++use futures_util::future::BoxFuture; ++use tokio::sync::mpsc; ++ ++#[derive(Clone, PartialEq, Eq)] ++pub(crate) struct WorkerControlMessageId(String); ++ ++impl WorkerControlMessageId { ++ #[must_use] ++ pub(crate) fn new(value: impl Into) -> Self { ++ Self(value.into()) ++ } ++ ++ #[must_use] ++ pub(crate) fn as_str(&self) -> &str { ++ &self.0 ++ } ++} ++ ++impl fmt::Debug for WorkerControlMessageId { ++ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { ++ f.debug_tuple("WorkerControlMessageId") ++ .field(&self.0) ++ .finish() ++ } ++} ++ ++impl fmt::Display for WorkerControlMessageId { ++ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { ++ f.write_str(&self.0) ++ } ++} ++ ++impl From for WorkerControlMessageId { ++ fn from(value: String) -> Self { ++ Self::new(value) ++ } ++} ++ ++impl From<&str> for WorkerControlMessageId { ++ fn from(value: &str) -> Self { ++ Self::new(value) ++ } ++} ++ ++/// Cursor for replaying a run's worker-control stream. ++/// ++/// Future Redis Streams mapping: ++/// - `Start` maps to `XREAD ... STREAMS fabro:run:{run_id}:control 0-0`. ++/// - `After(id)` maps to `XREAD ... STREAMS fabro:run:{run_id}:control {id}`. ++#[derive(Debug, Clone, PartialEq, Eq)] ++pub(crate) enum WorkerControlCursor { ++ Start, ++ After(WorkerControlMessageId), ++} ++ ++impl WorkerControlCursor { ++ pub(crate) fn from_after_query(after: Option<&str>) -> Result { ++ match after { ++ None => Ok(Self::Start), ++ Some(value) if value.trim().is_empty() => Err(WorkerControlBusError::InvalidCursor { ++ cursor: value.to_string(), ++ reason: "cursor id must not be empty".to_string(), ++ }), ++ Some(value) => Ok(Self::After(WorkerControlMessageId::from(value))), ++ } ++ } ++} ++ ++#[derive(Debug, Clone, PartialEq, Eq)] ++pub(crate) struct WorkerControlDelivery { ++ pub(crate) id: WorkerControlMessageId, ++ pub(crate) envelope: WorkerControlEnvelope, ++} ++ ++#[allow( ++ dead_code, ++ reason = "The local backend does not construct every cross-backend error variant." ++)] ++#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] ++pub(crate) enum WorkerControlBusError { ++ #[error("worker control backend is closed")] ++ Closed, ++ #[error("worker control backend is unavailable")] ++ Unavailable, ++ #[error("worker control cursor `{cursor}` is invalid: {reason}")] ++ InvalidCursor { cursor: String, reason: String }, ++ #[error("timed out publishing worker control message")] ++ PublishTimeout, ++} ++ ++impl WorkerControlBusError { ++ #[must_use] ++ pub(crate) fn invalid_cursor(cursor: impl Into, reason: impl Into) -> Self { ++ Self::InvalidCursor { ++ cursor: cursor.into(), ++ reason: reason.into(), ++ } ++ } ++} ++ ++pub(crate) type WorkerControlReceiver = ++ mpsc::Receiver>; ++ ++pub(crate) trait WorkerControlBus: Send + Sync { ++ fn publish( ++ &self, ++ run_id: RunId, ++ envelope: WorkerControlEnvelope, ++ ) -> BoxFuture<'_, Result>; ++ ++ fn subscribe( ++ &self, ++ run_id: RunId, ++ cursor: WorkerControlCursor, ++ ) -> BoxFuture<'_, Result>; ++ ++ fn cleanup_run(&self, run_id: RunId) -> BoxFuture<'_, ()>; ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ ++ #[test] ++ fn message_id_equality_and_debug_are_opaque() { ++ let first = WorkerControlMessageId::from("local:42"); ++ let same = WorkerControlMessageId::from("local:42"); ++ let different = WorkerControlMessageId::from("local:43"); ++ ++ assert_eq!(first, same); ++ assert_ne!(first, different); ++ assert_eq!(format!("{first}"), "local:42"); ++ assert_eq!(format!("{first:?}"), "WorkerControlMessageId(\"local:42\")"); ++ } ++ ++ #[test] ++ fn absent_after_query_parses_as_start_cursor() { ++ assert_eq!( ++ WorkerControlCursor::from_after_query(None).unwrap(), ++ WorkerControlCursor::Start ++ ); ++ } ++ ++ #[test] ++ fn present_after_query_parses_as_after_cursor() { ++ assert_eq!( ++ WorkerControlCursor::from_after_query(Some("local:42")).unwrap(), ++ WorkerControlCursor::After(WorkerControlMessageId::from("local:42")) ++ ); ++ } ++} +diff --git a/lib/crates/fabro-server/src/worker_control/local.rs b/lib/crates/fabro-server/src/worker_control/local.rs +new file mode 100644 +index 000000000..219ff18ec +--- /dev/null ++++ b/lib/crates/fabro-server/src/worker_control/local.rs +@@ -0,0 +1,458 @@ ++use std::collections::{HashMap, VecDeque}; ++use std::sync::atomic::{AtomicU64, Ordering}; ++use std::sync::{Arc, Mutex}; ++ ++use fabro_interview::WorkerControlEnvelope; ++use fabro_types::RunId; ++use futures_util::FutureExt; ++use futures_util::future::BoxFuture; ++use tokio::sync::{Notify, mpsc}; ++ ++use super::{ ++ WorkerControlBus, WorkerControlBusError, WorkerControlCursor, WorkerControlDelivery, ++ WorkerControlMessageId, WorkerControlReceiver, ++}; ++ ++pub(crate) const LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN: usize = 1024; ++const LOCAL_WORKER_CONTROL_SUBSCRIBER_BUFFER: usize = 64; ++ ++#[derive(Default)] ++pub(crate) struct LocalWorkerControlBus { ++ streams: Arc>>, ++ next_sequence: Arc, ++} ++ ++struct LocalRunControlStream { ++ messages: VecDeque, ++ notify: Arc, ++ has_trimmed: bool, ++} ++ ++#[derive(Clone)] ++struct LocalMessage { ++ sequence: u64, ++ delivery: WorkerControlDelivery, ++} ++ ++impl LocalWorkerControlBus { ++ #[must_use] ++ pub(crate) fn new() -> Self { ++ Self { ++ streams: Arc::new(Mutex::new(HashMap::new())), ++ next_sequence: Arc::new(AtomicU64::new(1)), ++ } ++ } ++ ++ fn next_message_id(&self) -> (u64, WorkerControlMessageId) { ++ let sequence = self.next_sequence.fetch_add(1, Ordering::Relaxed); ++ ( ++ sequence, ++ WorkerControlMessageId::new(format!("local:{sequence}")), ++ ) ++ } ++ ++ fn subscribe_inner( ++ &self, ++ run_id: RunId, ++ cursor: &WorkerControlCursor, ++ ) -> Result { ++ let (next_sequence, notify) = { ++ let mut streams = self ++ .streams ++ .lock() ++ .expect("worker control streams poisoned"); ++ let stream = match cursor { ++ WorkerControlCursor::Start => streams ++ .entry(run_id) ++ .or_insert_with(LocalRunControlStream::new), ++ WorkerControlCursor::After(id) => streams.get_mut(&run_id).ok_or_else(|| { ++ WorkerControlBusError::invalid_cursor( ++ id.as_str(), ++ "message id is not retained for this run", ++ ) ++ })?, ++ }; ++ let next_sequence = stream.next_sequence_for_cursor(cursor)?; ++ (next_sequence, Arc::clone(&stream.notify)) ++ }; ++ ++ let (tx, rx) = mpsc::channel(LOCAL_WORKER_CONTROL_SUBSCRIBER_BUFFER); ++ let streams = Arc::clone(&self.streams); ++ tokio::spawn(async move { ++ local_subscription_task(streams, run_id, notify, next_sequence, tx).await; ++ }); ++ Ok(rx) ++ } ++ ++ #[cfg(test)] ++ pub(crate) fn retained_len(&self, run_id: &RunId) -> usize { ++ let streams = self ++ .streams ++ .lock() ++ .expect("worker control streams poisoned"); ++ streams ++ .get(run_id) ++ .map_or(0, |stream| stream.messages.len()) ++ } ++} ++ ++impl LocalRunControlStream { ++ fn new() -> Self { ++ Self { ++ messages: VecDeque::new(), ++ notify: Arc::new(Notify::new()), ++ has_trimmed: false, ++ } ++ } ++ ++ fn next_sequence_for_cursor( ++ &self, ++ cursor: &WorkerControlCursor, ++ ) -> Result, WorkerControlBusError> { ++ match cursor { ++ WorkerControlCursor::Start if self.has_trimmed => Err( ++ WorkerControlBusError::invalid_cursor("start", "retained local stream is trimmed"), ++ ), ++ WorkerControlCursor::Start => Ok(self.messages.front().map(|message| message.sequence)), ++ WorkerControlCursor::After(id) => { ++ let requested_sequence = parse_local_sequence(id)?; ++ let Some(position) = self ++ .messages ++ .iter() ++ .position(|message| message.delivery.id == *id) ++ else { ++ return Err(WorkerControlBusError::invalid_cursor( ++ id.as_str(), ++ "message id is not retained for this run", ++ )); ++ }; ++ Ok(self ++ .messages ++ .get(position + 1) ++ .map(|message| message.sequence) ++ .or(Some(requested_sequence.saturating_add(1)))) ++ } ++ } ++ } ++ ++ fn trim_retained(&mut self) { ++ while self.messages.len() > LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN { ++ self.messages.pop_front(); ++ self.has_trimmed = true; ++ } ++ } ++} ++ ++fn parse_local_sequence(id: &WorkerControlMessageId) -> Result { ++ let Some(raw) = id.as_str().strip_prefix("local:") else { ++ return Err(WorkerControlBusError::invalid_cursor( ++ id.as_str(), ++ "local backend only understands local message ids", ++ )); ++ }; ++ raw.parse::().map_err(|_| { ++ WorkerControlBusError::invalid_cursor(id.as_str(), "local message id sequence is invalid") ++ }) ++} ++ ++async fn local_subscription_task( ++ streams: Arc>>, ++ run_id: RunId, ++ notify: Arc, ++ mut next_sequence: Option, ++ tx: mpsc::Sender>, ++) { ++ loop { ++ // Register interest *before* inspecting the stream so that a publish ++ // racing with this read does not cause a lost wakeup. `notify_waiters` ++ // does not leave a permit for future `notified()` calls. ++ let notified = notify.notified(); ++ tokio::pin!(notified); ++ notified.as_mut().enable(); ++ ++ let collected = { ++ let streams_guard = streams.lock().expect("worker control streams poisoned"); ++ match streams_guard.get(&run_id) { ++ None => None, ++ Some(stream) => { ++ // A `Start` subscriber that joined before any publish lazily ++ // adopts the first retained message as its cursor. ++ if next_sequence.is_none() && stream.has_trimmed { ++ Some(Err(WorkerControlBusError::invalid_cursor( ++ "start", ++ "retained local stream is trimmed", ++ ))) ++ } else { ++ if next_sequence.is_none() { ++ next_sequence = stream.messages.front().map(|message| message.sequence); ++ } ++ match next_sequence { ++ None => Some(Ok(Vec::new())), ++ Some(next) => Some(collect_messages_from(&stream.messages, next)), ++ } ++ } ++ } ++ } ++ }; ++ ++ let messages = match collected { ++ None => return, ++ Some(Err(err)) => { ++ let _ = tx.send(Err(err)).await; ++ return; ++ } ++ Some(Ok(messages)) => messages, ++ }; ++ ++ if messages.is_empty() { ++ notified.await; ++ continue; ++ } ++ ++ for message in messages { ++ next_sequence = Some(message.sequence.saturating_add(1)); ++ if tx.send(Ok(message.delivery)).await.is_err() { ++ return; ++ } ++ } ++ } ++} ++ ++/// Returns the retained messages with `sequence >= next`. Cheaper than scanning ++/// the whole deque: `partition_point` is O(log N) and we only clone the tail. ++fn collect_messages_from( ++ messages: &VecDeque, ++ next: u64, ++) -> Result, WorkerControlBusError> { ++ let Some(first_sequence) = messages.front().map(|message| message.sequence) else { ++ return Ok(Vec::new()); ++ }; ++ if next < first_sequence { ++ return Err(WorkerControlBusError::invalid_cursor( ++ format!("local:{next}"), ++ "subscriber fell behind retained local messages", ++ )); ++ } ++ let start = messages.partition_point(|message| message.sequence < next); ++ Ok(messages.iter().skip(start).cloned().collect()) ++} ++ ++impl WorkerControlBus for LocalWorkerControlBus { ++ fn publish( ++ &self, ++ run_id: RunId, ++ envelope: WorkerControlEnvelope, ++ ) -> BoxFuture<'_, Result> { ++ async move { ++ let (sequence, id) = self.next_message_id(); ++ let delivery = WorkerControlDelivery { ++ id: id.clone(), ++ envelope, ++ }; ++ let notify = { ++ let mut streams = self ++ .streams ++ .lock() ++ .expect("worker control streams poisoned"); ++ let stream = streams ++ .entry(run_id) ++ .or_insert_with(LocalRunControlStream::new); ++ stream ++ .messages ++ .push_back(LocalMessage { sequence, delivery }); ++ stream.trim_retained(); ++ Arc::clone(&stream.notify) ++ }; ++ notify.notify_waiters(); ++ Ok(id) ++ } ++ .boxed() ++ } ++ ++ fn subscribe( ++ &self, ++ run_id: RunId, ++ cursor: WorkerControlCursor, ++ ) -> BoxFuture<'_, Result> { ++ async move { self.subscribe_inner(run_id, &cursor) }.boxed() ++ } ++ ++ fn cleanup_run(&self, run_id: RunId) -> BoxFuture<'_, ()> { ++ async move { ++ let notify = { ++ let mut streams = self ++ .streams ++ .lock() ++ .expect("worker control streams poisoned"); ++ streams.remove(&run_id).map(|stream| stream.notify) ++ }; ++ if let Some(notify) = notify { ++ notify.notify_waiters(); ++ } ++ } ++ .boxed() ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use std::time::Duration; ++ ++ use fabro_interview::WorkerControlMessage; ++ use fabro_types::fixtures; ++ ++ use super::*; ++ ++ async fn recv_delivery(receiver: &mut WorkerControlReceiver) -> WorkerControlDelivery { ++ tokio::time::timeout(Duration::from_secs(1), receiver.recv()) ++ .await ++ .expect("delivery should arrive") ++ .expect("subscription should remain open") ++ .expect("delivery should be ok") ++ } ++ ++ #[tokio::test] ++ async fn messages_publish_in_order() { ++ let bus = LocalWorkerControlBus::new(); ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run()) ++ .await ++ .unwrap(); ++ ++ let mut receiver = bus ++ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start) ++ .await ++ .unwrap(); ++ ++ assert!(matches!( ++ recv_delivery(&mut receiver).await.envelope.message, ++ WorkerControlMessage::RunCancel ++ )); ++ assert!(matches!( ++ recv_delivery(&mut receiver).await.envelope.message, ++ WorkerControlMessage::RunPause ++ )); ++ } ++ ++ #[tokio::test] ++ async fn active_subscriber_receives_message_published_after_subscription() { ++ let bus = LocalWorkerControlBus::new(); ++ let mut receiver = bus ++ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start) ++ .await ++ .unwrap(); ++ ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ ++ assert!(matches!( ++ recv_delivery(&mut receiver).await.envelope.message, ++ WorkerControlMessage::RunCancel ++ )); ++ } ++ ++ #[tokio::test] ++ async fn messages_published_before_subscription_replay_from_start() { ++ let bus = LocalWorkerControlBus::new(); ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ ++ let mut receiver = bus ++ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start) ++ .await ++ .unwrap(); ++ ++ assert!(matches!( ++ recv_delivery(&mut receiver).await.envelope.message, ++ WorkerControlMessage::RunCancel ++ )); ++ } ++ ++ #[tokio::test] ++ async fn after_cursor_receives_only_later_messages() { ++ let bus = LocalWorkerControlBus::new(); ++ let first_id = bus ++ .publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run()) ++ .await ++ .unwrap(); ++ ++ let mut receiver = bus ++ .subscribe( ++ fixtures::RUN_1, ++ WorkerControlCursor::After(first_id.clone()), ++ ) ++ .await ++ .unwrap(); ++ ++ assert!(matches!( ++ recv_delivery(&mut receiver).await.envelope.message, ++ WorkerControlMessage::RunPause ++ )); ++ assert!(receiver.try_recv().is_err()); ++ } ++ ++ #[tokio::test] ++ async fn trimming_bounds_retained_messages_and_invalidates_old_cursor() { ++ let bus = LocalWorkerControlBus::new(); ++ let old_id = bus ++ .publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ for _ in 0..LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN { ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::pause_run()) ++ .await ++ .unwrap(); ++ } ++ ++ assert_eq!( ++ bus.retained_len(&fixtures::RUN_1), ++ LOCAL_WORKER_CONTROL_RETAINED_MESSAGES_PER_RUN ++ ); ++ let err = bus ++ .subscribe(fixtures::RUN_1, WorkerControlCursor::After(old_id)) ++ .await ++ .unwrap_err(); ++ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. })); ++ ++ let err = bus ++ .subscribe(fixtures::RUN_1, WorkerControlCursor::Start) ++ .await ++ .unwrap_err(); ++ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. })); ++ } ++ ++ #[tokio::test] ++ async fn after_cursor_for_unknown_run_does_not_create_stream() { ++ let bus = LocalWorkerControlBus::new(); ++ let err = bus ++ .subscribe( ++ fixtures::RUN_1, ++ WorkerControlCursor::After(WorkerControlMessageId::new("local:1")), ++ ) ++ .await ++ .unwrap_err(); ++ ++ assert!(matches!(err, WorkerControlBusError::InvalidCursor { .. })); ++ assert_eq!(bus.retained_len(&fixtures::RUN_1), 0); ++ } ++ ++ #[tokio::test] ++ async fn cleanup_removes_retained_messages() { ++ let bus = LocalWorkerControlBus::new(); ++ bus.publish(fixtures::RUN_1, WorkerControlEnvelope::cancel_run()) ++ .await ++ .unwrap(); ++ assert_eq!(bus.retained_len(&fixtures::RUN_1), 1); ++ ++ bus.cleanup_run(fixtures::RUN_1).await; ++ ++ assert_eq!(bus.retained_len(&fixtures::RUN_1), 0); ++ } ++} +diff --git a/lib/crates/fabro-server/src/worker_control/mod.rs b/lib/crates/fabro-server/src/worker_control/mod.rs +new file mode 100644 +index 000000000..6b6f70896 +--- /dev/null ++++ b/lib/crates/fabro-server/src/worker_control/mod.rs +@@ -0,0 +1,8 @@ ++mod bus; ++mod local; ++ ++pub(crate) use bus::{ ++ WorkerControlBus, WorkerControlBusError, WorkerControlCursor, WorkerControlDelivery, ++ WorkerControlMessageId, WorkerControlReceiver, ++}; ++pub(crate) use local::LocalWorkerControlBus; +diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs +index 9d3f08d1c..23c90f3ff 100644 +--- a/lib/crates/fabro-workflow/src/operations/start.rs ++++ b/lib/crates/fabro-workflow/src/operations/start.rs +@@ -26,6 +26,7 @@ use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind}; + use fabro_vault::Vault; + use tokio::runtime::Handle; + use tokio::sync::RwLock as AsyncRwLock; ++use tokio::time; + use tokio_util::sync::CancellationToken; + + use crate::artifact_upload::ArtifactSink; +@@ -987,6 +988,24 @@ impl Drop for DetachedRunBootstrapGuard { + + const POSTRUN_INTERRUPTED_MESSAGE: &str = "Run interrupted before post-run finalization completed."; + const POSTRUN_CANCELLED_MESSAGE: &str = "Run cancelled before post-run finalization completed."; ++const DETACHED_COMPLETION_GUARD_TERMINAL_GRACE: Duration = Duration::from_millis(25); ++ ++async fn run_store_reaches_terminal(run_store: &RunStoreHandle, timeout: Duration) -> bool { ++ let start = Instant::now(); ++ loop { ++ if run_store ++ .state() ++ .await ++ .is_ok_and(|state| state.status.is_terminal()) ++ { ++ return true; ++ } ++ if start.elapsed() >= timeout { ++ return false; ++ } ++ time::sleep(Duration::from_millis(10)).await; ++ } ++} + + struct DetachedRunCompletionGuard { + event_sink: RunEventSink, +@@ -1044,6 +1063,11 @@ impl Drop for DetachedRunCompletionGuard { + let run_store = self.run_store.clone(); + if let Ok(handle) = Handle::try_current() { + handle.spawn(async move { ++ if run_store_reaches_terminal(&run_store, DETACHED_COMPLETION_GUARD_TERMINAL_GRACE) ++ .await ++ { ++ return; ++ } + emit_workflow_run_failed( + run_id, + &run_store, diff --git a/stages/008-verify@1/output.log b/stages/008-verify@1/output.log new file mode 100644 index 000000000..82e2208c2 --- /dev/null +++ b/stages/008-verify@1/output.log @@ -0,0 +1 @@ +blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da \ No newline at end of file diff --git a/stages/008-verify@1/script_timing.json b/stages/008-verify@1/script_timing.json new file mode 100644 index 000000000..d612b392a --- /dev/null +++ b/stages/008-verify@1/script_timing.json @@ -0,0 +1,8 @@ +{ + "output": "blob://sha256/a197e7ab024f17d58dff943b5a0eeaa545ba6b50528d00269ea25c5ff79810da", + "exit_code": 0, + "duration_ms": 544732, + "termination": "exited", + "output_bytes": 213463, + "live_streaming": true +} \ No newline at end of file diff --git a/stages/008-verify@1/status.json b/stages/008-verify@1/status.json new file mode 100644 index 000000000..52fd7c6aa --- /dev/null +++ b/stages/008-verify@1/status.json @@ -0,0 +1,6 @@ +{ + "outcome": "succeeded", + "notes": "Script completed: git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\\bActorRef\\b|\\bActorKind\\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\\s*==\\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", + "failure_reason": null, + "timestamp": "2026-05-28T01:08:19.664723Z" +} \ No newline at end of file diff --git a/stages/009-exit@1/status.json b/stages/009-exit@1/status.json new file mode 100644 index 000000000..30bfadfc9 --- /dev/null +++ b/stages/009-exit@1/status.json @@ -0,0 +1,6 @@ +{ + "outcome": "succeeded", + "notes": null, + "failure_reason": null, + "timestamp": "2026-05-28T01:08:23.965487Z" +} \ No newline at end of file