mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-09 03:20:56 +00:00
parent
dea11c7ff8
commit
12280825a2
5 changed files with 1642 additions and 42 deletions
599
run.json
599
run.json
File diff suppressed because one or more lines are too long
722
stages/006-simplify_opus@1/diff.patch
Normal file
722
stages/006-simplify_opus@1/diff.patch
Normal file
|
|
@ -0,0 +1,722 @@
|
|||
diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs
|
||||
index d571f1216..17c96751d 100644
|
||||
--- a/lib/crates/fabro-cli/src/commands/run/runner.rs
|
||||
+++ b/lib/crates/fabro-cli/src/commands/run/runner.rs
|
||||
@@ -1,4 +1,3 @@
|
||||
-use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -10,7 +9,9 @@ use fabro_client::ServerTarget;
|
||||
use fabro_config::user::active_settings_path;
|
||||
use fabro_config::{ServerSettingsBuilder, Storage, load_llm_catalog_settings};
|
||||
use fabro_interview::{
|
||||
- AnswerSubmission, ControlInterviewer, WorkerControlDeliveryFrame, WorkerControlEnvelope,
|
||||
+ AnswerSubmission, ControlInterviewer, WORKER_CONTROL_INVALID_CURSOR_REASON,
|
||||
+ WORKER_CONTROL_PONG_TIMEOUT_REASON, WORKER_CONTROL_WS_LIVENESS_TIMEOUT,
|
||||
+ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlDeliveryFrame, WorkerControlEnvelope,
|
||||
WorkerControlMessage,
|
||||
};
|
||||
use fabro_model::Catalog;
|
||||
@@ -288,8 +289,6 @@ fn load_worker_vault(storage_dir: Option<&Path>) -> Result<Option<Arc<AsyncRwLoc
|
||||
|
||||
const WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF: Duration = Duration::from_millis(100);
|
||||
const WORKER_CONTROL_RECONNECT_MAX_BACKOFF: Duration = Duration::from_secs(5);
|
||||
-const WORKER_CONTROL_WS_PING_INTERVAL: Duration = Duration::from_secs(15);
|
||||
-const WORKER_CONTROL_WS_LIVENESS_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
|
||||
struct WorkerControlManagerHandle {
|
||||
first_connection: Option<oneshot::Receiver<Result<()>>>,
|
||||
@@ -440,16 +439,14 @@ async fn run_worker_control_manager(
|
||||
let mut first_tx = Some(first_tx);
|
||||
let mut fatal_tx = Some(fatal_tx);
|
||||
let mut backoff = WORKER_CONTROL_RECONNECT_INITIAL_BACKOFF;
|
||||
- let mut applied_ids = HashSet::new();
|
||||
- let last_applied_id = Arc::new(Mutex::new(None::<String>));
|
||||
+ let mut last_applied_id: Option<String> = None;
|
||||
|
||||
while !done.is_cancelled() {
|
||||
- let after = last_applied_id.lock().await.clone();
|
||||
let request = match build_worker_control_stream_request(
|
||||
&target,
|
||||
&run_id,
|
||||
&worker_token,
|
||||
- after.as_deref(),
|
||||
+ last_applied_id.as_deref(),
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
@@ -477,8 +474,7 @@ async fn run_worker_control_manager(
|
||||
&cancel_token,
|
||||
&steering_hub,
|
||||
&run_control,
|
||||
- &mut applied_ids,
|
||||
- &last_applied_id,
|
||||
+ &mut last_applied_id,
|
||||
&done,
|
||||
)
|
||||
.await
|
||||
@@ -649,18 +645,19 @@ async fn handle_worker_control_socket(
|
||||
cancel_token: &CancellationToken,
|
||||
steering_hub: &fabro_workflow::SteeringHub,
|
||||
run_control: &RunControlState,
|
||||
- applied_ids: &mut HashSet<String>,
|
||||
- last_applied_id: &Arc<Mutex<Option<String>>>,
|
||||
+ last_applied_id: &mut Option<String>,
|
||||
done: &CancellationToken,
|
||||
) -> Result<(), WorkerControlConnectError> {
|
||||
let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL);
|
||||
ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
let mut last_liveness = Instant::now();
|
||||
+ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);
|
||||
+ tokio::pin!(liveness_timeout);
|
||||
|
||||
loop {
|
||||
- let liveness_timeout =
|
||||
- time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);
|
||||
- tokio::pin!(liveness_timeout);
|
||||
+ liveness_timeout
|
||||
+ .as_mut()
|
||||
+ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);
|
||||
|
||||
tokio::select! {
|
||||
() = done.cancelled() => return Ok(()),
|
||||
@@ -674,7 +671,7 @@ async fn handle_worker_control_socket(
|
||||
let _ = socket
|
||||
.send(WebSocketMessage::Close(Some(protocol::CloseFrame {
|
||||
code: CloseCode::Away,
|
||||
- reason: "pong_timeout".into(),
|
||||
+ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(),
|
||||
})))
|
||||
.await;
|
||||
return Err(WorkerControlConnectError::Other(anyhow!(
|
||||
@@ -695,7 +692,6 @@ async fn handle_worker_control_socket(
|
||||
cancel_token,
|
||||
steering_hub,
|
||||
run_control,
|
||||
- applied_ids,
|
||||
last_applied_id,
|
||||
frame,
|
||||
)
|
||||
@@ -712,10 +708,9 @@ async fn handle_worker_control_socket(
|
||||
last_liveness = Instant::now();
|
||||
}
|
||||
Ok(WebSocketMessage::Close(frame)) => {
|
||||
- if frame
|
||||
- .as_ref()
|
||||
- .is_some_and(|frame| frame.reason.as_str() == "invalid_cursor")
|
||||
- {
|
||||
+ if frame.as_ref().is_some_and(|frame| {
|
||||
+ frame.reason.as_str() == WORKER_CONTROL_INVALID_CURSOR_REASON
|
||||
+ }) {
|
||||
return Err(WorkerControlConnectError::InvalidCursor);
|
||||
}
|
||||
return Ok(());
|
||||
@@ -730,50 +725,21 @@ async fn handle_worker_control_socket(
|
||||
}
|
||||
}
|
||||
|
||||
-#[cfg(test)]
|
||||
-fn parse_worker_control_line(line: &str) -> Option<WorkerControlEnvelope> {
|
||||
- if line.trim().is_empty() {
|
||||
- return None;
|
||||
- }
|
||||
-
|
||||
- serde_json::from_str::<WorkerControlEnvelope>(line).ok()
|
||||
-}
|
||||
-
|
||||
-#[cfg(test)]
|
||||
-async fn apply_worker_control_line(
|
||||
- interviewer: &ControlInterviewer,
|
||||
- cancel_token: &CancellationToken,
|
||||
- steering_hub: &fabro_workflow::SteeringHub,
|
||||
- run_control: &RunControlState,
|
||||
- line: &str,
|
||||
-) {
|
||||
- let Some(message) = parse_worker_control_line(line) else {
|
||||
- return;
|
||||
- };
|
||||
- apply_worker_control_message(
|
||||
- interviewer,
|
||||
- cancel_token,
|
||||
- steering_hub,
|
||||
- run_control,
|
||||
- message,
|
||||
- )
|
||||
- .await;
|
||||
-}
|
||||
-
|
||||
async fn apply_worker_control_delivery_frame(
|
||||
interviewer: &ControlInterviewer,
|
||||
cancel_token: &CancellationToken,
|
||||
steering_hub: &fabro_workflow::SteeringHub,
|
||||
run_control: &RunControlState,
|
||||
- applied_ids: &mut HashSet<String>,
|
||||
- last_applied_id: &Arc<Mutex<Option<String>>>,
|
||||
+ last_applied_id: &mut Option<String>,
|
||||
frame: WorkerControlDeliveryFrame,
|
||||
) -> bool {
|
||||
- if !applied_ids.insert(frame.id.clone()) {
|
||||
+ // Duplicate ids cannot reach us under normal operation: the server replays
|
||||
+ // strictly after `last_applied_id`. Guard against a server-side bug by
|
||||
+ // ignoring any frame whose id is not strictly newer than what we last
|
||||
+ // applied.
|
||||
+ if last_applied_id.as_deref() == Some(frame.id.as_str()) {
|
||||
return false;
|
||||
}
|
||||
-
|
||||
- let id = frame.id;
|
||||
apply_worker_control_message(
|
||||
interviewer,
|
||||
cancel_token,
|
||||
@@ -782,7 +748,7 @@ async fn apply_worker_control_delivery_frame(
|
||||
frame.envelope,
|
||||
)
|
||||
.await;
|
||||
- *last_applied_id.lock().await = Some(id);
|
||||
+ *last_applied_id = Some(frame.id);
|
||||
true
|
||||
}
|
||||
|
||||
@@ -1211,7 +1177,6 @@ fn install_signal_handlers(
|
||||
reason = "This test module prefers explicit type paths over extra imports."
|
||||
)]
|
||||
mod tests {
|
||||
- use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -1238,11 +1203,11 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
WorkerControlConnectError, WorkerControlSocket, WorkerTitlePhase,
|
||||
- apply_worker_control_delivery_frame, apply_worker_control_line,
|
||||
+ apply_worker_control_delivery_frame, apply_worker_control_message,
|
||||
build_worker_control_stream_request, connect_worker_control_stream,
|
||||
handle_worker_control_socket, initial_worker_title_phase, load_worker_vault,
|
||||
- next_worker_control_reconnect_backoff, parse_worker_control_line, stamp_system_worker,
|
||||
- worker_title, worker_title_phase_for_event,
|
||||
+ next_worker_control_reconnect_backoff, stamp_system_worker, worker_title,
|
||||
+ worker_title_phase_for_event,
|
||||
};
|
||||
use crate::args::RunWorkerMode;
|
||||
|
||||
@@ -1483,7 +1448,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
- async fn worker_control_line_routes_answer_by_question_id() {
|
||||
+ async fn worker_control_routes_answer_by_question_id() {
|
||||
let interviewer = Arc::new(ControlInterviewer::new());
|
||||
let cancel_token = CancellationToken::new();
|
||||
let run_control = RunControlState::new();
|
||||
@@ -1493,12 +1458,18 @@ mod tests {
|
||||
let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await });
|
||||
|
||||
let hub = test_steering_hub();
|
||||
- apply_worker_control_line(
|
||||
+ apply_worker_control_message(
|
||||
&interviewer,
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- r#"{"v":1,"type":"interview.answer","qid":"q-1","answer":{"kind":"yes"},"actor":{"kind":"system","system_kind":"engine"}}"#,
|
||||
+ WorkerControlEnvelope::interview_answer(
|
||||
+ "q-1",
|
||||
+ fabro_interview::AnswerSubmission::system(
|
||||
+ fabro_interview::Answer::yes(),
|
||||
+ fabro_types::SystemActorKind::Engine,
|
||||
+ ),
|
||||
+ ),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1508,7 +1479,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
- async fn worker_control_line_cancel_sets_cancel_token_and_interrupts_pending_interviews() {
|
||||
+ async fn worker_control_cancel_sets_cancel_token_and_interrupts_pending_interviews() {
|
||||
let interviewer = Arc::new(ControlInterviewer::new());
|
||||
let cancel_token = CancellationToken::new();
|
||||
let run_control = RunControlState::new();
|
||||
@@ -1519,12 +1490,12 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let hub = test_steering_hub();
|
||||
- apply_worker_control_line(
|
||||
+ apply_worker_control_message(
|
||||
&interviewer,
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- r#"{"v":1,"type":"run.cancel"}"#,
|
||||
+ WorkerControlEnvelope::cancel_run(),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1534,28 +1505,28 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
- async fn worker_control_line_pause_and_unpause_route_to_run_control() {
|
||||
+ async fn worker_control_pause_and_unpause_route_to_run_control() {
|
||||
let interviewer = Arc::new(ControlInterviewer::new());
|
||||
let cancel_token = CancellationToken::new();
|
||||
let run_control = RunControlState::new();
|
||||
let hub = test_steering_hub();
|
||||
|
||||
- apply_worker_control_line(
|
||||
+ apply_worker_control_message(
|
||||
&interviewer,
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- r#"{"v":1,"type":"run.pause"}"#,
|
||||
+ WorkerControlEnvelope::pause_run(),
|
||||
)
|
||||
.await;
|
||||
assert!(run_control.pause_requested());
|
||||
|
||||
- apply_worker_control_line(
|
||||
+ apply_worker_control_message(
|
||||
&interviewer,
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- r#"{"v":1,"type":"run.unpause"}"#,
|
||||
+ WorkerControlEnvelope::unpause_run(),
|
||||
)
|
||||
.await;
|
||||
assert!(!run_control.pause_requested());
|
||||
@@ -1567,8 +1538,7 @@ mod tests {
|
||||
let cancel_token = CancellationToken::new();
|
||||
let run_control = RunControlState::new();
|
||||
let hub = test_steering_hub();
|
||||
- let mut applied_ids = HashSet::new();
|
||||
- let last_applied_id = Arc::new(tokio::sync::Mutex::new(None));
|
||||
+ let mut last_applied_id: Option<String> = None;
|
||||
let frame = fabro_interview::WorkerControlDeliveryFrame {
|
||||
id: "local:1".to_string(),
|
||||
envelope: WorkerControlEnvelope::pause_run(),
|
||||
@@ -1580,8 +1550,7 @@ mod tests {
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- &mut applied_ids,
|
||||
- &last_applied_id,
|
||||
+ &mut last_applied_id,
|
||||
frame.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1592,25 +1561,13 @@ mod tests {
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- &mut applied_ids,
|
||||
- &last_applied_id,
|
||||
+ &mut last_applied_id,
|
||||
frame,
|
||||
)
|
||||
.await
|
||||
);
|
||||
|
||||
- assert_eq!(*last_applied_id.lock().await, Some("local:1".to_string()));
|
||||
- assert_eq!(applied_ids.len(), 1);
|
||||
- }
|
||||
-
|
||||
- #[test]
|
||||
- fn worker_control_line_parser_ignores_empty_and_invalid_lines() {
|
||||
- assert!(parse_worker_control_line("").is_none());
|
||||
- assert!(parse_worker_control_line("not json").is_none());
|
||||
- assert_eq!(
|
||||
- parse_worker_control_line(r#"{"v":1,"type":"run.cancel"}"#).unwrap(),
|
||||
- WorkerControlEnvelope::cancel_run()
|
||||
- );
|
||||
+ assert_eq!(last_applied_id, Some("local:1".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1691,8 +1648,7 @@ mod tests {
|
||||
let cancel_token = CancellationToken::new();
|
||||
let hub = test_steering_hub();
|
||||
let run_control = RunControlState::new();
|
||||
- let mut applied_ids = HashSet::new();
|
||||
- let last_applied_id = Arc::new(tokio::sync::Mutex::new(None));
|
||||
+ let mut last_applied_id: Option<String> = None;
|
||||
let done = CancellationToken::new();
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
@@ -1702,8 +1658,7 @@ mod tests {
|
||||
&cancel_token,
|
||||
&hub,
|
||||
&run_control,
|
||||
- &mut applied_ids,
|
||||
- &last_applied_id,
|
||||
+ &mut last_applied_id,
|
||||
&done,
|
||||
)
|
||||
.await
|
||||
diff --git a/lib/crates/fabro-interview/src/control_protocol.rs b/lib/crates/fabro-interview/src/control_protocol.rs
|
||||
index f9126bcec..f76b61121 100644
|
||||
--- a/lib/crates/fabro-interview/src/control_protocol.rs
|
||||
+++ b/lib/crates/fabro-interview/src/control_protocol.rs
|
||||
@@ -1,3 +1,5 @@
|
||||
+use std::time::Duration;
|
||||
+
|
||||
use fabro_types::{PairId, PairMessageId, PairTarget, Principal, RunId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -5,6 +7,25 @@ use crate::{Answer, AnswerSubmission, AnswerValue};
|
||||
|
||||
pub const WORKER_CONTROL_PROTOCOL_VERSION: u8 = 1;
|
||||
|
||||
+/// Interval between worker-control WebSocket ping frames.
|
||||
+///
|
||||
+/// Server and worker both initiate pings at this cadence; either side that
|
||||
+/// fails to observe inbound traffic for [`WORKER_CONTROL_WS_LIVENESS_TIMEOUT`]
|
||||
+/// closes the WebSocket.
|
||||
+pub const WORKER_CONTROL_WS_PING_INTERVAL: Duration = Duration::from_secs(15);
|
||||
+
|
||||
+/// Maximum quiet time allowed on a worker-control WebSocket before either side
|
||||
+/// declares the connection dead.
|
||||
+pub const WORKER_CONTROL_WS_LIVENESS_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
+
|
||||
+/// WebSocket close-frame reason used when the server can no longer prove
|
||||
+/// replay correctness for the requested cursor. Workers must treat this as
|
||||
+/// fatal control-channel loss.
|
||||
+pub const WORKER_CONTROL_INVALID_CURSOR_REASON: &str = "invalid_cursor";
|
||||
+
|
||||
+/// WebSocket close-frame reason used when the ping/pong watchdog fires.
|
||||
+pub const WORKER_CONTROL_PONG_TIMEOUT_REASON: &str = "pong_timeout";
|
||||
+
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerControlEnvelope {
|
||||
pub v: u8,
|
||||
diff --git a/lib/crates/fabro-interview/src/lib.rs b/lib/crates/fabro-interview/src/lib.rs
|
||||
index 26708cb0f..6d8c414a1 100644
|
||||
--- a/lib/crates/fabro-interview/src/lib.rs
|
||||
+++ b/lib/crates/fabro-interview/src/lib.rs
|
||||
@@ -222,7 +222,9 @@ pub use callback::CallbackInterviewer;
|
||||
pub use console::ConsoleInterviewer;
|
||||
pub use control::{ControlInterviewer, SubmitError};
|
||||
pub use control_protocol::{
|
||||
- WORKER_CONTROL_PROTOCOL_VERSION, WorkerControlAnswer, WorkerControlDeliveryFrame,
|
||||
+ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON,
|
||||
+ WORKER_CONTROL_PROTOCOL_VERSION, WORKER_CONTROL_WS_LIVENESS_TIMEOUT,
|
||||
+ WORKER_CONTROL_WS_PING_INTERVAL, WorkerControlAnswer, WorkerControlDeliveryFrame,
|
||||
WorkerControlEnvelope, WorkerControlMessage,
|
||||
};
|
||||
pub use queue::QueueInterviewer;
|
||||
diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs
|
||||
index 393b066b9..4447efa41 100644
|
||||
--- a/lib/crates/fabro-server/src/server.rs
|
||||
+++ b/lib/crates/fabro-server/src/server.rs
|
||||
@@ -3381,21 +3381,6 @@ async fn append_worker_exit_failure(
|
||||
}
|
||||
}
|
||||
|
||||
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
-enum WorkerCommandStdin {
|
||||
- Null,
|
||||
-}
|
||||
-
|
||||
-impl WorkerCommandStdin {
|
||||
- fn stdio(self) -> Stdio {
|
||||
- match self {
|
||||
- Self::Null => Stdio::null(),
|
||||
- }
|
||||
- }
|
||||
-}
|
||||
-
|
||||
-const WORKER_COMMAND_STDIN: WorkerCommandStdin = WorkerCommandStdin::Null;
|
||||
-
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Worker subprocess startup resolves Cargo's test binary env override when present."
|
||||
@@ -3442,7 +3427,7 @@ fn worker_command(
|
||||
.arg(run_id.to_string())
|
||||
.arg("--mode")
|
||||
.arg(worker_mode_arg(mode))
|
||||
- .stdin(WORKER_COMMAND_STDIN.stdio())
|
||||
+ .stdin(Stdio::null())
|
||||
.stdout(worker_stdout)
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
diff --git a/lib/crates/fabro-server/src/server/handler/worker_control.rs b/lib/crates/fabro-server/src/server/handler/worker_control.rs
|
||||
index 2a26402ad..682107748 100644
|
||||
--- a/lib/crates/fabro-server/src/server/handler/worker_control.rs
|
||||
+++ b/lib/crates/fabro-server/src/server/handler/worker_control.rs
|
||||
@@ -1,8 +1,11 @@
|
||||
use std::sync::Arc;
|
||||
-use std::time::Duration;
|
||||
|
||||
-use axum::extract::ws::{CloseFrame, Message as WsMessage, WebSocket, WebSocketUpgrade};
|
||||
-use fabro_interview::WorkerControlDeliveryFrame;
|
||||
+use axum::extract::ws::{CloseFrame, Message as WsMessage, WebSocket, WebSocketUpgrade, close_code};
|
||||
+use fabro_interview::{
|
||||
+ WORKER_CONTROL_INVALID_CURSOR_REASON, WORKER_CONTROL_PONG_TIMEOUT_REASON,
|
||||
+ WORKER_CONTROL_WS_LIVENESS_TIMEOUT, WORKER_CONTROL_WS_PING_INTERVAL,
|
||||
+ WorkerControlDeliveryFrame,
|
||||
+};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use tokio::time::{self, Instant, MissedTickBehavior};
|
||||
|
||||
@@ -12,9 +15,6 @@ use super::super::{
|
||||
};
|
||||
use crate::worker_control::{WorkerControlBusError, WorkerControlCursor, WorkerControlReceiver};
|
||||
|
||||
-const WORKER_CONTROL_WS_PING_INTERVAL: Duration = Duration::from_secs(15);
|
||||
-const WORKER_CONTROL_WS_LIVENESS_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
-
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct WorkerControlStreamQuery {
|
||||
after: Option<String>,
|
||||
@@ -86,11 +86,13 @@ async fn worker_control_websocket(socket: WebSocket, mut receiver: WorkerControl
|
||||
let mut ping_interval = time::interval(WORKER_CONTROL_WS_PING_INTERVAL);
|
||||
ping_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
let mut last_liveness = Instant::now();
|
||||
+ let liveness_timeout = time::sleep_until(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);
|
||||
+ tokio::pin!(liveness_timeout);
|
||||
|
||||
loop {
|
||||
- let liveness_deadline = last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT;
|
||||
- let liveness_timeout = time::sleep_until(liveness_deadline);
|
||||
- tokio::pin!(liveness_timeout);
|
||||
+ liveness_timeout
|
||||
+ .as_mut()
|
||||
+ .reset(last_liveness + WORKER_CONTROL_WS_LIVENESS_TIMEOUT);
|
||||
|
||||
tokio::select! {
|
||||
delivery = receiver.recv() => {
|
||||
@@ -145,8 +147,8 @@ async fn worker_control_websocket(socket: WebSocket, mut receiver: WorkerControl
|
||||
}
|
||||
() = &mut liveness_timeout => {
|
||||
let _ = sender.send(WsMessage::Close(Some(CloseFrame {
|
||||
- code: 1001,
|
||||
- reason: "pong_timeout".into(),
|
||||
+ code: close_code::AWAY,
|
||||
+ reason: WORKER_CONTROL_PONG_TIMEOUT_REASON.into(),
|
||||
}))).await;
|
||||
return;
|
||||
}
|
||||
@@ -156,7 +158,7 @@ async fn worker_control_websocket(socket: WebSocket, mut receiver: WorkerControl
|
||||
|
||||
fn invalid_cursor_close_message() -> WsMessage {
|
||||
WsMessage::Close(Some(CloseFrame {
|
||||
- code: 1008,
|
||||
- reason: "invalid_cursor".into(),
|
||||
+ code: close_code::POLICY,
|
||||
+ reason: WORKER_CONTROL_INVALID_CURSOR_REASON.into(),
|
||||
}))
|
||||
}
|
||||
diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs
|
||||
index 01560e972..f95273d74 100644
|
||||
--- a/lib/crates/fabro-server/src/server/tests.rs
|
||||
+++ b/lib/crates/fabro-server/src/server/tests.rs
|
||||
@@ -2075,7 +2075,6 @@ fn worker_command_uses_null_stdin_and_token_env() {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
- assert_eq!(WORKER_COMMAND_STDIN, WorkerCommandStdin::Null);
|
||||
assert_worker_command_passes_token_only_by_env(&cmd);
|
||||
}
|
||||
|
||||
@@ -2387,13 +2386,6 @@ methods = ["dev-token"]
|
||||
));
|
||||
}
|
||||
|
||||
-#[test]
|
||||
-fn build_app_state_uses_local_worker_control_bus_by_default() {
|
||||
- let state = test_app_state();
|
||||
-
|
||||
- assert_eq!(state.worker_control_bus.backend_name(), "local");
|
||||
-}
|
||||
-
|
||||
#[test]
|
||||
fn build_app_state_migrates_legacy_vault_file_on_boot() {
|
||||
let vault_path = test_secret_store_path();
|
||||
diff --git a/lib/crates/fabro-server/src/worker_control/bus.rs b/lib/crates/fabro-server/src/worker_control/bus.rs
|
||||
index af2287cab..b6fadedb5 100644
|
||||
--- a/lib/crates/fabro-server/src/worker_control/bus.rs
|
||||
+++ b/lib/crates/fabro-server/src/worker_control/bus.rs
|
||||
@@ -5,7 +5,7 @@ use fabro_types::RunId;
|
||||
use futures_util::future::BoxFuture;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
-#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
+#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct WorkerControlMessageId(String);
|
||||
|
||||
impl WorkerControlMessageId {
|
||||
@@ -119,12 +119,6 @@ pub(crate) trait WorkerControlBus: Send + Sync {
|
||||
) -> BoxFuture<'_, Result<WorkerControlReceiver, WorkerControlBusError>>;
|
||||
|
||||
fn cleanup_run(&self, run_id: RunId) -> BoxFuture<'_, ()>;
|
||||
-
|
||||
- #[allow(
|
||||
- dead_code,
|
||||
- reason = "Used by tests and diagnostics for backend identity."
|
||||
- )]
|
||||
- fn backend_name(&self) -> &'static str;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
diff --git a/lib/crates/fabro-server/src/worker_control/local.rs b/lib/crates/fabro-server/src/worker_control/local.rs
|
||||
index c85b053f0..51ea1421f 100644
|
||||
--- a/lib/crates/fabro-server/src/worker_control/local.rs
|
||||
+++ b/lib/crates/fabro-server/src/worker_control/local.rs
|
||||
@@ -55,7 +55,7 @@ impl LocalWorkerControlBus {
|
||||
run_id: RunId,
|
||||
cursor: &WorkerControlCursor,
|
||||
) -> Result<WorkerControlReceiver, WorkerControlBusError> {
|
||||
- let next_sequence = {
|
||||
+ let (next_sequence, notify) = {
|
||||
let mut streams = self
|
||||
.streams
|
||||
.lock()
|
||||
@@ -63,13 +63,14 @@ impl LocalWorkerControlBus {
|
||||
let stream = streams
|
||||
.entry(run_id)
|
||||
.or_insert_with(LocalRunControlStream::new);
|
||||
- stream.next_sequence_for_cursor(cursor)?
|
||||
+ let next_sequence = stream.next_sequence_for_cursor(cursor)?;
|
||||
+ (next_sequence, Arc::clone(&stream.notify))
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel(LOCAL_WORKER_CONTROL_SUBSCRIBER_BUFFER);
|
||||
let streams = Arc::clone(&self.streams);
|
||||
tokio::spawn(async move {
|
||||
- local_subscription_task(streams, run_id, next_sequence, tx).await;
|
||||
+ local_subscription_task(streams, run_id, notify, next_sequence, tx).await;
|
||||
});
|
||||
Ok(rx)
|
||||
}
|
||||
@@ -143,61 +144,47 @@ fn parse_local_sequence(id: &WorkerControlMessageId) -> Result<u64, WorkerContro
|
||||
async fn local_subscription_task(
|
||||
streams: Arc<Mutex<HashMap<RunId, LocalRunControlStream>>>,
|
||||
run_id: RunId,
|
||||
+ notify: Arc<Notify>,
|
||||
mut next_sequence: Option<u64>,
|
||||
tx: mpsc::Sender<Result<WorkerControlDelivery, WorkerControlBusError>>,
|
||||
) {
|
||||
loop {
|
||||
- let mut invalid_cursor = None;
|
||||
- let (messages, notify) = {
|
||||
+ // Register interest *before* inspecting the stream so that a publish
|
||||
+ // racing with this read does not cause a lost wakeup. `notify_waiters`
|
||||
+ // does not leave a permit for future `notified()` calls.
|
||||
+ let notified = notify.notified();
|
||||
+ tokio::pin!(notified);
|
||||
+ notified.as_mut().enable();
|
||||
+
|
||||
+ let collected = {
|
||||
let streams_guard = streams.lock().expect("worker control streams poisoned");
|
||||
- let Some(stream) = streams_guard.get(&run_id) else {
|
||||
- return;
|
||||
- };
|
||||
- let notify = Arc::clone(&stream.notify);
|
||||
- match next_sequence {
|
||||
- None => (Vec::new(), notify),
|
||||
- Some(next) => {
|
||||
- if let Some(first_sequence) =
|
||||
- stream.messages.front().map(|message| message.sequence)
|
||||
- {
|
||||
- if next < first_sequence {
|
||||
- invalid_cursor = Some(WorkerControlBusError::invalid_cursor(
|
||||
- format!("local:{next}"),
|
||||
- "subscriber fell behind retained local messages",
|
||||
- ));
|
||||
- (Vec::new(), notify)
|
||||
- } else {
|
||||
- let messages = stream
|
||||
- .messages
|
||||
- .iter()
|
||||
- .filter(|message| message.sequence >= next)
|
||||
- .cloned()
|
||||
- .collect::<Vec<_>>();
|
||||
- (messages, notify)
|
||||
- }
|
||||
- } else {
|
||||
- (Vec::new(), notify)
|
||||
+ match streams_guard.get(&run_id) {
|
||||
+ None => None,
|
||||
+ Some(stream) => {
|
||||
+ // A `Start` subscriber that joined before any publish lazily
|
||||
+ // adopts the first retained message as its cursor.
|
||||
+ if next_sequence.is_none() {
|
||||
+ next_sequence = stream.messages.front().map(|message| message.sequence);
|
||||
+ }
|
||||
+ match next_sequence {
|
||||
+ None => Some(Ok(Vec::new())),
|
||||
+ Some(next) => Some(collect_messages_from(&stream.messages, next)),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
- if let Some(err) = invalid_cursor {
|
||||
- let _ = tx.send(Err(err)).await;
|
||||
- return;
|
||||
- }
|
||||
+ let messages = match collected {
|
||||
+ None => return,
|
||||
+ Some(Err(err)) => {
|
||||
+ let _ = tx.send(Err(err)).await;
|
||||
+ return;
|
||||
+ }
|
||||
+ Some(Ok(messages)) => messages,
|
||||
+ };
|
||||
|
||||
if messages.is_empty() {
|
||||
- if next_sequence.is_none() {
|
||||
- let streams_guard = streams.lock().expect("worker control streams poisoned");
|
||||
- next_sequence = streams_guard
|
||||
- .get(&run_id)
|
||||
- .and_then(|stream| stream.messages.front().map(|message| message.sequence));
|
||||
- if next_sequence.is_some() {
|
||||
- continue;
|
||||
- }
|
||||
- }
|
||||
- notify.notified().await;
|
||||
+ notified.await;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -210,6 +197,25 @@ async fn local_subscription_task(
|
||||
}
|
||||
}
|
||||
|
||||
+/// Returns the retained messages with `sequence >= next`. Cheaper than scanning
|
||||
+/// the whole deque: `partition_point` is O(log N) and we only clone the tail.
|
||||
+fn collect_messages_from(
|
||||
+ messages: &VecDeque<LocalMessage>,
|
||||
+ next: u64,
|
||||
+) -> Result<Vec<LocalMessage>, WorkerControlBusError> {
|
||||
+ let Some(first_sequence) = messages.front().map(|message| message.sequence) else {
|
||||
+ return Ok(Vec::new());
|
||||
+ };
|
||||
+ if next < first_sequence {
|
||||
+ return Err(WorkerControlBusError::invalid_cursor(
|
||||
+ format!("local:{next}"),
|
||||
+ "subscriber fell behind retained local messages",
|
||||
+ ));
|
||||
+ }
|
||||
+ let start = messages.partition_point(|message| message.sequence < next);
|
||||
+ Ok(messages.iter().skip(start).cloned().collect())
|
||||
+}
|
||||
+
|
||||
impl WorkerControlBus for LocalWorkerControlBus {
|
||||
fn publish(
|
||||
&self,
|
||||
@@ -265,10 +271,6 @@ impl WorkerControlBus for LocalWorkerControlBus {
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
-
|
||||
- fn backend_name(&self) -> &'static str {
|
||||
- "local"
|
||||
- }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
6
stages/006-simplify_opus@1/status.json
Normal file
6
stages/006-simplify_opus@1/status.json
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"outcome": "succeeded",
|
||||
"notes": "Stage completed: simplify_opus",
|
||||
"failure_reason": null,
|
||||
"timestamp": "2026-05-27T23:21:57.938866Z"
|
||||
}
|
||||
352
stages/007-simplify_gpt@1/prompt.md
Normal file
352
stages/007-simplify_gpt@1/prompt.md
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
Goal: # Worker Control Bus Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace the server-to-worker stdin JSONL control pipe with a backend-agnostic worker control bus, implemented now with a local in-memory bus and delivered to workers over a worker-initiated WebSocket.
|
||||
|
||||
**Architecture:** API handlers publish `WorkerControlEnvelope` messages to a `WorkerControlBus`; the worker WebSocket route subscribes to that bus and forwards ordered delivery frames to the worker. Workers track the last fully applied delivery id and reconnect with `?after=<id>` after any unexpected WebSocket close. The first backend is an in-process `LocalWorkerControlBus` for local and single-node deployments. A Redis Streams backend must fit behind the same trait later, but Redis is explicitly out of scope for this implementation plan.
|
||||
|
||||
**Tech Stack:** Rust, Axum WebSockets, tokio-tungstenite, UnixStream, async-trait or boxed async traits, tokio channels/Notify, worker JWT auth, existing `WorkerControlEnvelope`.
|
||||
|
||||
---
|
||||
|
||||
## Key Decisions
|
||||
|
||||
- WebSocket fully replaces stdin control. Do not keep stdin JSONL as a compatibility path.
|
||||
- The worker protocol stays identical across local, single-node, ECS, and later SaaS deployments.
|
||||
- The server-side delivery backend is the only thing that varies by deployment.
|
||||
- This plan implements only `LocalWorkerControlBus`.
|
||||
- This plan does not add Redis dependencies, Redis configuration, Redis tests, Redis health checks, or Redis runtime behavior.
|
||||
- Redis Streams are covered only as a future backend contract so the local design does not paint us into a corner.
|
||||
- There is no `Latest` cursor. The first worker connection starts at the beginning of the run's retained control stream; reconnects resume after the worker's last fully applied delivery id.
|
||||
- Every WebSocket text frame is a delivery frame with an id and envelope. The worker advances `last_applied_id` only after applying the envelope.
|
||||
- Workers reconnect forever while the local run is not terminal, using backoff from 100ms, doubled after each failure, capped at 5s.
|
||||
- The worker must complete its first control-stream connection before starting or resuming workflow execution. Temporary first-connect failures wait and retry; they do not start a control-disconnected run.
|
||||
- Invalid cursor means the bus can no longer prove replay correctness. The worker treats it as fatal control-channel loss and fails/aborts the run as infrastructure failure, not as user cancellation.
|
||||
- WebSocket liveness is handled at the WebSocket layer with explicit ping/pong and timeout logic. The bus does not know about heartbeats.
|
||||
- ECS task launch, ECS stop/reconciliation, Redis-backed multi-node delivery, and remote hard-kill behavior are follow-up work.
|
||||
|
||||
## Redis Fit Later: Out of Scope Now
|
||||
|
||||
Redis should later implement the same `WorkerControlBus` API introduced here.
|
||||
|
||||
- `publish(run_id, envelope)` maps to `XADD fabro:run:{run_id}:control ...`.
|
||||
- First `subscribe(run_id, Start)` maps to `XREAD BLOCK ... STREAMS fabro:run:{run_id}:control 0-0`.
|
||||
- Reconnect `subscribe(run_id, After(id))` maps to `XREAD BLOCK ... STREAMS fabro:run:{run_id}:control {id}`.
|
||||
- Local message ids use an opaque string format such as `local:1`; Redis message ids can use Redis stream ids such as `1716810000000-0`.
|
||||
- The WebSocket route should not care whether the subscription source is local memory or Redis.
|
||||
- The worker should not care whether the frame came from a local bus or Redis.
|
||||
- Redis trimming/retention, consumer groups, per-tenant key naming, TLS/auth, reconnect-after-redeploy semantics, and SaaS config validation are not part of this plan.
|
||||
|
||||
## Proposed File Structure
|
||||
|
||||
- Create `lib/crates/fabro-server/src/worker_control/mod.rs`
|
||||
- Owns the server-side control bus abstraction and re-exports the local backend.
|
||||
- Create `lib/crates/fabro-server/src/worker_control/bus.rs`
|
||||
- Defines `WorkerControlBus`, `WorkerControlDelivery`, `WorkerControlMessageId`, `WorkerControlCursor`, and bus errors.
|
||||
- Create `lib/crates/fabro-server/src/worker_control/local.rs`
|
||||
- Implements `LocalWorkerControlBus` using process memory.
|
||||
- Create `lib/crates/fabro-server/src/server/handler/worker_control.rs`
|
||||
- Adds the worker-only WebSocket route.
|
||||
- Modify `lib/crates/fabro-server/src/server.rs`
|
||||
- Adds the bus to `AppState`, replaces subprocess `RunAnswerTransport` sends with bus publishes, removes stdin pumping.
|
||||
- Modify `lib/crates/fabro-server/src/server/handler/mod.rs`
|
||||
- Registers the worker control route.
|
||||
- Modify `lib/crates/fabro-server/src/server/handler/lifecycle.rs`
|
||||
- Sends pause/unpause/cancel controls through the transport/bus where appropriate.
|
||||
- Modify `lib/crates/fabro-cli/src/commands/run/runner.rs`
|
||||
- Replaces stdin reading with worker WebSocket client handling.
|
||||
- Modify `lib/crates/fabro-cli/Cargo.toml`
|
||||
- Adds `tokio-tungstenite` as a direct dependency if needed.
|
||||
- Modify `lib/crates/fabro-interview/src/control_protocol.rs`
|
||||
- Adds pause/unpause control messages and a transport delivery frame type shared by server and worker.
|
||||
|
||||
## Task 1: Define the Control Bus Contract
|
||||
|
||||
**Files:**
|
||||
- Create: `lib/crates/fabro-server/src/worker_control/mod.rs`
|
||||
- Create: `lib/crates/fabro-server/src/worker_control/bus.rs`
|
||||
- Modify: `lib/crates/fabro-server/src/lib.rs`
|
||||
|
||||
- [ ] Add a private `worker_control` module in `fabro-server`.
|
||||
- [ ] Define `WorkerControlMessageId` as an opaque cloneable id rather than a numeric type.
|
||||
- [ ] Define `WorkerControlCursor` with `Start` and `After(WorkerControlMessageId)` variants.
|
||||
- [ ] Define `WorkerControlDelivery { id: WorkerControlMessageId, envelope: WorkerControlEnvelope }`.
|
||||
- [ ] Define `WorkerControlBus` with async `publish(run_id, envelope)` and `subscribe(run_id, cursor)` methods.
|
||||
- [ ] Make `subscribe` return a stream-like receiver owned by the caller, so the WebSocket handler can forward messages without knowing the backend.
|
||||
- [ ] Define explicit bus errors for closed backend, unavailable backend, invalid cursor, and publish timeout.
|
||||
- [ ] Document in code comments that `Start` maps to Redis stream id `0-0` and `After(id)` maps to Redis `XREAD` after that id, but do not add Redis code.
|
||||
- [ ] Add unit tests for id equality/debug formatting and cursor parsing from the optional `after` query parameter.
|
||||
- [ ] Test that absent `after` parses as `WorkerControlCursor::Start`.
|
||||
- [ ] Test that present `after=local:42` parses as `WorkerControlCursor::After(...)`.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_control`.
|
||||
|
||||
## Task 2: Implement the Local In-Memory Bus
|
||||
|
||||
**Files:**
|
||||
- Create: `lib/crates/fabro-server/src/worker_control/local.rs`
|
||||
- Test: `lib/crates/fabro-server/src/worker_control/local.rs`
|
||||
|
||||
- [ ] Implement `LocalWorkerControlBus` as `Arc<Mutex<HashMap<RunId, LocalRunControlStream>>>`.
|
||||
- [ ] Store messages per run in insertion order with a monotonic local sequence id.
|
||||
- [ ] Wake active subscribers when `publish` appends a message.
|
||||
- [ ] Support `subscribe(run_id, Start)` for first worker startup; it must replay retained messages from the beginning of the run control stream.
|
||||
- [ ] Support `subscribe(run_id, After(id))` so reconnect uses the same API that later maps to Redis `XREAD`.
|
||||
- [ ] Allow `publish` before the worker subscribes; retained messages must be visible to the first `Start` subscriber.
|
||||
- [ ] Trim retained local messages to a bounded per-run size so a disconnected local worker cannot grow memory without bound. Use a named constant with initial value 1024 messages per run.
|
||||
- [ ] Return a clear `invalid cursor` error when a subscriber asks for an id that has been trimmed or belongs to a different local stream.
|
||||
- [ ] Add a cleanup method for terminal runs so completed/cancelled runs can release retained control messages.
|
||||
- [ ] Test that messages publish in order.
|
||||
- [ ] Test that an active subscriber receives a message published after subscription.
|
||||
- [ ] Test that messages published before subscription are replayed to a `Start` subscriber.
|
||||
- [ ] Test that `After(id)` receives only later messages.
|
||||
- [ ] Test that trimming bounds retained messages and reports an invalid old cursor.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_control`.
|
||||
|
||||
## Task 3: Add Control Bus to Server State
|
||||
|
||||
**Files:**
|
||||
- Modify: `lib/crates/fabro-server/src/server.rs`
|
||||
- Test: `lib/crates/fabro-server/src/server/tests.rs`
|
||||
|
||||
- [ ] Add `worker_control_bus: Arc<dyn WorkerControlBus>` to `AppState`.
|
||||
- [ ] Construct `LocalWorkerControlBus` in normal server state initialization.
|
||||
- [ ] Add a test-only way to inject a fake or local bus without exposing test helpers to production builds.
|
||||
- [ ] Keep demo/in-process execution behavior unchanged unless it currently depends on subprocess control.
|
||||
- [ ] Add a state construction test proving the default bus is local and available.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_control`.
|
||||
|
||||
## Task 4: Extend the Control Protocol
|
||||
|
||||
**Files:**
|
||||
- Modify: `lib/crates/fabro-interview/src/control_protocol.rs`
|
||||
- Test: `lib/crates/fabro-interview/src/control_protocol.rs`
|
||||
|
||||
- [ ] Add `WorkerControlEnvelope::pause_run()` and `WorkerControlEnvelope::unpause_run()` constructors.
|
||||
- [ ] Add `WorkerControlMessage::RunPause` serialized as `"run.pause"`.
|
||||
- [ ] Add `WorkerControlMessage::RunUnpause` serialized as `"run.unpause"`.
|
||||
- [ ] Add `WorkerControlDeliveryFrame { id: String, envelope: WorkerControlEnvelope }` as the WebSocket text-frame payload shared by server and worker.
|
||||
- [ ] Add round-trip serde tests for both new messages.
|
||||
- [ ] Add round-trip serde tests for `WorkerControlDeliveryFrame`.
|
||||
- [ ] Run `cargo nextest run -p fabro-interview control_protocol`.
|
||||
|
||||
## Task 5: Share Worker Message Handling
|
||||
|
||||
**Files:**
|
||||
- Modify: `lib/crates/fabro-cli/src/commands/run/runner.rs`
|
||||
- Test: `lib/crates/fabro-cli/src/commands/run/runner.rs`
|
||||
|
||||
- [ ] Split `apply_worker_control_line(...)` into parsing and `apply_worker_control_message(...)`.
|
||||
- [ ] Route WebSocket delivery frames through `apply_worker_control_message(...)`.
|
||||
- [ ] Route `run.pause` to `RunControlState::request_pause()`.
|
||||
- [ ] Route `run.unpause` to `RunControlState::request_unpause()`.
|
||||
- [ ] Add a small in-memory applied-id dedupe set in the worker control task; ignore duplicate delivery ids before applying envelopes.
|
||||
- [ ] Update `last_applied_id` only after `apply_worker_control_message(...)` returns.
|
||||
- [ ] Treat all current control messages as idempotent under delivery-id dedupe. `run.steer` must not be applied twice for the same delivery id.
|
||||
- [ ] Keep control stream close behavior explicit: an unexpected close triggers reconnect; a fatal invalid cursor interrupts pending interviews and fails/aborts the run as control-channel loss.
|
||||
- [ ] Update existing stdin-era tests to exercise the shared message handler directly.
|
||||
- [ ] Add tests for pause and unpause routing.
|
||||
- [ ] Add a test proving duplicate delivery ids are not applied twice.
|
||||
- [ ] Run `cargo nextest run -p fabro-cli runner`.
|
||||
|
||||
## Task 6: Add Worker WebSocket Client
|
||||
|
||||
**Files:**
|
||||
- Modify: `lib/crates/fabro-cli/Cargo.toml`
|
||||
- Modify: `lib/crates/fabro-cli/src/commands/run/runner.rs`
|
||||
- Test: `lib/crates/fabro-cli/src/commands/run/runner.rs`
|
||||
|
||||
- [ ] Add `tokio-tungstenite.workspace = true` as a direct `fabro-cli` dependency if the crate does not already have it.
|
||||
- [ ] Add a helper that builds the control-stream request for a `ServerTarget` and `RunId`.
|
||||
- [ ] For HTTP URLs, convert `http` to `ws` and `https` to `wss`.
|
||||
- [ ] For Unix socket paths, connect `tokio::net::UnixStream` and use `ws://fabro/api/v1/runs/{run_id}/worker/control-stream` for the handshake host/path.
|
||||
- [ ] Add the worker bearer token as an `Authorization: Bearer ...` request header.
|
||||
- [ ] On the first connection, omit the `after` query parameter so the server maps it to `WorkerControlCursor::Start`.
|
||||
- [ ] On reconnect, include `?after=<last_applied_id>` when `last_applied_id` is set.
|
||||
- [ ] Spawn a WebSocket control manager task in `execute(...)` after `ControlInterviewer`, `RunControlState`, `CancellationToken`, and `SteeringHub` are created, and before `operations::start` or `operations::resume`.
|
||||
- [ ] Gate `operations::start` and `operations::resume` on the first successful control-stream connection.
|
||||
- [ ] The control manager should keep reconnecting while the run is not locally terminal, with backoff starting at 100ms, doubling after each failure, and capped at 5s.
|
||||
- [ ] Deserialize each text frame into `WorkerControlDeliveryFrame`.
|
||||
- [ ] Apply each envelope through `apply_worker_control_message(...)`, then record the frame id as `last_applied_id`.
|
||||
- [ ] Respond to received WebSocket ping frames with pong frames.
|
||||
- [ ] Send worker-initiated ping frames every 15s.
|
||||
- [ ] Track pongs for worker-initiated pings and close the WebSocket after 45s without a matching pong or other proof of connection liveness.
|
||||
- [ ] Treat normal close/error as reconnectable while the run is not terminal.
|
||||
- [ ] Treat HTTP 410 Gone or a WebSocket close reason of `invalid_cursor` as fatal control-channel loss.
|
||||
- [ ] On fatal control-channel loss, interrupt pending interviews and fail/abort the run with an infrastructure/control-channel error, not a user cancellation.
|
||||
- [ ] Wire fatal control-channel loss back into `execute(...)` so the worker returns an error instead of silently continuing workflow execution.
|
||||
- [ ] Add tests for URL/request construction for `http`, `https`, and Unix socket targets.
|
||||
- [ ] Add tests proving first connection has no `after` query and reconnect includes `after=<last_applied_id>`.
|
||||
- [ ] Add tests for reconnect backoff bounds.
|
||||
- [ ] Add tests for ping/pong timeout behavior using paused Tokio time.
|
||||
- [ ] Add a local Unix-socket WebSocket test proving the client can complete a handshake against an Axum route.
|
||||
- [ ] Run `cargo nextest run -p fabro-cli runner`.
|
||||
|
||||
## Task 7: Add Worker-Only Control Stream Route
|
||||
|
||||
**Files:**
|
||||
- Create: `lib/crates/fabro-server/src/server/handler/worker_control.rs`
|
||||
- Modify: `lib/crates/fabro-server/src/server/handler/mod.rs`
|
||||
- Modify: `lib/crates/fabro-server/src/principal_middleware.rs`
|
||||
- Test: `lib/crates/fabro-server/src/server/tests.rs`
|
||||
|
||||
- [ ] Add a narrow helper or extractor that accepts only authenticated worker principals whose token run id matches the route run id.
|
||||
- [ ] Add `GET /runs/{id}/worker/control-stream` to real API routes only.
|
||||
- [ ] Reject missing runs, terminal runs, and archived runs before upgrading.
|
||||
- [ ] Reject user JWTs and cross-run worker JWTs.
|
||||
- [ ] Parse absent `after` into `WorkerControlCursor::Start`.
|
||||
- [ ] Parse present `after` into `WorkerControlCursor::After(id)`.
|
||||
- [ ] On upgrade, call `worker_control_bus.subscribe(run_id, cursor)`.
|
||||
- [ ] If `subscribe` returns invalid cursor before upgrade, reject with HTTP 410 Gone.
|
||||
- [ ] Serialize each `WorkerControlDelivery` to `WorkerControlDeliveryFrame` and send it as a WebSocket text frame.
|
||||
- [ ] Send server-initiated ping frames every 15s.
|
||||
- [ ] Respond to received WebSocket ping frames with pong frames.
|
||||
- [ ] Track pongs for server-initiated pings and close the WebSocket after 45s without a matching pong or other proof of connection liveness.
|
||||
- [ ] On timeout or disconnect, drop the bus subscription so local resources are released.
|
||||
- [ ] Do not store the live WebSocket sender in `ManagedRun`; the bus is now the delivery boundary.
|
||||
- [ ] Add tests for auth rejection, successful `Start` subscription, successful `After(id)` subscription, frame delivery, invalid cursor rejection as 410 Gone, ping/pong timeout cleanup, and cross-run worker rejection.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_control`.
|
||||
|
||||
## Task 8: Replace Server-Side Stdin Transport with Bus Publishing
|
||||
|
||||
**Files:**
|
||||
- Modify: `lib/crates/fabro-server/src/server.rs`
|
||||
- Modify: `lib/crates/fabro-server/src/server/handler/lifecycle.rs`
|
||||
- Modify: `lib/crates/fabro-server/src/server/handler/pair.rs`
|
||||
- Test: `lib/crates/fabro-server/src/server/tests.rs`
|
||||
|
||||
- [ ] Replace `RunAnswerTransport::Subprocess { control_tx }` with a bus-backed subprocess/worker variant.
|
||||
- [ ] Ensure the bus-backed variant has enough context to publish messages for the correct `RunId`.
|
||||
- [ ] Delete `pump_worker_control_jsonl(...)`.
|
||||
- [ ] Change `worker_command(...)` so `__run-worker` uses `stdin(Stdio::null())` instead of `stdin(Stdio::piped())`.
|
||||
- [ ] Remove child-stdin extraction and the control pump task from `execute_run_subprocess(...)`.
|
||||
- [ ] Keep stderr capture and worker exit handling unchanged.
|
||||
- [ ] Update `RunAnswerTransport` methods so answer, cancel, steer, interrupt, pair start/message/end all publish the existing envelope to `WorkerControlBus`.
|
||||
- [ ] Add `pause_run()` and `unpause_run()` methods on `RunAnswerTransport`.
|
||||
- [ ] Update pause/unpause lifecycle handlers to send `run.pause` and `run.unpause` over the bus for running workers.
|
||||
- [ ] Keep process signals only for hard cleanup paths such as cancel fallback, shutdown, terminal delete, and force removal.
|
||||
- [ ] Update existing tests that assert subprocess transport enqueue behavior to assert bus publish behavior instead.
|
||||
- [ ] Add a `worker_command` test proving stdin is null/not piped and `FABRO_WORKER_TOKEN` still travels only through env.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_command`.
|
||||
|
||||
## Task 9: End-to-End Local Control Flow Regression
|
||||
|
||||
**Files:**
|
||||
- Test: `lib/crates/fabro-cli/tests/it/cmd/runner.rs`
|
||||
- Test: `lib/crates/fabro-server/tests/it/scenario/lifecycle.rs`
|
||||
|
||||
- [ ] Add a test run where the worker connects to the control WebSocket and receives a cancel request through `LocalWorkerControlBus`.
|
||||
- [ ] Add a test where the server publishes a control message before the worker connects and the worker receives it on first connection.
|
||||
- [ ] Add a reconnect test where the worker applies message A, reconnects with `after=<A>`, and then receives only message B.
|
||||
- [ ] Add an invalid-cursor test proving the worker reports control-channel loss as infrastructure failure rather than user cancellation.
|
||||
- [ ] Add a human-interview test proving submitted answers reach the worker through the bus and WebSocket.
|
||||
- [ ] Add a steer or interrupt test proving live agent controls still reach the worker transport.
|
||||
- [ ] Add a local Unix-socket server test proving the default local server target works without stdin.
|
||||
- [ ] Run `cargo nextest run -p fabro-cli --test it runner`.
|
||||
- [ ] Run `cargo nextest run -p fabro-server --test it lifecycle`.
|
||||
|
||||
## Task 10: Final Verification
|
||||
|
||||
**Files:**
|
||||
- Modify only if failures expose necessary fixes.
|
||||
|
||||
- [ ] Run `cargo nextest run -p fabro-interview control_protocol`.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_control`.
|
||||
- [ ] Run `cargo nextest run -p fabro-cli runner`.
|
||||
- [ ] Run `cargo nextest run -p fabro-server worker_command`.
|
||||
- [ ] Run `cargo nextest run -p fabro-server --test it lifecycle`.
|
||||
- [ ] Run `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings`.
|
||||
- [ ] Confirm no code path still writes `WorkerControlEnvelope` to child stdin.
|
||||
- [ ] Confirm no Redis dependency, Redis config key, or Redis runtime path was added.
|
||||
- [ ] Confirm there is no `Latest` cursor or wait-for-subscriber behavior in the control bus.
|
||||
- [ ] Confirm WebSocket ping/pong handling is explicit on both worker and server.
|
||||
- [ ] Confirm `run.steer` and other controls are protected from duplicate delivery-id application.
|
||||
- [ ] Confirm `__run-worker` still scrubs `FABRO_WORKER_TOKEN` from process env before launching descendants.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- All worker control traffic uses the worker control bus plus WebSocket last-mile transport.
|
||||
- Local Unix-socket server targets and remote HTTP(S) server targets both support worker control without Redis.
|
||||
- Local and single-node deployments require no external control-channel service.
|
||||
- The bus API can later be implemented by Redis Streams without changing API handlers or worker message handling.
|
||||
- First worker connection replays retained messages from the beginning of the run control stream; reconnect resumes after the last fully applied id.
|
||||
- Invalid cursor is the only fatal control-stream replay failure and is surfaced as infrastructure/control-channel failure, not user cancellation.
|
||||
- WebSocket liveness is explicit and backend-agnostic.
|
||||
- Existing run event/blob/artifact HTTP paths are unchanged.
|
||||
- Existing worker JWT scope rules remain authoritative.
|
||||
- Temporary WebSocket disconnects reconnect and replay through the bus; only unrecoverable replay loss reports worker-control-unavailable behavior.
|
||||
- Worker stdout/stderr behavior remains unchanged except that stdin is no longer a control channel.
|
||||
- Redis is clearly documented as future work and is not required by this plan.
|
||||
|
||||
|
||||
## Completed stages
|
||||
- **toolchain**: succeeded
|
||||
- Script: `command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1`
|
||||
- Output:
|
||||
```
|
||||
cargo 1.95.0 (f2d3ce0bd 2026-03-21)
|
||||
```
|
||||
- **preflight_compile**: succeeded
|
||||
- Script: `cargo check -q --workspace 2>&1`
|
||||
- Output: (empty)
|
||||
- **preflight_lint**: succeeded
|
||||
- Script: `cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1`
|
||||
- Output: (empty)
|
||||
- **implement**: succeeded
|
||||
- Model: gpt-5.5, 3.4m tokens in / 94.1k out
|
||||
- Files: /home/daytona/workspace/fabro/lib/crates/fabro-server/src/server/handler/worker_control.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/worker_control/bus.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/worker_control/local.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/worker_control/mod.rs
|
||||
- **simplify_opus**: succeeded
|
||||
- Model: claude-opus-4-7, 148.3k tokens in / 40.1k out
|
||||
- Files: /home/daytona/workspace/fabro/lib/crates/fabro-cli/src/commands/run/runner.rs, /home/daytona/workspace/fabro/lib/crates/fabro-interview/src/control_protocol.rs, /home/daytona/workspace/fabro/lib/crates/fabro-interview/src/lib.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/server.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/server/handler/worker_control.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/server/tests.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/worker_control/bus.rs, /home/daytona/workspace/fabro/lib/crates/fabro-server/src/worker_control/local.rs
|
||||
|
||||
|
||||
# Simplify: Code Review and Cleanup
|
||||
|
||||
Review changes vs. origin for reuse, quality, and efficiency. Fix any issues found.
|
||||
|
||||
## Phase 1: Identify Changes
|
||||
|
||||
Run git diff (or git diff HEAD if there are staged changes) to see what changed. If there are no git changes, review the most recently modified files that the user mentioned or that you edited earlier in this conversation.
|
||||
|
||||
## Phase 2: Launch Three Review Agents in Parallel
|
||||
|
||||
Use the Agent tool to launch all three agents concurrently in a single message. Pass each agent the full diff so it has the complete context.
|
||||
|
||||
### Agent 1: Code Reuse Review
|
||||
|
||||
For each change:
|
||||
|
||||
1. Search for existing utilities and helpers that could replace newly written code. Use Grep to find similar patterns elsewhere in the codebase — common locations are utility directories, shared modules, and files adjacent to the changed ones.
|
||||
2. Flag any new function that duplicates existing functionality. Suggest the existing function to use instead.
|
||||
3. Flag any inline logic that could use an existing utility — hand-rolled string manipulation, manual path handling, custom environment checks, ad-hoc type guards, and similar patterns are common candidates.
|
||||
|
||||
Note: This is a greenfield app, so focus on maximizing simplicity and don't worry about changing things to achieve it.
|
||||
|
||||
### Agent 2: Code Quality Review
|
||||
|
||||
Review the same changes for hacky patterns:
|
||||
|
||||
1. Redundant state: state that duplicates existing state, cached values that could be derived, observers/effects that could be direct calls
|
||||
2. Parameter sprawl: adding new parameters to a function instead of generalizing or restructuring existing ones
|
||||
3. Copy-paste with slight variation: near-duplicate code blocks that should be unified with a shared abstraction
|
||||
4. Leaky abstractions: exposing internal details that should be encapsulated, or breaking existing abstraction boundaries
|
||||
5. Stringly-typed code: using raw strings where constants, enums (string unions), or branded types already exist in the codebase
|
||||
|
||||
Note: This is a greenfield app, so be aggressive in optimizing quality.
|
||||
|
||||
### Agent 3: Efficiency Review
|
||||
|
||||
Review the same changes for efficiency:
|
||||
|
||||
1. Unnecessary work: redundant computations, repeated file reads, duplicate network/API calls, N+1 patterns
|
||||
2. Missed concurrency: independent operations run sequentially when they could run in parallel
|
||||
3. Hot-path bloat: new blocking work added to startup or per-request/per-render hot paths
|
||||
4. Unnecessary existence checks: pre-checking file/resource existence before operating (TOCTOU anti-pattern) — operate directly and handle the error
|
||||
5. Memory: unbounded data structures, missing cleanup, event listener leaks
|
||||
6. Overly broad operations: reading entire files when only a portion is needed, loading all items when filtering for one
|
||||
|
||||
## Phase 3: Fix Issues
|
||||
|
||||
Wait for all three agents to complete. Aggregate their findings and fix each issue directly. If a finding is a false positive or not worth addressing, note it and move on — do not argue with the finding, just skip it.
|
||||
|
||||
When done, briefly summarize what was fixed (or confirm the code was already clean).
|
||||
5
stages/007-simplify_gpt@1/provider_used.json
Normal file
5
stages/007-simplify_gpt@1/provider_used.json
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mode": "agent",
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.5"
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue