From af2a1e4f6dfb875a08bcebd784b737ab201456fa Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 2 Apr 2026 10:40:19 -0700 Subject: [PATCH] Add server daemon management with Unix socket support Transform `fabro server` from foreground-only TCP into a proper daemon: - `server start` launches background daemon with flock-based locking - `server start --foreground` retains current blocking behavior - `server stop` sends SIGTERM, waits, escalates to SIGKILL - `server status` reports running/stopped with PID, bind, uptime (--json) - `--bind` replaces `--host`/`--port`, supporting Unix sockets and TCP - Default bind is `{storage_dir}/fabro.sock` (Unix socket) - Hidden `__serve` subcommand for daemon child process lifecycle - Graceful shutdown via SIGTERM/SIGINT signal handlers Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 1 + ...-001-feat-server-daemon-management-plan.md | 446 ++++++++++++++++++ lib/crates/fabro-cli/src/args.rs | 39 +- lib/crates/fabro-cli/src/commands/mod.rs | 2 + .../src/commands/server/foreground.rs | 39 ++ .../fabro-cli/src/commands/server/mod.rs | 67 +++ .../fabro-cli/src/commands/server/record.rs | 120 +++++ .../fabro-cli/src/commands/server/start.rs | 258 ++++++++++ .../fabro-cli/src/commands/server/status.rs | 52 ++ .../fabro-cli/src/commands/server/stop.rs | 45 ++ lib/crates/fabro-cli/src/main.rs | 15 +- lib/crates/fabro-proc/Cargo.toml | 3 + lib/crates/fabro-proc/src/flock.rs | 70 +++ lib/crates/fabro-proc/src/lib.rs | 5 + lib/crates/fabro-server/src/bind.rs | 112 +++++ lib/crates/fabro-server/src/lib.rs | 1 + lib/crates/fabro-server/src/serve.rs | 106 +++-- 17 files changed, 1341 insertions(+), 40 deletions(-) create mode 100644 docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md create mode 100644 lib/crates/fabro-cli/src/commands/server/foreground.rs create mode 100644 lib/crates/fabro-cli/src/commands/server/mod.rs create mode 100644 lib/crates/fabro-cli/src/commands/server/record.rs create mode 100644 lib/crates/fabro-cli/src/commands/server/start.rs create mode 100644 lib/crates/fabro-cli/src/commands/server/status.rs create mode 100644 lib/crates/fabro-cli/src/commands/server/stop.rs create mode 100644 lib/crates/fabro-proc/src/flock.rs create mode 100644 lib/crates/fabro-server/src/bind.rs diff --git a/Cargo.lock b/Cargo.lock index c47bfea78..bb20082c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,6 +1858,7 @@ version = "0.176.2" dependencies = [ "cc", "libc", + "tempfile", ] [[package]] diff --git a/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md b/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md new file mode 100644 index 000000000..cd2d56bff --- /dev/null +++ b/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md @@ -0,0 +1,446 @@ +--- +title: "feat: Add server daemon management with Unix socket support" +type: feat +status: completed +date: 2026-04-02 +deepened: 2026-04-02 +--- + +# feat: Add server daemon management with Unix socket support + +## Overview + +Transform `fabro server` from a foreground-only TCP server into a proper daemon with background/foreground modes, stop/status lifecycle commands, Unix socket binding, and flock-based locking to prevent thundering herd when multiple CLI invocations auto-start the server. + +## Problem Frame + +The fabro server currently runs only in the foreground on a TCP port. Users must manually manage the process lifecycle. There is no way to check if a server is running, stop it gracefully, or prevent duplicate instances. When the CLI eventually auto-starts the server on demand, concurrent CLI invocations could race to start multiple servers simultaneously. + +## Requirements Trace + +- R1. `fabro server start` launches as a background daemon by default +- R2. `fabro server start --foreground` retains current blocking behavior +- R3. `fabro server stop` sends SIGTERM, waits, escalates to SIGKILL +- R4. `fabro server status` reports running/stopped with PID, bind address, uptime +- R5. A JSON server record tracks PID and metadata; stale records are auto-cleaned +- R6. `--bind` replaces `--host`/`--port`, supporting both Unix sockets and TCP addresses +- R7. Default bind is `{storage_dir}/fabro.sock` (Unix socket) +- R8. flock-based locking prevents concurrent start attempts (thundering herd) +- R9. Only one server instance can run at a time per storage directory +- R10. TLS is not supported on Unix sockets (only on TCP) + +## Scope Boundaries + +- No systemd/launchd integration (out of scope) +- No log rotation or `server logs` subcommand beyond basic prev-file rotation (future work) +- Breaking change: `--host` and `--port` are removed, replaced by `--bind` +- Client-side Unix socket connectivity (TypeScript Axios client, CLI-to-server calls) is tracked as a follow-up concern -- this plan covers the server side only + +## Context & Research + +### Relevant Code and Patterns + +- `lib/crates/fabro-cli/src/commands/run/launcher.rs` -- JSON-based launcher records with PID tracking, stale detection via `process_alive()` + `ps` command-line matching, lazy cleanup on read +- `lib/crates/fabro-cli/src/commands/run/start.rs` -- Self-re-exec pattern: spawns `fabro __detached` with `pre_exec_setsid`, stdout/stderr to log file, writes record after spawn, checks `try_wait()` for immediate failure, forwards `--storage-dir` to child +- `lib/crates/fabro-cli/src/commands/run/detached.rs` -- `scopeguard::guard(launcher_path, remove_launcher_record)` for guaranteed cleanup on exit; `title_init()`/`title_set()` for proctitle; receives record path via `--launcher-path` arg +- `lib/crates/fabro-proc/src/` -- `process_alive`, `sigterm`, `sigkill`, `pre_exec_setsid`, `title_init`/`title_set`. All functions are thin libc wrappers -- no retry loops or higher-level logic +- `lib/crates/fabro-server/src/serve.rs` -- Current `serve_command()` with `ServeArgs`, config polling, webhook manager lifecycle. No graceful shutdown wired. TLS path uses manual `loop { listener.accept() }` via `hyper_util`, not `axum::serve` +- `lib/crates/fabro-cli/src/args.rs:650` -- `#[command(name = "__detached", hide = true)]` hidden subcommand pattern with its own args struct +- `lib/crates/fabro-cli/src/args.rs:972-984` -- `ServerNamespace`/`ServerCommand` enum +- `lib/crates/fabro-cli/src/main.rs:108-195` -- Config log level extraction and command dispatch for server + +### Institutional Learnings + +No `docs/solutions/` directory exists. Patterns are embedded in the launcher record system. + +## Key Technical Decisions + +- **Hidden subcommand for daemon child, not a flag**: The daemon spawns `fabro server __serve --record-path --bind ...` as a detached child. `__serve` is a hidden `ServerCommand` variant with its own args struct, matching the `__detached` pattern in `RunCommands`. This keeps `ServeArgs` focused on server runtime concerns and process lifecycle in `fabro-cli`. + +- **Process lifecycle stays in fabro-cli**: `fabro-server` has no dependency on `fabro-proc` and should not gain one. A `foreground.rs` in `fabro-cli/src/commands/server/` wraps `serve_command()` with record writes, scopeguard cleanup, and proctitle management -- same boundary as `detached.rs` wrapping workflow operations. + +- **Record ownership: parent writes, child cleans up**: In daemon mode, the parent writes `server.json` after spawn (and cleans up on failure). The child receives the record path via `--record-path` and sets up a `scopeguard` to remove it on exit. This matches the launcher record ownership model exactly. + +- **Server record, not bare PID file**: A JSON `ServerRecord` (pid, bind: Bind, log_path, started_at) stored at `{storage_dir}/server.json`, following the `LauncherRecord` pattern. The `bind` field uses the `Bind` enum (not a raw string) so consumers get type-safe access without re-parsing. Enables `status` to report rich info and supports PID-command-line validation against PID reuse. + +- **flock on a separate lock file**: `{storage_dir}/server.lock` is acquired with `LOCK_EX | LOCK_NB` before any start attempt. Losers of the race block on the lock (with timeout), then discover the server already running. The lock file is separate from `server.json` so the record can be atomically rewritten without interfering with the lock. `flock()` auto-releases on process crash. + +- **Keep fabro-proc thin**: Only `try_flock_exclusive(file) -> io::Result` goes in `fabro-proc` (thin libc wrapper). The timeout/retry loop lives in the caller (`server/start.rs`), consistent with every other function in `fabro-proc` being a single-syscall wrapper. + +- **`Bind` enum defined in fabro-server**: A `Bind` enum (`Bind::Unix(PathBuf) | Bind::Tcp(SocketAddr)`) lives in `fabro-server` since it's a server concern ("what am I binding to?"). Used in both `ServeArgs` resolution and `ServerRecord`. Serializes cleanly via serde tagged enum: `{"unix": "/path"}` or `{"tcp": "127.0.0.1:3000"}`. Consumers (stop, status) can match on the variant directly instead of re-parsing a string -- e.g., `stop` knows to clean up the socket file when `bind` is `Bind::Unix`. + +- **Bind address parsing**: `parse_bind(s: &str) -> Result` in `fabro-server`. If the value contains `/`, it's a Unix socket path. Otherwise it's `host:port` TCP. Default: `{storage_dir}/fabro.sock`. Unix socket paths are validated against the 104-byte limit on macOS (108 on Linux). + +- **Graceful shutdown via SIGTERM handler**: Wire `tokio::signal::unix::signal(SignalKind::terminate())` into `axum::serve().with_graceful_shutdown()`. The foreground mode also handles SIGINT (ctrl-c). **Known limitation:** The TLS codepath uses a manual `loop { listener.accept() }` via `hyper_util` and cannot use `with_graceful_shutdown`. Under TLS, `server stop` will rely on SIGTERM causing process exit (default behavior) but may need SIGKILL escalation. This is acceptable for now. + +- **Process title includes bind address**: `fabro: server {bind}` to support PID-command-line validation and disambiguate if the single-instance invariant ever relaxes. + +- **No TLS on Unix sockets**: When bind is a Unix socket, skip the TLS codepath entirely. TLS only applies to TCP binds. + +- **Log rotation on start**: Rename existing `server.log` to `server.log.prev` before starting, so crash diagnostics from the previous run are preserved. + +## Open Questions + +### Resolved During Planning + +- **Where does flock live?** Only `try_flock_exclusive` in `fabro-proc` (thin wrapper). Timeout loop in caller. +- **Default bind address?** `{storage_dir}/fabro.sock` (Unix socket). Resolved storage dir is used so it respects `--storage-dir` and config overrides. +- **How does the parent know the daemon is ready?** Poll-connect to the socket/port with a short timeout (up to 5s). Same approach as pg_ctl. +- **What happens to TLS + Unix socket?** Not supported. If `--bind` is a socket path and TLS is configured, emit a warning and skip TLS. +- **Hidden flag or hidden subcommand?** Hidden subcommand (`__serve`), matching the `__detached` pattern. Not a flag on `ServeArgs`. +- **Where does process lifecycle code live?** In `fabro-cli/src/commands/server/`, not in `fabro-server`. Same boundary as `detached.rs`. +- **Client connectivity with Unix socket default?** Tracked as follow-up -- this plan covers server-side only. The `server.json` record stores the bind address so clients can discover it. + +### Deferred to Implementation + +- Exact readiness polling interval and timeout values (start with 50ms interval, 5s timeout) +- Whether `server stop` should print tail of server.log on timeout before SIGKILL +- Exact chmod permissions on the Unix socket file (start with default, tighten if needed) + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +``` + fabro server start + | + acquire flock(server.lock) + / \ + (got lock) (blocked) + | | + read server.json wait for lock (retry loop in start.rs) + / \ | + (live pid) (none/stale) (got lock) + | | | + "already rotate server.log -> server.log.prev + running" spawn: fabro server __serve --record-path ... --bind ... + exit 1 pre_exec_setsid, stdout/stderr -> server.log + | + parent: write server.json with child PID + parent: poll-connect to bind address + / \ + (connected) (timeout) + | | + exit 0 print error + log tail + clean up server.json + exit 1 + + fabro server __serve --record-path --bind ... + | + title_init() + title_set("fabro: server ") + scopeguard(record_path, remove_server_record) + register SIGTERM/SIGINT shutdown + | + bind listener (Unix or TCP) + axum::serve(...).with_graceful_shutdown(...) + | + (on shutdown signal) + scopeguard fires: remove server.json + remove socket file (if Unix) +``` + +```mermaid +graph TB + U1[Unit 1: flock in fabro-proc] + U2[Unit 2: ServerRecord] + U3[Unit 3: Bind enum + --bind + Unix socket] + U4[Unit 4: daemon spawn + __serve] + U5[Unit 5: server stop] + U6[Unit 6: server status] + U7[Unit 7: main.rs dispatch] + + U1 --> U4 + U3 --> U2 + U2 --> U4 + U2 --> U5 + U2 --> U6 + U3 --> U4 + U4 --> U7 + U5 --> U7 + U6 --> U7 +``` + +Units 1 and 3 can run in parallel. Unit 2 depends on Unit 3 (for the `Bind` enum type). Units 5 and 6 can run in parallel after Unit 2. Unit 7 ties everything together. + +## Implementation Units + +- [x] **Unit 1: Add flock wrapper to fabro-proc** + + **Goal:** Provide `try_flock_exclusive` in `fabro-proc` as a thin libc wrapper for advisory file locking. + + **Requirements:** R8 + + **Dependencies:** None + + **Files:** + - Create: `lib/crates/fabro-proc/src/flock.rs` + - Modify: `lib/crates/fabro-proc/src/lib.rs` + - Test: `lib/crates/fabro-proc/src/flock.rs` (inline tests) + + **Approach:** + - `try_flock_exclusive(file: &File) -> io::Result` -- `libc::flock(fd, LOCK_EX | LOCK_NB)`, returns `Ok(false)` on `EWOULDBLOCK` + - `flock_unlock(file: &File) -> io::Result<()>` -- `libc::flock(fd, LOCK_UN)` for explicit unlock + - Unix-only (`#[cfg(unix)]`), consistent with existing `fabro-proc` gating + - No retry loops or timeout logic -- keep this crate at the syscall-wrapper level + + **Patterns to follow:** + - `lib/crates/fabro-proc/src/signal.rs` -- same style: thin wrapper over libc, `#[cfg(unix)]` gating, pub functions re-exported from `lib.rs` + - `lib/crates/fabro-proc/src/pre_exec.rs` -- unsafe block style and safety comments + + **Test scenarios:** + - Happy path: acquire lock on a temp file, confirm returns true + - Happy path: release lock (drop file), re-acquire succeeds + - Edge case: try_flock_exclusive on already-locked file (held by another fd) returns Ok(false) + + **Verification:** + - `cargo nextest run -p fabro-proc` passes + - `cargo clippy -p fabro-proc -- -D warnings` clean + +- [x] **Unit 2: Add ServerRecord and lifecycle helpers** + + **Goal:** Create a `ServerRecord` struct with read/write/remove/is_running helpers, mirroring `LauncherRecord`. + + **Requirements:** R5, R9 + + **Dependencies:** Unit 3 (for `Bind` enum type) + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/record.rs` + - Create: `lib/crates/fabro-cli/src/commands/server/mod.rs` + - Modify: `lib/crates/fabro-cli/src/commands/mod.rs` (add `pub(crate) mod server;` gated with `#[cfg(feature = "server")]`) + + **Approach:** + - `ServerRecord { pid: u32, bind: Bind, log_path: PathBuf, started_at: DateTime }` where `Bind` is the enum from `fabro-server` (Unit 3). The `bind` field serializes as a tagged enum (`{"unix": "/path"}` or `{"tcp": "127.0.0.1:3000"}`), giving consumers type-safe access -- e.g., `stop` matches on `Bind::Unix` to know it should remove the socket file + - Paths: `server_record_path(storage_dir) -> {storage_dir}/server.json`, `server_lock_path(storage_dir) -> {storage_dir}/server.lock`, `server_log_path(storage_dir) -> {storage_dir}/server.log` + - `server_record_is_running(record) -> bool` -- `process_alive(pid)` + `ps` command-line match for "fabro" and "server" + - `active_server_record(storage_dir) -> Option` -- read, check liveness, lazy-clean stale + - Path helpers only -- lock acquisition logic lives in Unit 4 + + **Patterns to follow:** + - `lib/crates/fabro-cli/src/commands/run/launcher.rs` -- identical record lifecycle pattern: `write_*`, `read_*`, `remove_*`, `*_is_running`, `active_*` + + **Test scenarios:** + - Happy path: write record, read it back, fields match + - Happy path: active_server_record returns None when no record file exists + - Edge case: active_server_record returns None and removes file when PID is dead (use pid u32::MAX) + - Edge case: active_server_record returns None and removes file when process doesn't match command-line check + + **Verification:** + - `cargo nextest run -p fabro-cli` for the new module's tests pass + +- [x] **Unit 3: Add Bind enum, replace --host/--port with --bind, add Unix socket listener** + + **Goal:** Define the `Bind` enum as the shared type for bind addresses. Change `ServeArgs` to use `--bind` instead of `--host`/`--port`. Support both Unix socket and TCP binding in `serve_command`. Wire graceful shutdown. + + **Requirements:** R6, R7, R10 + + **Dependencies:** None (parallel with Unit 1) + + **Files:** + - Create: `lib/crates/fabro-server/src/bind.rs` (Bind enum + parse_bind + Display impl) + - Modify: `lib/crates/fabro-server/src/lib.rs` (pub mod bind) + - Modify: `lib/crates/fabro-server/src/serve.rs` + - Modify: `lib/crates/fabro-server/Cargo.toml` (ensure tokio `net` feature includes unix support) + - Modify: `lib/crates/fabro-cli/src/main.rs` (update dispatch if ServeArgs shape changes) + - Test: `lib/crates/fabro-server/tests/it/api.rs` (update any tests using --host/--port) + - Test: `lib/crates/fabro-server/src/bind.rs` (inline tests for parse_bind) + + **Approach:** + - Define `Bind` enum in `bind.rs`: `Bind::Unix(PathBuf) | Bind::Tcp(SocketAddr)` with `Serialize`/`Deserialize` (serde tagged enum), `Display`, `Clone`, `Debug`, `PartialEq`. This type is used by both `serve_command` and `ServerRecord` (Unit 2) + - `parse_bind(bind: &str) -> Result`. Contains `/` -> Unix socket; otherwise `host:port` parsed as `SocketAddr`. Validate Unix socket path length (104 bytes on macOS, 108 on Linux) + - `Bind::display()` shows the address for human-readable output (used in proctitle, status, logs) + - Replace `--host` and `--port` on `ServeArgs` with `--bind` (Option, no default in clap -- default computed at runtime from resolved storage dir using the socket path) + - In `serve_command`, branch on `Bind` variant: `UnixListener::bind` vs `TcpListener::bind` + - For Unix sockets: remove stale socket file before bind, skip TLS codepath (log warning if TLS configured) + - Wire `axum::serve(listener, router).with_graceful_shutdown(shutdown_signal())` for the non-TLS path, where `shutdown_signal` awaits SIGTERM or SIGINT. **Note:** The TLS path (`serve_tls`) uses a manual accept loop and cannot use `with_graceful_shutdown` -- document as known limitation, SIGTERM will still cause process exit + - Derive `Clone` on `ServeArgs` to simplify the config polling task (currently manually clones each field) + + **Patterns to follow:** + - Current `serve.rs` listener binding and axum::serve pattern + - Axum 0.8 supports `UnixListener` directly via `axum::serve` + + **Test scenarios:** + - Happy path: parse_bind with "127.0.0.1:3000" returns Tcp variant + - Happy path: parse_bind with "/tmp/fabro.sock" returns Unix variant + - Edge case: parse_bind with invalid address returns error + - Edge case: parse_bind with path exceeding 104 bytes returns error on macOS + - Happy path: server binds to Unix socket and accepts HTTP requests over it + - Happy path: server binds to TCP address (existing behavior preserved) + - Edge case: stale socket file is removed before binding + - Integration: graceful shutdown on SIGTERM -- server stops accepting connections and exits cleanly + + **Verification:** + - `cargo nextest run -p fabro-server` passes + - Existing server tests still pass (adapted for --bind) + +- [x] **Unit 4: Add daemon spawn, __serve hidden subcommand, and foreground wrapper** + + **Goal:** Make `server start` launch a background daemon by default. `--foreground` retains current behavior. Both modes write/clean server records. Daemon mode uses flock to prevent thundering herd. `__serve` is the hidden subcommand the daemon child runs. + + **Requirements:** R1, R2, R8, R9 + + **Dependencies:** Units 1, 2, 3 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/start.rs` (daemon spawn logic + flock retry loop) + - Create: `lib/crates/fabro-cli/src/commands/server/foreground.rs` (wraps `serve_command` with record lifecycle, scopeguard, proctitle) + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `__Serve` hidden variant to `ServerCommand` with its own args struct; add `--foreground` flag to `Start` variant args) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` (dispatch) + + **Approach:** + - **`ServerCommand::__Serve(ServeChildArgs)`**: Hidden subcommand with `--record-path`, `--bind`, plus forwarded args (`--model`, `--provider`, `--dry-run`, `--sandbox`, `--max-concurrent-runs`, `--config`, `--storage-dir`). Dispatches to `foreground.rs`. + - **`foreground.rs`**: `title_init()` + `title_set("fabro: server {bind}")`, `scopeguard::guard(record_path, remove_server_record)`, then calls `serve_command()`. Mirrors `detached.rs` wrapping workflow operations. + - **`start.rs` daemon path**: Open `server.lock`, retry `try_flock_exclusive` in a loop (50ms intervals, 5s timeout); check `active_server_record`; if running, print "already running" and exit 1; rotate `server.log` to `server.log.prev`; spawn self with `fabro server __serve --record-path --bind ...` using `pre_exec_setsid`, stdout/stderr to `server.log`, stdin null, `env_remove("FABRO_JSON")`; write `server.json` with child PID; check `try_wait()` for immediate failure; poll-connect to bind address (50ms intervals, 5s timeout); on success print "server started (pid N) on ", exit 0; on failure print error + tail of log, clean up, exit 1 + - **`start.rs` foreground path** (`--foreground`): Write `server.json`, register scopeguard for cleanup, then call `serve_command()` directly (no re-exec) + - **Forward all relevant args to child**: `--storage-dir`, `--config`, `--model`, `--provider`, `--dry-run`, `--sandbox`, `--max-concurrent-runs` + + **Patterns to follow:** + - `lib/crates/fabro-cli/src/commands/run/start.rs` -- self-re-exec with `pre_exec_setsid`, log redirect, record write, `try_wait` check, `--storage-dir` forwarding, `env_remove("FABRO_JSON")` + - `lib/crates/fabro-cli/src/commands/run/detached.rs` -- `scopeguard` cleanup, `title_init`/`title_set`, receives record path via arg + - `lib/crates/fabro-cli/src/args.rs:650` -- `#[command(name = "__detached", hide = true)]` pattern + + **Test scenarios:** + - Happy path: `server start` spawns daemon, writes server.json, exits 0 + - Happy path: `server start --foreground` runs in foreground, writes server.json, cleans up on exit + - Edge case: `server start` when already running prints "already running" and exits 1 + - Edge case: `server start` with stale server.json (dead PID) cleans up and starts fresh + - Happy path: flock prevents concurrent start -- second caller waits and finds server running + - Edge case: daemon fails to start (bad bind address) -- parent reports error, cleans up record + - Edge case: daemon child exits immediately -- parent detects via try_wait, reports error + - Integration: after daemon start, server.json contains correct PID and bind address + - Integration: server.log.prev contains previous log content after restart + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + - Manual: `fabro server start` starts daemon, `fabro server start` again says "already running" + +- [x] **Unit 5: Add server stop subcommand** + + **Goal:** `fabro server stop` sends SIGTERM, waits for graceful exit, escalates to SIGKILL, cleans up. + + **Requirements:** R3 + + **Dependencies:** Unit 2 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/stop.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `Stop` variant to `ServerCommand` with `StopArgs { timeout }`) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` + + **Approach:** + - Read `active_server_record` -- if None, print "not running", exit 1 + - `fabro_proc::sigterm(pid)` + - Poll `process_alive(pid)` at 100ms intervals up to `--timeout` (default 10s) + - If still alive after timeout, `fabro_proc::sigkill(pid)` + - Remove `server.json` and socket file (match on `record.bind` -- if `Bind::Unix(path)`, remove the socket file) + - Print "server stopped" + + **Patterns to follow:** + - `fabro_proc::sigterm`/`sigkill`/`process_alive` for signal management + - `launcher.rs::remove_launcher_record` for cleanup + + **Test scenarios:** + - Happy path: stop a running server -- sends SIGTERM, process exits, record cleaned up + - Edge case: stop when not running -- prints "not running", exits 1 + - Edge case: stop with stale record (dead PID) -- cleans up record, prints "not running", exits 1 + - Edge case: process doesn't exit within timeout -- escalates to SIGKILL + - Happy path: Unix socket file is removed after stop + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + - Manual: `fabro server start && fabro server stop` completes cleanly + +- [x] **Unit 6: Add server status subcommand** + + **Goal:** `fabro server status` reports running/stopped state with metadata. Supports `--json`. + + **Requirements:** R4 + + **Dependencies:** Unit 2 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/status.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `Status` variant to `ServerCommand` with `StatusArgs { json }`) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` + + **Approach:** + - Read `active_server_record` -- if None, print "not running", exit 1 + - Compute uptime from `started_at` + - Human output: "running (pid N) on , started X ago" + - `--json`: `{ "status": "running", "pid": N, "bind": "...", "started_at": "...", "uptime_seconds": N }` + - Exit code: 0 = running, 1 = not running (same as `pg_ctl status`) + + **Patterns to follow:** + - Other CLI commands that support `--json` output (check `globals.json` usage pattern) + + **Test scenarios:** + - Happy path: status when running -- prints info, exits 0 + - Happy path: status --json when running -- outputs valid JSON with expected fields + - Edge case: status when not running -- prints "not running", exits 1 + - Edge case: status with stale record -- cleans up, prints "not running", exits 1 + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + +- [x] **Unit 7: Update main.rs dispatch and config loading for new server subcommands** + + **Goal:** Wire all server subcommands (start, stop, status, __serve) into CLI dispatch. Fix config log level extraction to handle new ServerCommand variants. + + **Requirements:** R1, R2, R3, R4 + + **Dependencies:** Units 4, 5, 6 + + **Files:** + - Modify: `lib/crates/fabro-cli/src/main.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (update `Commands::name()` match arm for all new variants) + + **Approach:** + - Change `let ServerCommand::Start(args) = ns.command;` to `match ns.command { Start(..) => ..., Stop(..) => ..., Status(..) => ..., __Serve(..) => ... }` in both the config log level block and the dispatch block + - `__Serve` and `Start` (when in foreground/daemon mode) load server settings for log level; `Stop` and `Status` load user settings + - Update `Commands::name()` to return `"server start"`, `"server stop"`, `"server status"`, `"server __serve"` for telemetry + - Log prefix: `"server"` for `Start`/`__Serve`, `"cli"` for `Stop`/`Status` + - `#[cfg(feature = "server")]` gating on all new paths + + **Patterns to follow:** + - Existing dispatch pattern in `main.rs` for other namespace commands (e.g., `Commands::RunCmd`) + - Feature gating on all server references + + **Test scenarios:** + - Happy path: `fabro server stop --help` prints help text + - Happy path: `fabro server status --help` prints help text + - Happy path: `fabro server start --help` still works with new --bind flag and --foreground flag + - Happy path: telemetry name returns correct values for each subcommand + + **Verification:** + - `cargo nextest run -p fabro-cli` passes (including existing server tests) + - `cargo clippy --workspace -- -D warnings` clean + +## System-Wide Impact + +- **Interaction graph:** `serve_command()` gains a graceful shutdown signal handler. A new `foreground.rs` wrapper in `fabro-cli` manages server record lifecycle around `serve_command()`. Config polling task and webhook manager lifecycle preserved unchanged. Webhook manager shutdown (line 274-277 of `serve.rs`) becomes reachable for the first time via graceful shutdown -- the existing code is correct but the implementing agent should not add a scopeguard that drops the tokio runtime before async shutdown runs. +- **Error propagation:** Daemon start failures surface to the parent via poll-connect timeout + log tail. Stop failures surface via exit code. +- **State lifecycle risks:** Stale `server.json` after crash -- mitigated by PID liveness + command-line check on every read, with lazy cleanup (same proven pattern as launcher records). Stale socket file -- removed before bind attempt. +- **API surface parity:** The `--bind` change is breaking for anyone using `--host`/`--port`. No API endpoint changes. Client-side connectivity to Unix sockets (TypeScript Axios client, CLI HTTP calls) is not addressed in this plan and needs follow-up. +- **Integration coverage:** Unit tests can verify record lifecycle and parse_bind. Integration tests should cover the full start/status/stop cycle with a real server process. +- **Unchanged invariants:** All HTTP routes, auth, config reloading, webhook manager, and SSE streaming behavior are unchanged. The server's runtime behavior is identical once it's listening. + +## Risks & Dependencies + +| Risk | Mitigation | +|------|------------| +| PID reuse after crash leads to signaling wrong process | Two-phase check: `process_alive` + `ps` command-line matching for "fabro" and "server" with bind address in proctitle, same proven pattern as launcher records | +| flock not supported on all filesystems (e.g., NFS) | Storage dir is local by convention (~/.fabro). Document that network filesystems are unsupported for storage | +| Unix socket path exceeds 104-byte limit on macOS | `parse_bind` validates path length at parse time with a clear error message | +| Axum 0.8 UnixListener support | Axum 0.8 is already the workspace version; `serve()` accepts `UnixListener` natively | +| Breaking --host/--port removal | Acceptable per scope decision. Users see a clear clap error pointing to --bind | +| Race between parent writing server.json and child exiting | Check `try_wait()` immediately after spawn (same pattern as `start.rs`); if child already exited, clean up and report error | +| TLS path lacks graceful shutdown | Documented as known limitation. SIGTERM still causes process exit; stop command escalates to SIGKILL after timeout. TLS + daemon is an uncommon combination | +| Client-side code assumes TCP | Tracked as follow-up. Server record stores bind address for client discovery | + +## Sources & References + +- Related code: `lib/crates/fabro-cli/src/commands/run/launcher.rs`, `lib/crates/fabro-cli/src/commands/run/start.rs`, `lib/crates/fabro-cli/src/commands/run/detached.rs` +- Related code: `lib/crates/fabro-proc/src/signal.rs`, `lib/crates/fabro-proc/src/pre_exec.rs` +- Related code: `lib/crates/fabro-server/src/serve.rs` +- Related code: `lib/crates/fabro-cli/src/args.rs:650` (`__detached` hidden subcommand pattern), `lib/crates/fabro-cli/src/args.rs:972-984` (`ServerCommand`) +- Related code: `lib/crates/fabro-cli/src/main.rs:108-195` diff --git a/lib/crates/fabro-cli/src/args.rs b/lib/crates/fabro-cli/src/args.rs index 1601a1e47..4336cf698 100644 --- a/lib/crates/fabro-cli/src/args.rs +++ b/lib/crates/fabro-cli/src/args.rs @@ -857,7 +857,10 @@ impl Commands { }, #[cfg(feature = "server")] Self::Server(ns) => match &ns.command { - ServerCommand::Start(_) => "server start", + ServerCommand::Start { .. } => "server start", + ServerCommand::Stop { .. } => "server stop", + ServerCommand::Status { .. } => "server status", + ServerCommand::Serve { .. } => "server __serve", }, Self::Doctor { .. } => "doctor", Self::Repo(ns) => match &ns.command { @@ -976,11 +979,43 @@ pub(crate) struct ServerNamespace { pub(crate) command: ServerCommand, } +#[cfg(feature = "server")] +use fabro_server::serve::ServeArgs; + #[cfg(feature = "server")] #[derive(Subcommand)] pub(crate) enum ServerCommand { /// Start the HTTP API server - Start(fabro_server::serve::ServeArgs), + Start { + /// Run in the foreground instead of daemonizing + #[arg(long)] + foreground: bool, + + #[command(flatten)] + serve_args: ServeArgs, + }, + /// Stop the HTTP API server + Stop { + /// Seconds to wait for graceful shutdown before SIGKILL + #[arg(long, default_value = "10")] + timeout: u64, + }, + /// Show server status + Status { + /// Output as JSON + #[arg(long)] + json: bool, + }, + /// Internal: run the server process (spawned by `start`) + #[command(name = "__serve", hide = true)] + Serve { + /// Path to the server record file + #[arg(long)] + record_path: PathBuf, + + #[command(flatten)] + serve_args: ServeArgs, + }, } #[derive(Args)] diff --git a/lib/crates/fabro-cli/src/commands/mod.rs b/lib/crates/fabro-cli/src/commands/mod.rs index ab45ccf85..e7fb7a942 100644 --- a/lib/crates/fabro-cli/src/commands/mod.rs +++ b/lib/crates/fabro-cli/src/commands/mod.rs @@ -15,6 +15,8 @@ pub(crate) mod run; pub(crate) mod runs; pub(crate) mod sandbox; pub(crate) mod secret; +#[cfg(feature = "server")] +pub(crate) mod server; pub(crate) mod skill; pub(crate) mod store; pub(crate) mod system; diff --git a/lib/crates/fabro-cli/src/commands/server/foreground.rs b/lib/crates/fabro-cli/src/commands/server/foreground.rs new file mode 100644 index 000000000..009c01dbf --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/foreground.rs @@ -0,0 +1,39 @@ +use std::path::PathBuf; + +use anyhow::Result; +use fabro_server::bind::Bind; +use fabro_server::serve; +use fabro_server::serve::ServeArgs; +use fabro_util::terminal::Styles; + +use super::record; + +pub(crate) async fn execute( + record_path: PathBuf, + mut serve_args: ServeArgs, + bind: Bind, + storage_dir: Option, + styles: &'static Styles, +) -> Result<()> { + let _ = fabro_proc::title_init(); + fabro_proc::title_set(&format!("fabro: server {bind}")); + + // Ensure serve_args carries the resolved bind so the server binds correctly. + serve_args.bind = Some(bind.to_string()); + + let _record_guard = scopeguard::guard(record_path, |path| { + record::remove_server_record(&path); + }); + + // If Unix socket, clean up socket file on exit + let _socket_guard = if let Bind::Unix(ref path) = bind { + let path = path.clone(); + Some(scopeguard::guard(path, |p| { + let _ = std::fs::remove_file(p); + })) + } else { + None + }; + + serve::serve_command(serve_args, styles, storage_dir).await +} diff --git a/lib/crates/fabro-cli/src/commands/server/mod.rs b/lib/crates/fabro-cli/src/commands/server/mod.rs new file mode 100644 index 000000000..e17c14815 --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/mod.rs @@ -0,0 +1,67 @@ +pub(crate) mod foreground; +pub(crate) mod record; +pub(crate) mod start; +pub(crate) mod status; +pub(crate) mod stop; + +use std::time::Duration; + +use anyhow::Result; +use fabro_server::bind; +use fabro_server::bind::Bind; +use fabro_util::terminal::Styles; + +use crate::args::{GlobalArgs, ServerCommand}; +use crate::user_config; + +pub(crate) async fn dispatch(command: ServerCommand, globals: &GlobalArgs) -> Result<()> { + match command { + ServerCommand::Start { + foreground, + serve_args, + } => { + let settings = user_config::load_user_settings_with_globals(globals)?; + let storage_dir = settings.storage_dir(); + let bind_addr = match serve_args.bind.as_deref() { + Some(s) => bind::parse_bind(s)?, + None => Bind::Unix(storage_dir.join("fabro.sock")), + }; + let styles: &'static Styles = Box::leak(Box::new(Styles::detect_stderr())); + start::execute(bind_addr, foreground, serve_args, storage_dir, styles).await + } + ServerCommand::Stop { timeout } => { + let settings = user_config::load_user_settings_with_globals(globals)?; + let storage_dir = settings.storage_dir(); + stop::execute(&storage_dir, Duration::from_secs(timeout)); + Ok(()) + } + ServerCommand::Status { json } => { + let settings = user_config::load_user_settings_with_globals(globals)?; + let storage_dir = settings.storage_dir(); + status::execute(&storage_dir, json) + } + ServerCommand::Serve { + record_path, + serve_args, + } => { + let bind_addr = match serve_args.bind.as_deref() { + Some(s) => bind::parse_bind(s)?, + None => { + // __serve should always receive an explicit --bind from the parent, + // but fall back to the storage dir default if missing. + let settings = user_config::load_user_settings_with_globals(globals)?; + Bind::Unix(settings.storage_dir().join("fabro.sock")) + } + }; + let styles: &'static Styles = Box::leak(Box::new(Styles::detect_stderr())); + foreground::execute( + record_path, + serve_args, + bind_addr, + globals.storage_dir.clone(), + styles, + ) + .await + } + } +} diff --git a/lib/crates/fabro-cli/src/commands/server/record.rs b/lib/crates/fabro-cli/src/commands/server/record.rs new file mode 100644 index 000000000..2e197dd2c --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/record.rs @@ -0,0 +1,120 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use fabro_server::bind::Bind; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct ServerRecord { + pub pid: u32, + pub bind: Bind, + pub log_path: PathBuf, + pub started_at: DateTime, +} + +pub(crate) fn server_record_path(storage_dir: &Path) -> PathBuf { + storage_dir.join("server.json") +} + +pub(crate) fn server_lock_path(storage_dir: &Path) -> PathBuf { + storage_dir.join("server.lock") +} + +pub(crate) fn server_log_path(storage_dir: &Path) -> PathBuf { + storage_dir.join("server.log") +} + +pub(crate) fn write_server_record(path: &Path, record: &ServerRecord) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, serde_json::to_string_pretty(record)?) + .with_context(|| format!("Failed to write server metadata to {}", path.display())) +} + +pub(crate) fn read_server_record(path: &Path) -> Option { + let content = std::fs::read_to_string(path).ok()?; + serde_json::from_str(&content).ok() +} + +pub(crate) fn remove_server_record(path: &Path) { + let _ = std::fs::remove_file(path); +} + +pub(crate) fn server_record_is_running(record: &ServerRecord) -> bool { + fabro_proc::process_alive(record.pid) && server_process_matches(record) +} + +pub(crate) fn active_server_record(storage_dir: &Path) -> Option { + let path = server_record_path(storage_dir); + let record = read_server_record(&path)?; + if server_record_is_running(&record) { + Some(record) + } else { + remove_server_record(&path); + None + } +} + +#[cfg(unix)] +fn server_process_matches(record: &ServerRecord) -> bool { + let output = match std::process::Command::new("ps") + .args(["-ww", "-o", "command=", "-p", &record.pid.to_string()]) + .output() + { + Ok(output) if output.status.success() => output, + _ => return false, + }; + let command = String::from_utf8_lossy(&output.stdout); + command.contains("fabro") && command.contains("server") +} + +#[cfg(not(unix))] +fn server_process_matches(_record: &ServerRecord) -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_record(bind: Bind) -> ServerRecord { + ServerRecord { + pid: std::process::id(), + bind, + log_path: PathBuf::from("/tmp/server.log"), + started_at: Utc::now(), + } + } + + #[test] + fn write_and_read_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = server_record_path(dir.path()); + let record = test_record(Bind::Tcp("127.0.0.1:3000".parse().unwrap())); + write_server_record(&path, &record).unwrap(); + + let loaded = read_server_record(&path).unwrap(); + assert_eq!(loaded.pid, record.pid); + assert_eq!(loaded.bind, record.bind); + } + + #[test] + fn active_server_record_returns_none_when_no_file() { + let dir = tempfile::tempdir().unwrap(); + assert!(active_server_record(dir.path()).is_none()); + } + + #[test] + fn active_server_record_cleans_stale_dead_pid() { + let dir = tempfile::tempdir().unwrap(); + let path = server_record_path(dir.path()); + let mut record = test_record(Bind::Tcp("127.0.0.1:3000".parse().unwrap())); + record.pid = u32::MAX; // definitely not alive + write_server_record(&path, &record).unwrap(); + + assert!(active_server_record(dir.path()).is_none()); + assert!(!path.exists()); // lazy cleanup removed file + } +} diff --git a/lib/crates/fabro-cli/src/commands/server/start.rs b/lib/crates/fabro-cli/src/commands/server/start.rs new file mode 100644 index 000000000..085732728 --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/start.rs @@ -0,0 +1,258 @@ +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use anyhow::{Result, bail}; +use chrono::Utc; +use fabro_server::bind::Bind; +use fabro_server::serve; +use fabro_server::serve::ServeArgs; +use fabro_util::terminal::Styles; + +use super::record; + +pub(crate) async fn execute( + bind: Bind, + foreground: bool, + mut serve_args: ServeArgs, + storage_dir: PathBuf, + styles: &'static Styles, +) -> Result<()> { + // Ensure serve_args carries the resolved bind address so the server knows + // what to listen on. + serve_args.bind = Some(bind.to_string()); + + if foreground { + execute_foreground(bind, serve_args, storage_dir, styles).await + } else { + execute_daemon(&bind, &serve_args, &storage_dir) + } +} + +// --------------------------------------------------------------------------- +// Foreground mode +// --------------------------------------------------------------------------- + +async fn execute_foreground( + bind: Bind, + serve_args: ServeArgs, + storage_dir: PathBuf, + styles: &'static Styles, +) -> Result<()> { + let lock_file = acquire_lock(&storage_dir)?; + let _lock_file = lock_file; // keep alive for the duration + + if let Some(existing) = record::active_server_record(&storage_dir) { + bail!( + "Server already running (pid {}) on {}", + existing.pid, + existing.bind + ); + } + + let record_path = record::server_record_path(&storage_dir); + record::write_server_record( + &record_path, + &record::ServerRecord { + pid: std::process::id(), + bind: bind.clone(), + log_path: record::server_log_path(&storage_dir), + started_at: Utc::now(), + }, + )?; + + let _record_guard = scopeguard::guard(record_path, |path| { + record::remove_server_record(&path); + }); + + // Clean up Unix socket on exit + let _socket_guard = if let Bind::Unix(ref path) = bind { + let path = path.clone(); + Some(scopeguard::guard(path, |p| { + let _ = std::fs::remove_file(p); + })) + } else { + None + }; + + serve::serve_command(serve_args, styles, Some(storage_dir)).await +} + +// --------------------------------------------------------------------------- +// Daemon mode +// --------------------------------------------------------------------------- + +fn execute_daemon(bind: &Bind, serve_args: &ServeArgs, storage_dir: &Path) -> Result<()> { + let lock_file = acquire_lock(storage_dir)?; + let _lock_file = lock_file; // keep alive until function returns + + if let Some(existing) = record::active_server_record(storage_dir) { + eprintln!( + "Server already running (pid {}) on {}", + existing.pid, existing.bind + ); + std::process::exit(1); + } + + // Rotate logs + let log_path = record::server_log_path(storage_dir); + let prev_path = log_path.with_extension("log.prev"); + let _ = std::fs::rename(&log_path, &prev_path); + + // Spawn child: `fabro server __serve --record-path --bind ...` + let record_path = record::server_record_path(storage_dir); + let log_file = std::fs::File::create(&log_path)?; + let stdout_log = log_file.try_clone()?; + let exe = std::env::current_exe()?; + + let mut cmd = std::process::Command::new(&exe); + cmd.args(["server", "__serve"]) + .arg("--record-path") + .arg(&record_path) + .arg("--bind") + .arg(bind.to_string()); + + // Forward optional serve args + if let Some(ref model) = serve_args.model { + cmd.args(["--model", model]); + } + if let Some(ref provider) = serve_args.provider { + cmd.args(["--provider", provider]); + } + if serve_args.dry_run { + cmd.arg("--dry-run"); + } + if let Some(ref sandbox) = serve_args.sandbox { + cmd.args(["--sandbox", &sandbox.to_string()]); + } + if let Some(max) = serve_args.max_concurrent_runs { + cmd.args(["--max-concurrent-runs", &max.to_string()]); + } + if let Some(ref config) = serve_args.config { + cmd.arg("--config").arg(config); + } + + // Forward global --storage-dir + cmd.arg("--storage-dir").arg(storage_dir); + + cmd.env_remove("FABRO_JSON"); + cmd.stdout(stdout_log) + .stderr(log_file) + .stdin(std::process::Stdio::null()); + + #[cfg(unix)] + fabro_proc::pre_exec_setsid(&mut cmd); + + let mut child = cmd.spawn()?; + + // Write server record with child PID + record::write_server_record( + &record_path, + &record::ServerRecord { + pid: child.id(), + bind: bind.clone(), + log_path: log_path.clone(), + started_at: Utc::now(), + }, + )?; + + // Check if the child already exited + if let Ok(Some(status)) = child.try_wait() { + record::remove_server_record(&record_path); + let tail = read_log_tail(&log_path, 20); + if !tail.is_empty() { + eprintln!("{tail}"); + } + bail!("Server exited immediately with status {status}"); + } + + // Poll-connect until the server is ready + let poll_interval = Duration::from_millis(50); + let timeout = Duration::from_secs(5); + let mut elapsed = Duration::ZERO; + + while elapsed < timeout { + if try_connect(bind) { + eprintln!("Server started (pid {}) on {bind}", child.id()); + return Ok(()); + } + + // Check that the child hasn't died while we poll + if let Ok(Some(status)) = child.try_wait() { + record::remove_server_record(&record_path); + if let Bind::Unix(ref path) = *bind { + let _ = std::fs::remove_file(path); + } + let tail = read_log_tail(&log_path, 20); + if !tail.is_empty() { + eprintln!("{tail}"); + } + bail!("Server exited during startup with status {status}"); + } + + thread::sleep(poll_interval); + elapsed += poll_interval; + } + + // Timed out waiting for connection + record::remove_server_record(&record_path); + if let Bind::Unix(ref path) = *bind { + let _ = std::fs::remove_file(path); + } + let _ = child.kill(); + let _ = child.wait(); + let tail = read_log_tail(&log_path, 20); + if !tail.is_empty() { + eprintln!("{tail}"); + } + bail!("Server did not become ready within {timeout:?}"); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn acquire_lock(storage_dir: &Path) -> Result { + let lock_path = record::server_lock_path(storage_dir); + if let Some(parent) = lock_path.parent() { + std::fs::create_dir_all(parent)?; + } + let lock_file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&lock_path)?; + + let poll_interval = Duration::from_millis(50); + let timeout = Duration::from_secs(5); + let mut elapsed = Duration::ZERO; + + while !fabro_proc::try_flock_exclusive(&lock_file)? { + if elapsed >= timeout { + bail!("timed out waiting for server lock"); + } + thread::sleep(poll_interval); + elapsed += poll_interval; + } + + Ok(lock_file) +} + +fn try_connect(bind: &Bind) -> bool { + match bind { + Bind::Tcp(addr) => { + std::net::TcpStream::connect_timeout(addr, Duration::from_millis(100)).is_ok() + } + Bind::Unix(path) => std::os::unix::net::UnixStream::connect(path).is_ok(), + } +} + +fn read_log_tail(log_path: &Path, lines: usize) -> String { + match std::fs::read_to_string(log_path) { + Ok(content) => { + let tail: Vec<&str> = content.lines().rev().take(lines).collect(); + tail.into_iter().rev().collect::>().join("\n") + } + Err(_) => String::new(), + } +} diff --git a/lib/crates/fabro-cli/src/commands/server/status.rs b/lib/crates/fabro-cli/src/commands/server/status.rs new file mode 100644 index 000000000..2d84fbd90 --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/status.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +use anyhow::Result; +use chrono::Utc; + +use super::record; + +pub(crate) fn execute(storage_dir: &Path, json: bool) -> Result<()> { + let Some(record) = record::active_server_record(storage_dir) else { + if json { + println!(r#"{{"status":"stopped"}}"#); + } else { + eprintln!("Server is not running"); + } + std::process::exit(1); + }; + + if json { + let uptime_seconds = (Utc::now() - record.started_at).num_seconds().max(0); + let output = serde_json::json!({ + "status": "running", + "pid": record.pid, + "bind": record.bind.to_string(), + "started_at": record.started_at.to_rfc3339(), + "uptime_seconds": uptime_seconds, + }); + println!("{}", serde_json::to_string_pretty(&output)?); + } else { + let uptime = format_uptime(Utc::now() - record.started_at); + eprintln!( + "Server running (pid {}) on {}, started {} ago", + record.pid, record.bind, uptime + ); + } + + Ok(()) +} + +fn format_uptime(duration: chrono::Duration) -> String { + let total_seconds = duration.num_seconds().max(0); + let hours = total_seconds / 3600; + let minutes = (total_seconds % 3600) / 60; + let seconds = total_seconds % 60; + + if hours > 0 { + format!("{hours}h {minutes}m {seconds}s") + } else if minutes > 0 { + format!("{minutes}m {seconds}s") + } else { + format!("{seconds}s") + } +} diff --git a/lib/crates/fabro-cli/src/commands/server/stop.rs b/lib/crates/fabro-cli/src/commands/server/stop.rs new file mode 100644 index 000000000..6815adbed --- /dev/null +++ b/lib/crates/fabro-cli/src/commands/server/stop.rs @@ -0,0 +1,45 @@ +use std::path::Path; +use std::thread; +use std::time::Duration; + +use fabro_server::bind::Bind; + +use super::record; + +pub(crate) fn execute(storage_dir: &Path, timeout: Duration) { + let Some(record) = record::active_server_record(storage_dir) else { + eprintln!("Server is not running"); + std::process::exit(1); + }; + + fabro_proc::sigterm(record.pid); + + // Poll for process exit + let poll_interval = Duration::from_millis(100); + let mut elapsed = Duration::ZERO; + while elapsed < timeout { + if !fabro_proc::process_alive(record.pid) { + break; + } + thread::sleep(poll_interval); + elapsed += poll_interval; + } + + // Escalate to SIGKILL if still alive + if fabro_proc::process_alive(record.pid) { + fabro_proc::sigkill(record.pid); + // Brief wait for SIGKILL to take effect + thread::sleep(Duration::from_millis(100)); + } + + // Clean up record file + let record_path = record::server_record_path(storage_dir); + record::remove_server_record(&record_path); + + // Clean up Unix socket file if applicable + if let Bind::Unix(ref path) = record.bind { + let _ = std::fs::remove_file(path); + } + + eprintln!("Server stopped"); +} diff --git a/lib/crates/fabro-cli/src/main.rs b/lib/crates/fabro-cli/src/main.rs index f283e9990..cd584689e 100644 --- a/lib/crates/fabro-cli/src/main.rs +++ b/lib/crates/fabro-cli/src/main.rs @@ -109,7 +109,13 @@ async fn main_inner() -> (String, Result<()>) { #[cfg(feature = "server")] { if let Commands::Server(ServerNamespace { - command: ServerCommand::Start(args), + command: + ServerCommand::Start { + serve_args: args, .. + } + | ServerCommand::Serve { + serve_args: args, .. + }, }) = command.as_ref() { match fabro_config::server::load_server_settings(args.config.as_deref()) { @@ -141,7 +147,7 @@ async fn main_inner() -> (String, Result<()>) { } }; - let log_prefix = if command_name == "server start" { + let log_prefix = if command_name == "server start" || command_name == "server __serve" { "server" } else { "cli" @@ -188,10 +194,7 @@ async fn main_inner() -> (String, Result<()>) { Commands::Model { command } => commands::model::execute(command, &globals).await?, #[cfg(feature = "server")] Commands::Server(ns) => { - let ServerCommand::Start(args) = ns.command; - let styles: &'static Styles = Box::leak(Box::new(Styles::detect_stderr())); - fabro_server::serve::serve_command(args, styles, globals.storage_dir.clone()) - .await?; + commands::server::dispatch(ns.command, &globals).await?; } Commands::Doctor { verbose, dry_run } => { let cli_settings = user_config::load_user_settings()?; diff --git a/lib/crates/fabro-proc/Cargo.toml b/lib/crates/fabro-proc/Cargo.toml index 1d0adeff2..4ecdb8ab8 100644 --- a/lib/crates/fabro-proc/Cargo.toml +++ b/lib/crates/fabro-proc/Cargo.toml @@ -12,5 +12,8 @@ workspace = true [target.'cfg(unix)'.dependencies] libc = "0.2" +[dev-dependencies] +tempfile = "3" + [build-dependencies] cc = "1" diff --git a/lib/crates/fabro-proc/src/flock.rs b/lib/crates/fabro-proc/src/flock.rs new file mode 100644 index 000000000..09c8f0c60 --- /dev/null +++ b/lib/crates/fabro-proc/src/flock.rs @@ -0,0 +1,70 @@ +use std::fs::File; +use std::io; +use std::os::unix::io::AsRawFd; + +/// Try to acquire an exclusive (write) lock on `file` without blocking. +/// +/// Returns `Ok(true)` if the lock was acquired, `Ok(false)` if another +/// process/fd already holds the lock, and `Err` for unexpected errors. +pub fn try_flock_exclusive(file: &File) -> io::Result { + // SAFETY: flock() on a valid fd is safe; LOCK_NB makes it non-blocking. + let ret = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if ret == 0 { + Ok(true) + } else { + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EWOULDBLOCK) => Ok(false), + _ => Err(err), + } + } +} + +/// Release any lock held on `file`. +pub fn flock_unlock(file: &File) -> io::Result<()> { + // SAFETY: flock() with LOCK_UN on a valid fd is safe. + let ret = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_UN) }; + if ret == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + + #[test] + fn acquire_exclusive_lock() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lock"); + let file = File::create(&path).unwrap(); + + let acquired = try_flock_exclusive(&file).unwrap(); + assert!(acquired); + } + + #[test] + fn unlock_then_reacquire() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lock"); + let file = File::create(&path).unwrap(); + + assert!(try_flock_exclusive(&file).unwrap()); + flock_unlock(&file).unwrap(); + assert!(try_flock_exclusive(&file).unwrap()); + } + + #[test] + fn second_fd_blocked() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lock"); + let file1 = File::create(&path).unwrap(); + let file2 = File::open(&path).unwrap(); + + assert!(try_flock_exclusive(&file1).unwrap()); + assert!(!try_flock_exclusive(&file2).unwrap()); + } +} diff --git a/lib/crates/fabro-proc/src/lib.rs b/lib/crates/fabro-proc/src/lib.rs index 5872cd2e5..dd63c64d4 100644 --- a/lib/crates/fabro-proc/src/lib.rs +++ b/lib/crates/fabro-proc/src/lib.rs @@ -1,5 +1,7 @@ #![allow(unsafe_code)] +#[cfg(unix)] +mod flock; #[cfg(unix)] mod pre_exec; mod signal; @@ -11,6 +13,9 @@ pub use signal::process_alive; #[cfg(unix)] pub use signal::{sigkill, sigterm, sigterm_process_group}; +#[cfg(unix)] +pub use flock::{flock_unlock, try_flock_exclusive}; + #[cfg(target_os = "linux")] pub use pre_exec::pre_exec_pdeathsig; #[cfg(unix)] diff --git a/lib/crates/fabro-server/src/bind.rs b/lib/crates/fabro-server/src/bind.rs new file mode 100644 index 000000000..ac417c18f --- /dev/null +++ b/lib/crates/fabro-server/src/bind.rs @@ -0,0 +1,112 @@ +use std::fmt; +use std::net::SocketAddr; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Bind { + Unix(PathBuf), + Tcp(SocketAddr), +} + +impl fmt::Display for Bind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unix(path) => write!(f, "{}", path.display()), + Self::Tcp(addr) => write!(f, "{addr}"), + } + } +} + +/// Parse a bind address string into a `Bind` value. +/// +/// If the string contains `/`, it is treated as a Unix socket path. Otherwise +/// it is parsed as a TCP `host:port` address. +/// +/// # Errors +/// +/// Returns an error if the TCP address cannot be parsed, or if a Unix socket +/// path exceeds the OS limit (104 bytes on macOS, 108 on Linux). +pub fn parse_bind(s: &str) -> anyhow::Result { + if s.contains('/') { + let path = PathBuf::from(s); + validate_unix_path_length(&path)?; + Ok(Bind::Unix(path)) + } else { + let addr: SocketAddr = s + .parse() + .map_err(|e| anyhow::anyhow!("invalid TCP address '{s}': {e}"))?; + Ok(Bind::Tcp(addr)) + } +} + +fn validate_unix_path_length(path: &std::path::Path) -> anyhow::Result<()> { + #[cfg(target_os = "macos")] + const MAX_UNIX_PATH: usize = 104; + #[cfg(not(target_os = "macos"))] + const MAX_UNIX_PATH: usize = 108; + + let path_bytes = path.as_os_str().as_encoded_bytes().len(); + if path_bytes >= MAX_UNIX_PATH { + anyhow::bail!( + "Unix socket path is too long ({path_bytes} bytes, max {MAX_UNIX_PATH}): {}", + path.display() + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_tcp_address() { + let bind = parse_bind("127.0.0.1:3000").unwrap(); + assert_eq!(bind, Bind::Tcp("127.0.0.1:3000".parse().unwrap())); + } + + #[test] + fn parse_unix_socket_path() { + let bind = parse_bind("/tmp/fabro.sock").unwrap(); + assert_eq!(bind, Bind::Unix(PathBuf::from("/tmp/fabro.sock"))); + } + + #[test] + fn parse_invalid_tcp_address() { + let result = parse_bind("not-an-address"); + assert!(result.is_err()); + } + + #[test] + fn parse_unix_path_exceeding_limit() { + // Build a path that exceeds the OS limit + #[cfg(target_os = "macos")] + const LIMIT: usize = 104; + #[cfg(not(target_os = "macos"))] + const LIMIT: usize = 108; + + let long_path = format!("/{}", "a".repeat(LIMIT)); + let result = parse_bind(&long_path); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("too long"), + "expected 'too long' in error: {err_msg}" + ); + } + + #[test] + fn display_tcp() { + let bind = Bind::Tcp("0.0.0.0:8080".parse().unwrap()); + assert_eq!(bind.to_string(), "0.0.0.0:8080"); + } + + #[test] + fn display_unix() { + let bind = Bind::Unix(PathBuf::from("/run/fabro.sock")); + assert_eq!(bind.to_string(), "/run/fabro.sock"); + } +} diff --git a/lib/crates/fabro-server/src/lib.rs b/lib/crates/fabro-server/src/lib.rs index 537b3ec18..12772d0b6 100644 --- a/lib/crates/fabro-server/src/lib.rs +++ b/lib/crates/fabro-server/src/lib.rs @@ -3,6 +3,7 @@ allow(clippy::absolute_paths, clippy::await_holding_lock, clippy::float_cmp) )] +pub mod bind; #[allow(clippy::wildcard_imports, clippy::absolute_paths)] mod demo; pub mod error; diff --git a/lib/crates/fabro-server/src/serve.rs b/lib/crates/fabro-server/src/serve.rs index 7dfeed151..60e8f987f 100644 --- a/lib/crates/fabro-server/src/serve.rs +++ b/lib/crates/fabro-server/src/serve.rs @@ -5,7 +5,7 @@ use std::time::Duration; use fabro_config::server::{load_server_settings, resolve_storage_dir}; use fabro_util::terminal::Styles; use object_store::local::LocalFileSystem; -use tokio::net::TcpListener; +use tokio::net::{TcpListener, UnixListener}; use tokio::time::interval; use tracing::{error, info, warn}; @@ -13,6 +13,7 @@ use clap::Args; use fabro_types::Settings; +use crate::bind::{self, Bind}; use crate::github_webhooks::WebhookManager; use crate::jwt_auth::{AuthMode, AuthStrategy, resolve_auth_mode}; use crate::server::{build_router, create_app_state_with_store, spawn_scheduler}; @@ -20,15 +21,11 @@ use crate::tls::{ClientAuth, build_rustls_config, serve_tls}; use fabro_llm::client::Client as LlmClient; use fabro_sandbox::SandboxProvider; -#[derive(Args)] +#[derive(Args, Clone)] pub struct ServeArgs { - /// Port to listen on - #[arg(long, default_value = "3000")] - pub port: u16, - - /// Host address to bind to - #[arg(long, default_value = "127.0.0.1")] - pub host: String, + /// Address to bind to (host:port for TCP, or path containing / for Unix socket) + #[arg(long)] + pub bind: Option, /// Override default LLM model #[arg(long)] @@ -151,16 +148,18 @@ pub async fn serve_command( spawn_scheduler(Arc::clone(&state)); let router = build_router(state, auth_mode); - let addr = format!("{}:{}", args.host, args.port); - let listener = TcpListener::bind(&addr).await?; + let bind_addr = match args.bind { + Some(ref s) => bind::parse_bind(s)?, + None => Bind::Tcp("127.0.0.1:3000".parse().unwrap()), + }; - info!(host = %args.host, port = args.port, dry_run = dry_run_mode, "API server started"); + info!(bind = %bind_addr, dry_run = dry_run_mode, "API server started"); eprintln!( "{}", styles.bold.apply_to(format!( "Fabro server listening on {}", - styles.cyan.apply_to(&addr) + styles.cyan.apply_to(&bind_addr) )), ); if dry_run_mode { @@ -215,16 +214,7 @@ pub async fn serve_command( // Spawn config polling task let settings_for_poll = Arc::clone(&shared_settings); let config_path_for_poll = config_path.clone(); - let args_for_poll = ServeArgs { - port: args.port, - host: args.host.clone(), - model: args.model.clone(), - provider: args.provider.clone(), - dry_run: args.dry_run, - sandbox: args.sandbox, - max_concurrent_runs: args.max_concurrent_runs, - config: config_path.clone(), - }; + let args_for_poll = args.clone(); tokio::spawn(async move { let mut interval = interval(Duration::from_secs(5)); interval.tick().await; // skip first immediate tick @@ -251,24 +241,48 @@ pub async fn serve_command( } }); - // Branch: TLS or plain HTTP + // Branch: TLS, plain TCP, or Unix socket let tls_settings = shared_settings .read() .expect("config lock poisoned") .api .as_ref() .and_then(|a| a.tls.clone()); - if let Some(ref tls_settings) = tls_settings { - let client_auth = client_auth.unwrap(); - let rustls_config = build_rustls_config(tls_settings, client_auth); - let tls_acceptor = tokio_rustls::TlsAcceptor::from(rustls_config); + match bind_addr { + Bind::Unix(ref path) => { + if tls_settings.is_some() { + warn!("TLS is configured but not supported on Unix sockets; ignoring TLS settings"); + } - info!("TLS enabled"); + // Remove stale socket file before binding + if path.exists() { + std::fs::remove_file(path)?; + } - serve_tls(listener, tls_acceptor, router).await?; - } else { - axum::serve(listener, router).await?; + let listener = UnixListener::bind(path)?; + axum::serve(listener, router) + .with_graceful_shutdown(shutdown_signal()) + .await?; + } + Bind::Tcp(addr) => { + let listener = TcpListener::bind(addr).await?; + + if let Some(ref tls_settings) = tls_settings { + let client_auth = client_auth.unwrap(); + let rustls_config = build_rustls_config(tls_settings, client_auth); + let tls_acceptor = tokio_rustls::TlsAcceptor::from(rustls_config); + + info!("TLS enabled"); + + // TLS uses a manual accept loop and cannot use with_graceful_shutdown + serve_tls(listener, tls_acceptor, router).await?; + } else { + axum::serve(listener, router) + .with_graceful_shutdown(shutdown_signal()) + .await?; + } + } } // Clean up webhook listener on shutdown @@ -279,6 +293,34 @@ pub async fn serve_command( Ok(()) } +async fn shutdown_signal() { + use tokio::signal; + + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install SIGTERM handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + () = ctrl_c => {}, + () = terminate => {}, + } + + info!("Shutdown signal received, stopping server"); +} + /// Derive client certificate verification mode from the resolved auth strategies. fn client_auth_from_mode(auth_mode: &AuthMode) -> ClientAuth { let strategies = match auth_mode {