fabro/lib/crates/fabro-cli/tests/it/scenario/smoke.rs
fabro-sh-0530[bot] 54bc67017e
feat: Replace duration/elapsed fields with wall_time_ms and StageTiming (#343)
## Summary

Replaces the ambiguous `runtime_secs`, `elapsed_secs`, and `duration_ms`
timing fields on run/stage public API surfaces with explicit
`wall_time_ms` (elapsed clock time) and a `StageTiming` value object
that also carries `inference_time_ms`, `tool_time_ms`, and
`active_time_ms`.

This is a greenfield breaking change — no compatibility shims are
preserved.

### What changed

**API shape**
- `RunBillingStage.runtime_secs` → `RunBillingStage.timing: StageTiming`
- `RunBillingTotals.runtime_secs` → `RunBillingTotals.timing:
StageTiming`
- `RunSummary.timestamps.duration_ms` / `elapsed_secs` removed; a
top-level `timing: StageTiming | null` field added
- Stage list item `duration_secs` → `wall_time_ms`

**Web app (`apps/fabro-web`)**
- `run-billing.tsx`: `liveRuntimeSecs` → `liveWallTimeMs`; live ticking
now returns milliseconds and the footer total sums `wallTimeMs` across
rows
- `stage-sidebar.ts`: `duration_secs` → `wall_time_ms` for the per-stage
duration display
- `runs.ts`: `elapsed_secs` lookup replaced with `timing.wall_time_ms`
- `formatElapsedSecs` / `formatDurationSecs` call sites replaced with
`formatDurationMs`

**Lockfile / tooling**
- `@openapitools/openapi-generator-cli@2.20.2` added as a dev dependency
to `@qltysh/fabro-api-client` to support regenerating the TypeScript
client after schema edits; several transitive deps pulled in alongside
it.

### Design notes

- **Units are now consistent**: every timing value on run/stage surfaces
is in milliseconds; the old API mixed seconds (`runtime_secs`,
`elapsed_secs`) with milliseconds (`duration_ms`).
- **Live ticking** still works correctly: the in-flight billing row
computes `now - startedAt` in ms and sums across rows for the footer,
avoiding a server round-trip during a running stage.
- **`StageTiming.active_time_ms = inference_time_ms + tool_time_ms`** —
parallel work is summed, so run active time can exceed wall time.
- Subsystem-internal `duration_ms` fields (sandbox setup, devcontainer
lifecycle, hooks) are intentionally left unchanged; only public
run/stage timing surfaces are affected.


### Fabro Details

<details>
<summary>Ran 9 stages in 115m 53s for $108.50</summary>

| Stage | Duration | Cost | Retries |
|---|---|---|---|
| start | 0s | – | 0 |
| toolchain | 1s | – | 0 |
| preflight_compile | 2m 6s | – | 0 |
| preflight_lint | 2m 18s | – | 0 |
| implement | 80m 53s | $101.97 | 0 |
| simplify_opus | 21m 35s | $4.09 | 0 |
| simplify_gpt | 5m 1s | $2.44 | 0 |
| verify | 3m 11s | – | 0 |
| fmt | 3s | – | 0 |
| **Total** | **115m 53s** | **$108.50** | **0** |

</details>

<details>
<summary>Ran <code>ImplementPlan.fabro</code> (12 nodes and 15
edges)</summary>

```dot
digraph ImplementPlan {
    graph [
        goal="Implement and simplify",
        model_stylesheet="
            * { model: claude-opus-4-7; }
        "
    ]
    rankdir=LR

    start [shape=Mdiamond, label="Start"]
    exit  [shape=Msquare, label="Exit"]

    toolchain         [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0]
    preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0]
    preflight_lint    [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0]
    fix_lints         [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3]
    implement         [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."]
    simplify_opus     [label="Simplify (Opus)", prompt="@prompts/simplify.md"]
    simplify_gpt      [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"]
    verify            [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"]
    fixup             [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3]
    fmt               [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0]

    start -> toolchain
    toolchain -> preflight_compile [condition="outcome=succeeded"]
    toolchain -> exit
    preflight_compile -> preflight_lint [condition="outcome=succeeded"]
    preflight_compile -> exit
    preflight_lint -> implement [condition="outcome=succeeded"]
    preflight_lint -> fix_lints
    fix_lints -> preflight_lint
    implement -> simplify_opus -> simplify_gpt -> verify
    verify -> fmt   [condition="outcome=succeeded"]
    verify -> fixup
    fixup -> verify
    fmt -> exit
}

```

</details>

⚒️ Generated with [Fabro](https://fabro.sh)

---------

Co-authored-by: Fabro <noreply@fabro.sh>
Co-authored-by: Bryan Helmkamp <bryan@brynary.com>
2026-05-21 21:06:40 -04:00

390 lines
14 KiB
Rust

use fabro_test::{fabro_snapshot, test_context};
use httpmock::MockServer;
use crate::cmd::support::remote_run_summary_json;
use crate::support::{LightweightCli, run_projection_json, unique_run_id};
fn live_run_state_response(run_id: &str) -> serde_json::Value {
run_projection_json(
run_id,
&serde_json::json!({
"kind": "running"
}),
)
}
fn run_sse_body(run_id: &str) -> String {
let completed = serde_json::json!({
"seq": 2,
"event": "run.completed",
"id": "evt-run-completed",
"run_id": run_id,
"ts": "2026-04-05T12:00:01Z",
"properties": {
"timing": {"wall_time_ms": 12, "inference_time_ms": 0, "tool_time_ms": 0, "active_time_ms": 0},
"artifact_count": 0,
"status": "succeeded",
"reason": "completed"
}
});
format!("data: {completed}\n\n")
}
#[test]
fn help_smoke_covers_high_cost_commands() {
let cli = LightweightCli::new();
let mut artifact = cli.command();
artifact.args(["artifact", "--help"]);
fabro_snapshot!(artifact, @"
success: true
exit_code: 0
----- stdout -----
Inspect and copy run artifacts (screenshots, reports, traces)
Usage: fabro artifact [OPTIONS] <COMMAND>
Commands:
list List artifacts for a workflow run
cp Copy artifacts from a workflow run
help Print this message or the help of the given subcommand(s)
Options:
--json Output as JSON [env: FABRO_JSON=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
let mut artifact_list = cli.command();
artifact_list.args(["artifact", "list", "--help"]);
fabro_snapshot!(artifact_list, @"
success: true
exit_code: 0
----- stdout -----
List artifacts for a workflow run
Usage: fabro artifact list [OPTIONS] <RUN_ID>
Arguments:
<RUN_ID> Run ID (or prefix)
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--node <NODE> Filter to artifacts from a specific node
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--retry <RETRY> Filter to artifacts from a specific retry attempt
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
let mut artifact_cp = cli.command();
artifact_cp.args(["artifact", "cp", "--help"]);
fabro_snapshot!(artifact_cp, @"
success: true
exit_code: 0
----- stdout -----
Copy artifacts from a workflow run
Usage: fabro artifact cp [OPTIONS] <SOURCE> [DEST]
Arguments:
<SOURCE> Source: RUN_ID (all artifacts) or RUN_ID:path (specific artifact)
[DEST] Destination directory (defaults to current directory) [default: .]
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--node <NODE> Filter to artifacts from a specific node
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--retry <RETRY> Filter to artifacts from a specific retry attempt
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--tree Preserve {node_slug}/retry_{N}/ directory structure
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
let mut settings = cli.command();
settings.args(["settings", "--help"]);
fabro_snapshot!(settings, @"
success: true
exit_code: 0
----- stdout -----
Inspect effective settings
Usage: fabro settings [OPTIONS]
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
let mut attach = cli.command();
attach.args(["attach", "--help"]);
fabro_snapshot!(attach, @"
success: true
exit_code: 0
----- stdout -----
Attach to a running or finished workflow run
Usage: fabro attach [OPTIONS] <RUN>
Arguments:
<RUN> Run ID prefix or workflow name
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
}
#[test]
fn completion_smoke_covers_help_and_generation() {
let cli = LightweightCli::new();
let mut help = cli.command();
help.args(["completion", "--help"]);
fabro_snapshot!(help, @"
success: true
exit_code: 0
----- stdout -----
Generate shell completions
Usage: fabro completion [OPTIONS] <SHELL>
Arguments:
<SHELL> Shell to generate completions for [possible values: bash, elvish, fish, powershell, zsh]
Options:
--json Output as JSON [env: FABRO_JSON=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
let mut zsh = cli.command();
zsh.args(["completion", "zsh"]);
zsh.assert().success();
let mut fish = cli.command();
fish.args(["completion", "fish"]);
fish.assert().success();
}
#[test]
fn attach_smoke_covers_arg_validation_and_remote_server_behaviors() {
let context = test_context!();
let mut missing_arg = context.command();
missing_arg.arg("attach");
fabro_snapshot!(context.filters(), missing_arg, @"
success: false
exit_code: 2
----- stdout -----
----- stderr -----
error: the following required arguments were not provided:
<RUN>
Usage: fabro attach --no-upgrade-check <RUN>
For more information, try '--help'.
");
let success_server = MockServer::start();
let success_run_id = unique_run_id();
let resolve_mock = success_server.mock(|when, then| {
when.method("GET")
.path("/api/v1/runs/resolve")
.query_param("selector", success_run_id.as_str());
then.status(200)
.header("Content-Type", "application/json")
.body(
remote_run_summary_json(
&success_run_id,
"Remote Workflow",
"remote-workflow",
"Remote output",
&serde_json::json!({
"kind": "running"
}),
"2026-04-05T12:00:00Z",
)
.to_string(),
);
});
success_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{success_run_id}/events"));
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [{
"seq": 1,
"event": "run.running",
"id": "evt-run-running",
"run_id": success_run_id,
"ts": "2026-04-05T12:00:00Z",
"properties": {}
}],
"meta": { "has_more": false }
})
.to_string(),
);
});
success_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{success_run_id}/state"));
then.status(200)
.header("Content-Type", "application/json")
.body(live_run_state_response(success_run_id.as_str()).to_string());
});
success_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{success_run_id}/questions"))
.query_param("page[limit]", "100")
.query_param("page[offset]", "0");
then.status(200)
.header("Content-Type", "application/json")
.body(r#"{"data":[],"meta":{"has_more":false}}"#);
});
let attach_mock = success_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{success_run_id}/attach"))
.query_param("since_seq", "2");
then.status(200)
.header("Content-Type", "text/event-stream")
.body(run_sse_body(success_run_id.as_str()));
});
context.set_http_target(&success_server.base_url());
let success_output = context
.command()
.args(["--json", "attach", &success_run_id])
.output()
.expect("attach should execute");
assert!(
success_output.status.success(),
"attach failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&success_output.stdout),
String::from_utf8_lossy(&success_output.stderr)
);
resolve_mock.assert();
attach_mock.assert();
let success_stdout = String::from_utf8(success_output.stdout).expect("stdout should be UTF-8");
assert!(
success_stdout.contains("\"event\":\"run.completed\""),
"{success_stdout}"
);
let eof_server = MockServer::start();
let eof_run_id = unique_run_id();
eof_server.mock(|when, then| {
when.method("GET")
.path("/api/v1/runs/resolve")
.query_param("selector", eof_run_id.as_str());
then.status(200)
.header("Content-Type", "application/json")
.body(
remote_run_summary_json(
&eof_run_id,
"Remote Workflow",
"remote-workflow",
"Remote output",
&serde_json::json!({
"kind": "running"
}),
"2026-04-05T12:00:00Z",
)
.to_string(),
);
});
eof_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{eof_run_id}/events"));
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [{
"seq": 1,
"event": "run.running",
"id": "evt-run-running",
"run_id": eof_run_id,
"ts": "2026-04-05T12:00:00Z",
"properties": {}
}],
"meta": { "has_more": false }
})
.to_string(),
);
});
eof_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{eof_run_id}/state"));
then.status(200)
.header("Content-Type", "application/json")
.body(live_run_state_response(eof_run_id.as_str()).to_string());
});
eof_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{eof_run_id}/questions"))
.query_param("page[limit]", "100")
.query_param("page[offset]", "0");
then.status(200)
.header("Content-Type", "application/json")
.body(r#"{"data":[],"meta":{"has_more":false}}"#);
});
eof_server.mock(|when, then| {
when.method("GET")
.path(format!("/api/v1/runs/{eof_run_id}/attach"))
.query_param("since_seq", "2");
then.status(200)
.header("Content-Type", "text/event-stream")
.body("");
});
context.set_http_target(&eof_server.base_url());
let eof_output = context
.command()
.args(["attach", &eof_run_id])
.output()
.expect("attach should execute");
assert!(
!eof_output.status.success(),
"attach should fail on premature EOF"
);
let eof_stderr = String::from_utf8(eof_output.stderr).expect("stderr should be UTF-8");
assert!(
eof_stderr.contains("terminal run event"),
"expected a protocol error, got:\n{eof_stderr}"
);
}