From 02a87cb65007bef2b267ea238cad82e3e7ef5b68 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp <19+brynary@users.noreply.github.com> Date: Mon, 25 May 2026 18:28:18 -0400 Subject: [PATCH] fix(settings): show scheduler slot usage (#404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the Settings Resources concurrency meter so it reports scheduler capacity usage instead of all non-terminal runs. `/api/v1/system/info` now exposes `runs.scheduler_slots_used`, computed from the same status predicate the scheduler uses, while `runs.active` remains unchanged for existing lifecycle semantics. The settings page uses only the new slot count, so pending approval runs and runnable queued runs no longer make the concurrency meter look full. ## Verification - `cargo build -p fabro-api` - `cargo nextest run -p fabro-server --features test-support worker_started_child_run_requires_approval_before_becoming_runnable` - `cargo nextest run -p fabro-server --features test-support scheduler_capacity_counts_only_runs_occupying_slots` - `cargo nextest run -p fabro-server --features test-support get_system_info_returns_runtime_fields` - `cargo nextest run -p fabro-server --features test-support test_app_state_with_options_respects_max_concurrent_runs` - `cargo nextest run -p fabro-server --features test-support openapi_conformance` - `bun test app/routes/settings-monitoring.test.tsx` - `bun run typecheck` --- [![Compound Engineering](https://img.shields.io/badge/Compound_Engineering-6366f1)](https://github.com/EveryInc/compound-engineering-plugin) 🤖 Generated with GPT-5 (context unknown, reasoning unknown) via [Codex](https://openai.com/codex) --- .../app/routes/settings-monitoring.test.tsx | 5 ++- .../app/routes/settings-monitoring.tsx | 11 ++---- docs/public/api-reference/fabro-api.yaml | 4 ++ lib/crates/fabro-server/src/server.rs | 20 +++++----- .../fabro-server/src/server/handler/system.rs | 19 +++++---- lib/crates/fabro-server/src/server/tests.rs | 39 +++++++++++++++++++ .../fabro-server/tests/it/api/system.rs | 10 +++++ .../src/models/system-run-counts.ts | 4 ++ 8 files changed, 87 insertions(+), 25 deletions(-) diff --git a/apps/fabro-web/app/routes/settings-monitoring.test.tsx b/apps/fabro-web/app/routes/settings-monitoring.test.tsx index 10227b98a..070276f8d 100644 --- a/apps/fabro-web/app/routes/settings-monitoring.test.tsx +++ b/apps/fabro-web/app/routes/settings-monitoring.test.tsx @@ -106,7 +106,7 @@ function sampleServerSettings(maxConcurrentRuns = 8): ServerSettings { describe("SettingsMonitoring route", () => { beforeEach(() => { teardownReactTestEnv = setupReactTestEnv(); - systemInfo = { runs: { active: 3, total: 12 } }; + systemInfo = { runs: { active: 3, scheduler_slots_used: 1, total: 12 } }; serverSettings = sampleServerSettings(); }); @@ -133,7 +133,8 @@ describe("SettingsMonitoring route", () => { expect(text).toContain("5s"); expect(text).toContain("3 GiB"); expect(text).toContain("8 GiB"); - expect(text).toContain("3 / 8 active"); + expect(text).toContain("1 / 8 slots used"); + expect(text).not.toContain("3 / 8 active"); }); test("shows CPU warmup state while usage is null", () => { diff --git a/apps/fabro-web/app/routes/settings-monitoring.tsx b/apps/fabro-web/app/routes/settings-monitoring.tsx index b058e6d27..cde3d733c 100644 --- a/apps/fabro-web/app/routes/settings-monitoring.tsx +++ b/apps/fabro-web/app/routes/settings-monitoring.tsx @@ -59,17 +59,14 @@ function RunsPanel() { return ; } - const active = info.runs?.active ?? 0; + const slotsUsed = info.runs?.scheduler_slots_used ?? 0; const max = settings.server.scheduler.max_concurrent_runs; - const percent = max > 0 ? (active / max) * 100 : null; + const percent = max > 0 ? (slotsUsed / max) * 100 : null; return ( - - + + ); diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index dfa032fe0..c998b3ea2 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -12291,6 +12291,10 @@ components: type: integer format: int64 description: Runs currently pending, runnable, or executing. + scheduler_slots_used: + type: integer + format: int64 + description: Runs currently occupying scheduler concurrency slots. SystemResourcesResponse: description: Server-visible runtime resource usage for the active Fabro process environment. diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 0c6c602d0..d224b9991 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -2467,6 +2467,16 @@ fn compute_queue_positions(runs: &HashMap) -> HashMap bool { + matches!( + status, + RunStatus::Starting + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } + ) +} + #[allow( clippy::result_large_err, reason = "Run ID parsing returns HTTP 400 responses directly." @@ -4063,15 +4073,7 @@ pub fn spawn_scheduler(state: Arc) { let runs = state.runs.lock().expect("runs lock poisoned"); let active = runs .values() - .filter(|r| { - matches!( - r.status, - RunStatus::Starting - | RunStatus::Running - | RunStatus::Blocked { .. } - | RunStatus::Paused { .. } - ) - }) + .filter(|r| counts_toward_scheduler_capacity(r.status)) .count(); let available = state.max_concurrent_runs.saturating_sub(active); if available == 0 { diff --git a/lib/crates/fabro-server/src/server/handler/system.rs b/lib/crates/fabro-server/src/server/handler/system.rs index ce836f39b..667d7d81d 100644 --- a/lib/crates/fabro-server/src/server/handler/system.rs +++ b/lib/crates/fabro-server/src/server/handler/system.rs @@ -7,9 +7,9 @@ use super::super::{ BillingByModel, DfParams, FABRO_VERSION, GithubIntegrationStrategy, IntoResponse, Json, Path, PruneRunsRequest, PruneRunsResponse, Query, RequiredUser, Response, Router, RunStatus, State, StatusCode, SystemInfoResponse, SystemRepairRunIssue, SystemRepairRunsResponse, - SystemRunCounts, build_disk_usage_response, build_prune_plan, delete_run_internal, diagnostics, - get, post, resolve_interp_string, resource_sampler, spawn_blocking, system_sandbox_provider, - to_i64, + SystemRunCounts, build_disk_usage_response, build_prune_plan, counts_toward_scheduler_capacity, + delete_run_internal, diagnostics, get, post, resolve_interp_string, resource_sampler, + spawn_blocking, system_sandbox_provider, to_i64, }; pub(super) fn routes() -> Router> { @@ -44,7 +44,7 @@ async fn get_server_settings(_auth: RequiredUser, State(state): State>) -> Response { let manifest_run_settings = state.manifest_run_settings(); let server_settings = state.server_settings(); - let (total_runs, active_runs) = { + let (total_runs, active_runs, scheduler_slots_used) = { let runs = state.runs.lock().expect("runs lock poisoned"); let active = runs .values() @@ -60,7 +60,11 @@ async fn get_system_info(_auth: RequiredUser, State(state): State> ) }) .count(); - (runs.len(), active) + let scheduler_slots_used = runs + .values() + .filter(|run| counts_toward_scheduler_capacity(run.status)) + .count(); + (runs.len(), active, scheduler_slots_used) }; let response = SystemInfoResponse { @@ -75,8 +79,9 @@ async fn get_system_info(_auth: RequiredUser, State(state): State> storage_dir: Some(state.server_storage_dir().display().to_string()), uptime_secs: Some(to_i64(state.started_at.elapsed().as_secs())), runs: Some(SystemRunCounts { - total: Some(to_i64(total_runs)), - active: Some(to_i64(active_runs)), + total: Some(to_i64(total_runs)), + active: Some(to_i64(active_runs)), + scheduler_slots_used: Some(to_i64(scheduler_slots_used)), }), sandbox_provider: Some(system_sandbox_provider(&manifest_run_settings)), }; diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index b8ba80cbd..1c2a81375 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -9513,6 +9513,20 @@ async fn worker_started_child_run_requires_approval_before_becoming_runnable() { Some("pending") ); + let response = app + .clone() + .oneshot(bearer_request( + Method::GET, + "/system/info", + &user_jwt, + Body::empty(), + )) + .await + .unwrap(); + let info_body = response_json!(response, StatusCode::OK).await; + assert_eq!(info_body["runs"]["active"], 1); + assert_eq!(info_body["runs"]["scheduler_slots_used"], 0); + { let runs = state.runs.lock().expect("runs lock poisoned"); assert_eq!( @@ -13052,6 +13066,31 @@ async fn queue_position_reported_for_runnable_runs() { assert_eq!(positions.get(&second_id).copied(), Some(2)); } +#[test] +fn scheduler_capacity_counts_only_runs_occupying_slots() { + assert!(!counts_toward_scheduler_capacity(RunStatus::Submitted)); + assert!(!counts_toward_scheduler_capacity(RunStatus::Pending { + reason: PendingReason::ApprovalRequired, + })); + assert!(!counts_toward_scheduler_capacity(RunStatus::Runnable)); + assert!(counts_toward_scheduler_capacity(RunStatus::Starting)); + assert!(counts_toward_scheduler_capacity(RunStatus::Running)); + assert!(counts_toward_scheduler_capacity(RunStatus::Blocked { + blocked_reason: BlockedReason::HumanInputRequired, + })); + assert!(counts_toward_scheduler_capacity(RunStatus::Paused { + prior_block: None, + })); + assert!(!counts_toward_scheduler_capacity(RunStatus::Removing)); + assert!(!counts_toward_scheduler_capacity(RunStatus::Succeeded { + reason: SuccessReason::Completed, + })); + assert!(!counts_toward_scheduler_capacity(RunStatus::Failed { + reason: FailureReason::WorkflowError, + })); + assert!(!counts_toward_scheduler_capacity(RunStatus::Dead)); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn concurrency_limit_respected() { let state = test_app_state_with_options(default_test_server_settings(), RunLayer::default(), 1); diff --git a/lib/crates/fabro-server/tests/it/api/system.rs b/lib/crates/fabro-server/tests/it/api/system.rs index a798d5c26..e7b2abbe0 100644 --- a/lib/crates/fabro-server/tests/it/api/system.rs +++ b/lib/crates/fabro-server/tests/it/api/system.rs @@ -266,6 +266,16 @@ async fn test_app_state_with_options_respects_max_concurrent_runs() { .is_some_and(std::vec::Vec::is_empty), "second run should still be waiting for scheduler capacity while the first waits at the human gate: {second_questions}" ); + + let request = Request::builder() + .method("GET") + .uri(api("/system/info")) + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + let body = response_json(response, StatusCode::OK, "GET /api/v1/system/info").await; + assert_eq!(body["runs"]["active"], 2); + assert_eq!(body["runs"]["scheduler_slots_used"], 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/lib/packages/fabro-api-client/src/models/system-run-counts.ts b/lib/packages/fabro-api-client/src/models/system-run-counts.ts index ad6854320..3fe29ae9b 100644 --- a/lib/packages/fabro-api-client/src/models/system-run-counts.ts +++ b/lib/packages/fabro-api-client/src/models/system-run-counts.ts @@ -26,4 +26,8 @@ export interface SystemRunCounts { * Runs currently pending, runnable, or executing. */ 'active'?: number; + /** + * Runs currently occupying scheduler concurrency slots. + */ + 'scheduler_slots_used'?: number; }