fabro/lib/apps/fabro-server/src/server/tests.rs

use std::collections::HashMap;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
#[cfg(unix)]
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc as StdArc, Mutex as StdMutex};

use async_zip::base::read::mem::ZipFileReader;
use axum::body::Body;
use axum::http::{Method, Request, header};
use chrono::{Duration as ChronoDuration, SubsecRound as _, Utc};
use fabro_automation::{AutomationId, AutomationTarget};
use fabro_config::bind::Bind;
use fabro_config::{
    EnvironmentLayer, MergeMap, RunLayer, ServerSettingsBuilder, WorkflowSettingsBuilder,
};
use fabro_interview::{
    AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlDeliveryFrame,
    WorkerControlEnvelope, WorkerControlMessage,
};
use fabro_llm::types::{Message as LlmMessage, Request as LlmRequest, TokenCounts};
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, ModelRef, ProviderId, ReasoningEffort, Speed};
use fabro_types::settings::ServerAuthMethod;
use fabro_types::settings::run::EnvironmentProvider;
use fabro_types::{
    AgentBackend, AttrValue, AuthMethod, BlobHash, CommandTermination, FailureCategory,
    FailureDetail, Graph, InterviewQuestionRecord, Node, Outcome, ParallelBranchId, QuestionType,
    RunId, RunSpec, SandboxProviderKind, StageContextWindowBreakdownItem,
    StageContextWindowCategory, StageContextWindowCountMethod, StageContextWindowProjection,
    StageContextWindowStaleness, StageContextWindowWarning, StageModelUsage, StageTiming,
    SuccessReason, SystemActorKind, WorkflowSettings, fixtures, test_support,
};
use fabro_util::check_report::CheckStatus;
use fabro_workflow::records::CheckpointExt;
use httpmock::Method::{GET, POST};
use httpmock::MockServer;
use serde_json::json;
use tokio::sync::Notify;
use tokio_stream::StreamExt as _;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::protocol::Message as WebSocketMessage;
use tower::ServiceExt;
use tracing::field::{Field, Visit};
use tracing::{Event as TracingEvent, Subscriber, subscriber};
use tracing_subscriber::layer::Context as SubscriberContext;
use tracing_subscriber::prelude::*;
use tracing_subscriber::{Layer, Registry};

use super::*;
use crate::automation_materializer::AutomationRunMaterializeInput;
use crate::github_webhooks::compute_signature;
use crate::jwt_auth::{AuthMode, ConfiguredAuth};
use crate::test_support::*;
use crate::worker_control::{
    LocalWorkerControlBus, WorkerControlBus, WorkerControlCursor, WorkerControlReceiver,
};
use crate::worker_runtime::{
    LocalWorkerRuntime, StartedWorker, WorkerLaunchSpec, WorkerRef, WorkerRuntime,
};

const MINIMAL_DOT: &str = r#"digraph Test {
    graph [goal="Test"]
    start [shape=Mdiamond]
    exit  [shape=Msquare]
    start -> exit
}"#;
const TEST_WEBHOOK_SECRET: &str = "webhook-secret";
const TEST_DEV_TOKEN: &str =
    "fabro_dev_abababababababababababababababababababababababababababababababab";
const TEST_SESSION_SECRET: &str = "server-test-session-key-0123456789";
const TEST_JWT_ISSUER: &str = "https://fabro.example";
const WRONG_DEV_TOKEN: &str =
    "fabro_dev_cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd";

fn manifest_run_defaults_from_toml(source: &str) -> fabro_config::RunLayer {
    let mut document: toml::Table = source.parse().expect("run defaults should parse");
    document
        .remove("run")
        .map(toml::Value::try_into::<fabro_config::RunLayer>)
        .transpose()
        .expect("run defaults should parse")
        .unwrap_or_default()
}

fn test_environment_store(
    default_provider: Option<EnvironmentProvider>,
    local_enabled: bool,
) -> (tempfile::TempDir, EnvironmentStore) {
    let temp = tempfile::tempdir().expect("environment store tempdir should be created");
    let db_path = temp.path().join("fabro.sqlite3");
    let pool = std::thread::spawn(move || {
        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .expect("environment store setup runtime should build");
        runtime.block_on(async move {
            let database = fabro_db::Database::connect(db_path)
                .await
                .expect("test environment database should connect");
            database
                .migrate()
                .await
                .expect("test environment database should migrate");
            if let Some(provider) = default_provider {
                fabro_environment::seed_default_environment(database.pool(), provider)
                    .await
                    .expect("test default environment should seed");
            }
            database.clone_pool()
        })
    })
    .join()
    .expect("environment store setup thread should not panic");
    let store = load_store_blocking("environment store", move || async move {
        EnvironmentStore::load(pool, local_enabled)
            .await
            .map_err(anyhow::Error::new)
    })
    .expect("test environment store should load");
    (temp, store)
}

fn test_mcp_server_store() -> (tempfile::TempDir, McpServerStore) {
    let temp = tempfile::tempdir().expect("MCP server store tempdir should be created");
    let db_path = temp.path().join("fabro.sqlite3");
    let pool = std::thread::spawn(move || {
        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .expect("MCP server store setup runtime should build");
        runtime.block_on(async move {
            let database = fabro_db::Database::connect(db_path)
                .await
                .expect("test MCP server database should connect");
            database
                .migrate()
                .await
                .expect("test MCP server database should migrate");
            database.clone_pool()
        })
    })
    .join()
    .expect("MCP server store setup thread should not panic");
    let mcps_dir = temp.path().join("mcps");
    let store = load_store_blocking("MCP server store", move || async move {
        McpServerStore::open(pool, mcps_dir)
            .await
            .map_err(anyhow::Error::new)
    })
    .expect("test MCP server store should load");
    (temp, store)
}

fn server_settings_from_toml(source: &str) -> ServerSettings {
    ServerSettingsBuilder::from_toml(source).expect("server settings should resolve")
}

fn resolved_runtime_settings_from_toml(source: &str) -> ResolvedAppStateSettings {
    resolved_runtime_settings_for_tests(
        server_settings_from_toml(source),
        manifest_run_defaults_from_toml(source),
        LlmCatalogSettings::default(),
    )
}

fn test_app_with() -> Router {
    let state = test_app_state();
    crate::test_support::build_test_router_with_options(state, RouterOptions {
        static_asset_root: Some(spa_fixture_root()),
        ..RouterOptions::default()
    })
}

fn spa_fixture_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/spa")
}

fn state_test_catalog() -> Arc<Catalog> {
    Arc::new(Catalog::from_builtin().expect("default catalog should build"))
}

fn test_app_with_scheduler(state: Arc<AppState>) -> Router {
    spawn_scheduler(Arc::clone(&state));
    crate::test_support::build_test_router(state)
}

fn test_app_state_with_isolated_storage() -> Arc<AppState> {
    let storage_dir = std::env::temp_dir().join(format!("fabro-server-test-{}", Ulid::new()));
    std::fs::create_dir_all(&storage_dir).expect("test storage dir should be creatable");
    let source = format!(
        r#"
_version = 1

[server.storage]
root = "{}"

[server.auth]
methods = ["dev-token"]
"#,
        storage_dir.display()
    );

    test_app_state_with_options(
        server_settings_from_toml(&source),
        manifest_run_defaults_from_toml(&source),
        5,
    )
}

async fn body_json(body: Body) -> serde_json::Value {
    let bytes = to_bytes(body, usize::MAX).await.unwrap();
    serde_json::from_slice(&bytes).unwrap()
}

fn run_json_id(run: &serde_json::Value) -> Option<&str> {
    run["id"].as_str().or_else(|| run["run_id"].as_str())
}

fn run_json_status(run: &serde_json::Value) -> &serde_json::Value {
    &run["lifecycle"]["status"]
}

fn run_json_pending_control(run: &serde_json::Value) -> &serde_json::Value {
    &run["lifecycle"]["pending_control"]
}

fn run_json_archived(run: &serde_json::Value) -> bool {
    run["lifecycle"]["archived"].as_bool().unwrap_or(false)
}

async fn mock_daytona_auth_probe(server: &MockServer) -> httpmock::Mock<'_> {
    server
        .mock_async(|when, then| {
            when.method(GET)
                .path("/sandbox/paginated")
                .query_param("page", "1")
                .query_param("limit", "1");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(json!({
                    "items": [],
                    "total": 0,
                    "page": 1,
                    "totalPages": 0
                }));
        })
        .await
}

async fn mock_daytona_current_key<'a>(
    server: &'a MockServer,
    permissions: Vec<&'static str>,
) -> httpmock::Mock<'a> {
    server
        .mock_async(move |when, then| {
            when.method(GET)
                .path("/api-keys/current")
                .header("authorization", "Bearer dtn_test");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(json!({
                    "name": "delete-only",
                    "value": "dtn_****",
                    "createdAt": "2026-05-01T00:00:00Z",
                    "permissions": permissions,
                    "lastUsedAt": null,
                    "expiresAt": null,
                    "userId": "user_123"
                }));
        })
        .await
}

fn openai_oauth_credential() -> fabro_auth::OAuthCredential {
    fabro_auth::OAuthCredential {
        tokens:     fabro_auth::OAuthTokens {
            access_token:  "access".to_string(),
            refresh_token: Some("refresh".to_string()),
            expires_at:    Utc::now() + ChronoDuration::hours(1),
        },
        config:     fabro_auth::OAuthConfig {
            auth_url:     "https://auth.openai.com".to_string(),
            token_url:    "https://auth.openai.com/oauth/token".to_string(),
            client_id:    "client".to_string(),
            scopes:       vec!["openid".to_string()],
            redirect_uri: Some("https://auth.openai.com/deviceauth/callback".to_string()),
            use_pkce:     true,
        },
        account_id: Some("acct_123".to_string()),
    }
}

fn openai_oauth_credential_json() -> String {
    serde_json::to_string(&openai_oauth_credential()).unwrap()
}

fn openai_responses_payload(text: &str) -> serde_json::Value {
    json!({
        "id": "resp_1",
        "model": "gpt-5.4",
        "output": [
            {
                "type": "message",
                "role": "assistant",
                "content": [
                    {
                        "type": "output_text",
                        "text": text
                    }
                ]
            }
        ],
        "status": "completed",
        "usage": {
            "input_tokens": 10,
            "output_tokens": 20
        }
    })
}

macro_rules! assert_status {
    ($response:expr, $expected:expr) => {
        fabro_test::assert_axum_status($response, $expected, concat!(file!(), ":", line!()))
    };
}

macro_rules! checked_response {
    ($response:expr, $expected:expr) => {
        fabro_test::expect_axum_status($response, $expected, concat!(file!(), ":", line!()))
    };
}

#[derive(Clone, Debug)]
struct CapturedTracingEvent {
    fields: Vec<(String, String)>,
}

#[derive(Default)]
struct CaptureVisitor {
    fields: Vec<(String, String)>,
}

impl Visit for CaptureVisitor {
    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
        self.fields
            .push((field.name().to_string(), format!("{value:?}")));
    }

    fn record_str(&mut self, field: &Field, value: &str) {
        self.fields
            .push((field.name().to_string(), value.to_string()));
    }

    fn record_i64(&mut self, field: &Field, value: i64) {
        self.fields
            .push((field.name().to_string(), value.to_string()));
    }

    fn record_u64(&mut self, field: &Field, value: u64) {
        self.fields
            .push((field.name().to_string(), value.to_string()));
    }
}

struct ServerLogCaptureLayer {
    events: StdArc<StdMutex<Vec<CapturedTracingEvent>>>,
}

impl<S: Subscriber> Layer<S> for ServerLogCaptureLayer {
    fn on_event(&self, event: &TracingEvent<'_>, _ctx: SubscriberContext<'_, S>) {
        if !event
            .metadata()
            .target()
            .starts_with("fabro_server::server")
        {
            return;
        }
        let mut visitor = CaptureVisitor::default();
        event.record(&mut visitor);
        if visitor
            .fields
            .iter()
            .any(|(name, value)| name == "message" && value == "HTTP response")
        {
            self.events
                .lock()
                .expect("captured log events lock poisoned")
                .push(CapturedTracingEvent {
                    fields: visitor.fields,
                });
        }
    }
}

fn capture_server_logs() -> (
    tracing::dispatcher::DefaultGuard,
    StdArc<StdMutex<Vec<CapturedTracingEvent>>>,
) {
    let events = StdArc::new(StdMutex::new(Vec::new()));
    let subscriber = Registry::default().with(ServerLogCaptureLayer {
        events: StdArc::clone(&events),
    });
    let guard = subscriber::set_default(subscriber);
    (guard, events)
}

fn captured_field<'a>(event: &'a CapturedTracingEvent, name: &str) -> Option<&'a str> {
    event
        .fields
        .iter()
        .find_map(|(field_name, value)| (field_name == name).then_some(value.as_str()))
}

fn assert_log_field(event: &CapturedTracingEvent, name: &str, expected: &str) {
    let actual = captured_field(event, name)
        .unwrap_or_else(|| panic!("expected log field {name}; fields were {:?}", event.fields));
    let debug_expected = format!("{expected:?}");
    assert!(
        actual == expected || actual == debug_expected,
        "expected field {name} to be {expected:?}, got {actual:?}; fields were {:?}",
        event.fields
    );
}

fn assert_log_field_absent(event: &CapturedTracingEvent, name: &str) {
    assert!(
        captured_field(event, name).is_none(),
        "expected log field {name} to be absent; fields were {:?}",
        event.fields
    );
}

macro_rules! response_json {
    ($response:expr, $expected:expr) => {
        fabro_test::expect_axum_json($response, $expected, concat!(file!(), ":", line!()))
    };
}

macro_rules! response_bytes {
    ($response:expr, $expected:expr) => {
        fabro_test::expect_axum_bytes($response, $expected, concat!(file!(), ":", line!()))
    };
}

fn api(path: &str) -> String {
    format!("/api/v1{path}")
}

#[tokio::test(flavor = "current_thread")]
async fn http_log_omits_unset_optional_auth_fields() {
    let (_guard, events) = capture_server_logs();
    let app = test_app_with();

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/health")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let events = events.lock().expect("captured log events").clone();
    assert_eq!(events.len(), 1);
    let field_names = events[0]
        .fields
        .iter()
        .map(|(name, _)| name.as_str())
        .collect::<Vec<_>>();
    assert!(field_names.contains(&"principal_kind"));
    assert!(field_names.contains(&"auth_status"));
    assert!(!field_names.contains(&"auth_error_code"));
    assert!(!field_names.contains(&"user_auth_method"));
    assert!(!field_names.contains(&"idp_issuer"));
    assert!(!field_names.contains(&"run_id"));
}

#[tokio::test(flavor = "current_thread")]
async fn http_log_records_user_principal_fields() {
    let (_state, app) = jwt_auth_app();
    let bearer = issue_test_user_jwt();
    let (_guard, events) = capture_server_logs();

    let response = app
        .oneshot(bearer_request(Method::GET, "/runs", &bearer, Body::empty()))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let events = events.lock().expect("captured log events").clone();
    assert_eq!(events.len(), 1);
    let event = &events[0];
    assert_log_field(event, "principal_kind", "user");
    assert_log_field(event, "auth_status", "authenticated");
    assert_log_field(event, "user_auth_method", "github");
    assert_log_field(event, "idp_issuer", "https://github.com");
    assert_log_field(event, "idp_subject", "12345");
    assert_log_field(event, "login", "octocat");
    assert_log_field_absent(event, "auth_error_code");
}

#[tokio::test(flavor = "current_thread")]
async fn http_log_records_worker_principal_fields() {
    let (_state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_bearer).await;
    let worker_bearer = issue_test_worker_token(&run_id);
    let (_guard, events) = capture_server_logs();

    let response = app
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/state"),
            &worker_bearer,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let events = events.lock().expect("captured log events").clone();
    assert_eq!(events.len(), 1);
    let event = &events[0];
    assert_log_field(event, "principal_kind", "worker");
    assert_log_field(event, "auth_status", "authenticated");
    assert_log_field(event, "run_id", &run_id.to_string());
    assert_log_field_absent(event, "auth_error_code");
}

#[tokio::test(flavor = "current_thread")]
async fn http_log_records_webhook_principal_fields() {
    let body = br#"{"repository":{"full_name":"owner/repo"},"action":"opened"}"#;
    let signature = compute_signature(TEST_WEBHOOK_SECRET.as_bytes(), body);
    let app = webhook_test_app(dev_token_auth_mode());
    let (_guard, events) = capture_server_logs();

    let response = app
        .oneshot(webhook_request(Some(&signature), None, body))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let events = events.lock().expect("captured log events").clone();
    assert_eq!(events.len(), 1);
    let event = &events[0];
    assert_log_field(event, "principal_kind", "webhook");
    assert_log_field(event, "auth_status", "authenticated");
    assert_log_field(event, "delivery_id", "delivery-1");
    assert_log_field_absent(event, "auth_error_code");
}

#[allow(
    clippy::needless_pass_by_value,
    reason = "Test helper mirrors the public build_router convenience API."
)]
fn webhook_test_app(auth_mode: AuthMode) -> Router {
    let state = TestAppStateBuilder::new()
        .env_lookup(|_| None)
        .vault_entries([(WEBHOOK_SECRET_ENV, TEST_WEBHOOK_SECRET)])
        .build();
    build_router_with_options(state, &auth_mode, RouterOptions {
        web_enabled: false,
        ..RouterOptions::default()
    })
}

fn webhook_request(
    signature: Option<&str>,
    authorization: Option<&str>,
    body: &[u8],
) -> Request<Body> {
    let mut builder = Request::builder()
        .method("POST")
        .uri(api("/webhooks/github"))
        .header("x-github-delivery", "delivery-1")
        .header("x-github-event", "pull_request");
    if let Some(sig) = signature {
        builder = builder.header("x-hub-signature-256", sig);
    }
    if let Some(value) = authorization {
        builder = builder.header(header::AUTHORIZATION, value);
    }
    builder.body(Body::from(body.to_vec())).unwrap()
}

fn dev_token_auth_mode() -> AuthMode {
    AuthMode::Enabled(ConfiguredAuth {
        methods:    vec![ServerAuthMethod::DevToken],
        dev_token:  Some(TEST_DEV_TOKEN.to_string()),
        jwt_key:    None,
        jwt_issuer: None,
    })
}

fn jwt_auth_mode() -> AuthMode {
    AuthMode::Enabled(ConfiguredAuth {
        methods:    vec![ServerAuthMethod::Github],
        dev_token:  None,
        jwt_key:    Some(
            auth::derive_jwt_key(TEST_SESSION_SECRET.as_bytes())
                .expect("test JWT key should derive"),
        ),
        jwt_issuer: Some(TEST_JWT_ISSUER.to_string()),
    })
}

fn jwt_auth_state() -> Arc<AppState> {
    test_app_state_with_session_key(
        default_test_server_settings(),
        RunLayer::default(),
        Some(TEST_SESSION_SECRET),
    )
}

fn jwt_auth_app() -> (Arc<AppState>, Router) {
    let state = jwt_auth_state();
    let app = build_router(Arc::clone(&state), jwt_auth_mode());
    (state, app)
}

fn test_user_subject() -> auth::JwtSubject {
    auth::JwtSubject {
        identity:    fabro_types::IdpIdentity::new("https://github.com", "12345").unwrap(),
        login:       "octocat".to_string(),
        name:        "The Octocat".to_string(),
        email:       "octocat@example.com".to_string(),
        avatar_url:  "https://example.com/octocat.png".to_string(),
        user_url:    "https://github.com/octocat".to_string(),
        auth_method: AuthMethod::Github,
    }
}

fn issue_test_user_jwt() -> String {
    let key =
        auth::derive_jwt_key(TEST_SESSION_SECRET.as_bytes()).expect("test JWT key should derive");
    auth::issue(
        &key,
        TEST_JWT_ISSUER,
        &test_user_subject(),
        ChronoDuration::minutes(10),
    )
}

fn issue_test_worker_token(run_id: &RunId) -> String {
    let keys = WorkerTokenKeys::from_master_secret(TEST_SESSION_SECRET.as_bytes())
        .expect("worker keys should derive");
    crate::worker_token::issue_worker_token(&keys, run_id).expect("worker token should issue")
}

fn issue_test_run_tools_worker_token(run_id: &RunId) -> String {
    let keys = WorkerTokenKeys::from_master_secret(TEST_SESSION_SECRET.as_bytes())
        .expect("worker keys should derive");
    crate::worker_token::issue_worker_token_with_scopes(
        &keys,
        run_id,
        crate::worker_token::WorkerScopeSet::run_worker_with_agent_run_tools(),
    )
    .expect("worker token should issue")
}

async fn create_run_with_bearer(app: &Router, bearer: &str) -> RunId {
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header(header::AUTHORIZATION, format!("Bearer {bearer}"))
                .header(header::CONTENT_TYPE, "application/json")
                .body(manifest_body(MINIMAL_DOT))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    body["id"].as_str().unwrap().parse().unwrap()
}

fn pair_test_target() -> PairTarget {
    PairTarget {
        stage_id:   StageId::new("agent", 1),
        node_label: "Agent".to_string(),
    }
}

async fn append_pair_transcript_fixture(state: &Arc<AppState>, run_id: RunId) -> PairId {
    let pair_id = "01HZX6M29F1CD5YYMHT1F5D7WQ".parse().unwrap();
    let run_store = state
        .stores
        .runs
        .open_run(&run_id)
        .await
        .expect("test run should be openable");
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::RunPairStarted {
            pair_id,
            target: pair_test_target(),
            actor: None,
        },
    )
    .await
    .unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::AgentPairUserMessage {
            node_id: "agent".to_string(),
            visit: 1,
            session_id: "session-1".to_string(),
            pair_id,
            message_id: PairMessageId::new(),
            client_message_id: None,
            text: "hello pair".to_string(),
            actor: None,
        },
    )
    .await
    .unwrap();
    pair_id
}

fn bearer_request(method: Method, path: &str, bearer: &str, body: Body) -> Request<Body> {
    Request::builder()
        .method(method)
        .uri(api(path))
        .header(header::AUTHORIZATION, format!("Bearer {bearer}"))
        .body(body)
        .unwrap()
}

struct WorkerControlWsTestServer {
    base_url: String,
    task:     tokio::task::JoinHandle<()>,
}

impl WorkerControlWsTestServer {
    async fn spawn(app: Router) -> Self {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
            .await
            .expect("test WebSocket listener should bind");
        let addr = listener
            .local_addr()
            .expect("test WebSocket listener should have a local address");
        let task = tokio::spawn(async move {
            let result = axum::serve(
                listener,
                app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
            )
            .await;
            if let Err(err) = result {
                tracing::debug!(error = %err, "test WebSocket server stopped");
            }
        });
        Self {
            base_url: format!("ws://{addr}"),
            task,
        }
    }

    fn worker_control_url(&self, run_id: RunId, after: Option<&str>) -> String {
        let mut url = format!(
            "{}/api/v1/runs/{run_id}/worker/control-stream",
            self.base_url
        );
        if let Some(after) = after {
            url.push_str("?after=");
            url.push_str(after);
        }
        url
    }
}

impl Drop for WorkerControlWsTestServer {
    fn drop(&mut self) {
        self.task.abort();
    }
}

fn worker_control_ws_request(
    server: &WorkerControlWsTestServer,
    run_id: RunId,
    bearer: Option<&str>,
    after: Option<&str>,
) -> Request<()> {
    let mut request = server
        .worker_control_url(run_id, after)
        .into_client_request()
        .expect("test worker-control WebSocket request should build");
    if let Some(bearer) = bearer {
        request.headers_mut().insert(
            header::AUTHORIZATION,
            format!("Bearer {bearer}")
                .parse()
                .expect("test bearer header should parse"),
        );
    }
    request
}

async fn connect_worker_control_ws(
    server: &WorkerControlWsTestServer,
    run_id: RunId,
    bearer: &str,
    after: Option<&str>,
) -> tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>> {
    let request = worker_control_ws_request(server, run_id, Some(bearer), after);
    let (socket, _) = tokio_tungstenite::connect_async(request)
        .await
        .expect("worker-control WebSocket should connect");
    socket
}

async fn assert_worker_control_ws_rejected(
    server: &WorkerControlWsTestServer,
    run_id: RunId,
    bearer: Option<&str>,
    after: Option<&str>,
    expected: StatusCode,
) {
    let request = worker_control_ws_request(server, run_id, bearer, after);
    let error = tokio_tungstenite::connect_async(request)
        .await
        .expect_err("worker-control WebSocket should be rejected");
    match error {
        tokio_tungstenite::tungstenite::Error::Http(response) => {
            assert_eq!(response.status(), expected);
        }
        other => panic!("expected HTTP rejection {expected}, got {other:#}"),
    }
}

async fn next_worker_control_frame(
    socket: &mut tokio_tungstenite::WebSocketStream<
        tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
    >,
) -> WorkerControlDeliveryFrame {
    let message = tokio::time::timeout(std::time::Duration::from_secs(2), async {
        loop {
            let message = futures_util::StreamExt::next(socket)
                .await
                .expect("worker-control WebSocket should remain open")
                .expect("worker-control WebSocket frame should be ok");
            match message {
                WebSocketMessage::Text(text) => return text,
                WebSocketMessage::Ping(payload) => {
                    futures_util::SinkExt::send(socket, WebSocketMessage::Pong(payload))
                        .await
                        .expect("test worker-control pong should send");
                }
                WebSocketMessage::Pong(_)
                | WebSocketMessage::Binary(_)
                | WebSocketMessage::Frame(_) => {}
                WebSocketMessage::Close(frame) => {
                    panic!("worker-control WebSocket closed before text frame: {frame:?}");
                }
            }
        }
    })
    .await
    .expect("worker-control frame should arrive");
    serde_json::from_str(message.as_str()).expect("worker-control delivery frame should parse")
}

#[tokio::test(flavor = "current_thread")]
async fn worker_control_stream_rejects_missing_user_and_cross_run_auth() {
    let (_state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_bearer).await;
    let worker_bearer = issue_test_worker_token(&run_id);
    let other_run_id = create_run_with_bearer(&app, &user_bearer).await;
    let other_worker_bearer = issue_test_worker_token(&other_run_id);
    let server = WorkerControlWsTestServer::spawn(app).await;

    assert_worker_control_ws_rejected(&server, run_id, None, None, StatusCode::UNAUTHORIZED).await;
    assert_worker_control_ws_rejected(
        &server,
        run_id,
        Some(&user_bearer),
        None,
        StatusCode::FORBIDDEN,
    )
    .await;
    assert_worker_control_ws_rejected(
        &server,
        run_id,
        Some(&other_worker_bearer),
        None,
        StatusCode::FORBIDDEN,
    )
    .await;

    let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await;
    futures_util::SinkExt::send(&mut socket, WebSocketMessage::Close(None))
        .await
        .unwrap();
}

#[tokio::test(flavor = "current_thread")]
async fn worker_control_stream_start_subscription_delivers_frames() {
    let (state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_bearer).await;
    let worker_bearer = issue_test_worker_token(&run_id);
    let server = WorkerControlWsTestServer::spawn(app).await;
    let mut socket = connect_worker_control_ws(&server, run_id, &worker_bearer, None).await;

    let expected = WorkerControlEnvelope::cancel_run();
    let id = state
        .worker_control_bus
        .publish(run_id, expected.clone())
        .await
        .unwrap();
    let frame = next_worker_control_frame(&mut socket).await;

    assert_eq!(frame.id, id.to_string());
    assert_eq!(frame.envelope, expected);
}

#[tokio::test(flavor = "current_thread")]
async fn worker_control_stream_after_subscription_delivers_only_later_frames() {
    let (state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_bearer).await;
    let worker_bearer = issue_test_worker_token(&run_id);
    let first = state
        .worker_control_bus
        .publish(run_id, WorkerControlEnvelope::cancel_run())
        .await
        .unwrap();
    let expected = WorkerControlEnvelope::pause_run();
    let second = state
        .worker_control_bus
        .publish(run_id, expected.clone())
        .await
        .unwrap();
    let server = WorkerControlWsTestServer::spawn(app).await;

    let mut socket =
        connect_worker_control_ws(&server, run_id, &worker_bearer, Some(first.as_str())).await;
    let frame = next_worker_control_frame(&mut socket).await;

    assert_eq!(frame.id, second.to_string());
    assert_eq!(frame.envelope, expected);
}

#[tokio::test(flavor = "current_thread")]
async fn worker_control_stream_invalid_cursor_is_http_gone_before_upgrade() {
    let (_state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_bearer).await;
    let worker_bearer = issue_test_worker_token(&run_id);
    let server = WorkerControlWsTestServer::spawn(app).await;

    assert_worker_control_ws_rejected(
        &server,
        run_id,
        Some(&worker_bearer),
        Some("local:999"),
        StatusCode::GONE,
    )
    .await;
}

#[tokio::test(flavor = "current_thread")]
async fn worker_control_stream_rejects_missing_terminal_and_archived_runs() {
    let (state, app) = jwt_auth_app();
    let user_bearer = issue_test_user_jwt();
    let missing_run_id = RunId::new();
    let missing_worker_bearer = issue_test_worker_token(&missing_run_id);
    let terminal_run_id = RunId::new();
    create_succeeded_run(&state, terminal_run_id).await;
    let terminal_worker_bearer = issue_test_worker_token(&terminal_run_id);
    let archived_run_id = RunId::new();
    create_succeeded_run(&state, archived_run_id).await;
    let archived_worker_bearer = issue_test_worker_token(&archived_run_id);
    let archive_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{archived_run_id}/archive")))
                .header(header::AUTHORIZATION, format!("Bearer {user_bearer}"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    response_json!(archive_response, StatusCode::OK).await;
    let server = WorkerControlWsTestServer::spawn(app).await;

    assert_worker_control_ws_rejected(
        &server,
        missing_run_id,
        Some(&missing_worker_bearer),
        None,
        StatusCode::NOT_FOUND,
    )
    .await;
    assert_worker_control_ws_rejected(
        &server,
        terminal_run_id,
        Some(&terminal_worker_bearer),
        None,
        StatusCode::CONFLICT,
    )
    .await;
    assert_worker_control_ws_rejected(
        &server,
        archived_run_id,
        Some(&archived_worker_bearer),
        None,
        StatusCode::CONFLICT,
    )
    .await;
}

fn json_bearer_request(
    method: Method,
    path: &str,
    bearer: &str,
    body: &serde_json::Value,
) -> Request<Body> {
    Request::builder()
        .method(method)
        .uri(api(path))
        .header(header::AUTHORIZATION, format!("Bearer {bearer}"))
        .header(header::CONTENT_TYPE, "application/json")
        .body(Body::from(serde_json::to_vec(&body).unwrap()))
        .unwrap()
}

fn json_request(method: Method, path: &str, body: &serde_json::Value) -> Request<Body> {
    Request::builder()
        .method(method)
        .uri(api(path))
        .header(header::CONTENT_TYPE, "application/json")
        .body(Body::from(serde_json::to_vec(&body).unwrap()))
        .unwrap()
}

fn canonical_origin_settings(url: &str) -> ServerSettings {
    server_settings_from_toml(&format!(
        r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "{url}"
"#
    ))
}

fn canonical_host_test_app() -> Router {
    let state = test_app_state_with_options(
        canonical_origin_settings("http://127.0.0.1:32276"),
        RunLayer::default(),
        5,
    );
    crate::test_support::build_test_router_with_options(state, RouterOptions::default())
}

#[tokio::test]
async fn router_redirects_web_page_requests_to_canonical_host() {
    let app = canonical_host_test_app();

    let response = app
        .oneshot(
            Request::builder()
                .method(Method::GET)
                .uri("/login")
                .header(header::HOST, "localhost:32276")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let response = checked_response!(response, StatusCode::PERMANENT_REDIRECT).await;
    assert_eq!(
        response.headers().get(header::LOCATION).unwrap(),
        "http://127.0.0.1:32276/login"
    );
}

#[tokio::test]
async fn router_does_not_redirect_api_requests_to_canonical_host() {
    let app = canonical_host_test_app();

    let response = app
        .oneshot(
            Request::builder()
                .method(Method::GET)
                .uri(api("/openapi.json"))
                .header(header::HOST, "localhost:32276")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::OK).await;
}

#[test]
fn replace_settings_rejects_invalid_canonical_origin_and_keeps_previous_settings() {
    for invalid in [
        "",
        "/relative/path",
        "ftp://fabro.example.com",
        "http://0.0.0.0:32276",
    ] {
        // No FABRO_WEB_URL override: web.url is plain config now, so the
        // invalid value is rejected from the settings literal and the kept
        // previous settings stay valid.
        let state = test_app_state_with_env_lookup(
            canonical_origin_settings("http://valid.example.com"),
            RunLayer::default(),
            5,
            |_| None,
        );

        let err = state
            .replace_runtime_settings(resolved_runtime_settings_from_toml(&format!(
                r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "{invalid}"
"#,
            )))
            .expect_err("invalid canonical origin should be rejected");
        assert!(
            err.to_string()
                .contains("server.web.url is required and must be an absolute http(s) URL"),
            "unexpected error for {invalid}: {err}"
        );
        assert_eq!(
            state.canonical_origin().unwrap(),
            "http://valid.example.com".to_string()
        );
    }
}

#[test]
fn canonical_origin_prefers_fabro_web_url_env_override() {
    // FABRO_WEB_URL is the native control-plane override; it wins over the
    // plain `server.web.url` settings literal.
    let state = test_app_state_with_env_lookup(
        canonical_origin_settings("http://settings.example.com"),
        RunLayer::default(),
        5,
        |name| (name == "FABRO_WEB_URL").then(|| "http://env.example.com".to_string()),
    );

    assert_eq!(state.canonical_origin().unwrap(), "http://env.example.com");
}

#[test]
fn canonical_origin_uses_settings_literal_without_env_override() {
    // Without FABRO_WEB_URL set, the plain `server.web.url` literal is used.
    let state = test_app_state_with_env_lookup(
        canonical_origin_settings("http://settings.example.com"),
        RunLayer::default(),
        5,
        |_| None,
    );

    assert_eq!(
        state.canonical_origin().unwrap(),
        "http://settings.example.com"
    );
}

#[test]
fn replace_settings_updates_layer_and_typed_server_settings() {
    let state = test_app_state_with_options(
        server_settings_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://old.example.com"

[server.storage]
root = "/srv/old"
"#,
        ),
        manifest_run_defaults_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://old.example.com"

[server.storage]
root = "/srv/old"
"#,
        ),
        5,
    );

    let updated = r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://new.example.com"

[run.execution]
mode = "dry_run"

[server.storage]
root = "/srv/new"
"#;

    state
        .replace_runtime_settings(resolved_runtime_settings_from_toml(updated))
        .expect("valid settings should replace current state");

    assert_eq!(state.canonical_origin().unwrap(), "http://new.example.com");
    assert_eq!(state.server_settings().server.storage.root, "/srv/new");
    assert_eq!(
        state
            .manifest_run_settings()
            .expect("manifest run settings should resolve")
            .execution
            .mode,
        RunMode::DryRun
    );
    let manifest_run_defaults = state.manifest_run_defaults();
    assert_eq!(
        manifest_run_defaults
            .execution
            .as_ref()
            .and_then(|execution| execution.mode),
        Some(RunMode::DryRun)
    );
}

#[test]
fn replace_settings_caches_invalid_manifest_run_settings_tolerantly() {
    let state = test_app_state_with_options(
        server_settings_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://old.example.com"
"#,
        ),
        manifest_run_defaults_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://old.example.com"
"#,
        ),
        5,
    );

    let updated = r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
url = "http://new.example.com"

[run.environment]
id = "missing"
"#;

    state
        .replace_runtime_settings(resolved_runtime_settings_from_toml(updated))
        .expect("invalid run defaults should not block replace");

    assert_eq!(state.canonical_origin().unwrap(), "http://new.example.com");
    assert!(
        state.manifest_run_settings().is_err(),
        "manifest run settings should stay tolerant for invalid defaults"
    );
}

#[test]
fn system_sandbox_provider_uses_manifest_defaults() {
    let (_environment_temp, environment_store) =
        test_environment_store(Some(EnvironmentProvider::Daytona), true);
    let (_mcp_temp, mcp_server_store) = test_mcp_server_store();
    let source = r#"
_version = 1

[run.environment]
id = "default"
"#;
    let manifest_run_settings = resolve_manifest_run_settings_with_catalog(
        &run_manifest::manifest_run_defaults(Some(&manifest_run_defaults_from_toml(source))),
        &environment_store,
        &mcp_server_store,
    );

    assert_eq!(system_sandbox_provider(&manifest_run_settings), "daytona");
}

#[test]
fn system_sandbox_provider_defaults_when_manifest_run_settings_do_not_resolve() {
    let (_environment_temp, environment_store) = test_environment_store(None, true);
    let (_mcp_temp, mcp_server_store) = test_mcp_server_store();
    let source = r#"
_version = 1

[run.environment]
id = "missing"
"#;
    let manifest_run_settings = resolve_manifest_run_settings_with_catalog(
        &run_manifest::manifest_run_defaults(Some(&manifest_run_defaults_from_toml(source))),
        &environment_store,
        &mcp_server_store,
    );

    assert_eq!(
        system_sandbox_provider(&manifest_run_settings),
        SandboxProviderKind::default().to_string()
    );
}

#[test]
fn sandbox_provider_policy_error_reports_disabled_provider() {
    let settings = server_settings_from_toml(
        r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.sandbox.providers.daytona]
enabled = false
"#,
    );

    assert_eq!(
        crate::run_manifest::sandbox_provider_policy_error(&settings, SandboxProviderKind::Daytona)
            .as_deref(),
        Some(
            "sandbox provider \"daytona\" is disabled by server.sandbox.providers.daytona.enabled"
        )
    );
}

#[test]
fn clone_sandbox_credentials_are_available_for_clone_based_providers() {
    use fabro_types::settings::run::EnvironmentProvider;
    assert!(EnvironmentProvider::Docker.is_clone_based());
    assert!(EnvironmentProvider::Daytona.is_clone_based());
    assert!(!EnvironmentProvider::Local.is_clone_based());
}

#[tokio::test]
async fn create_secret_stores_file_secret_outside_token_lookups() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "/tmp/test.pem",
                "value": "pem-data",
                "type": "file",
                "description": "Test certificate",
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["name"], "/tmp/test.pem");
    assert_eq!(body["type"], "file");
    assert_eq!(body["description"], "Test certificate");

    let vault = state.stores.vault.snapshot().await.unwrap();
    assert_eq!(
        vault.get_entry("/tmp/test.pem").unwrap().secret_type,
        SecretType::File
    );
    assert_eq!(vault.file_secrets(), vec![(
        "/tmp/test.pem".to_string(),
        "pem-data".to_string()
    )]);
}

fn create_token_secret_request(name: &str, value: &str) -> Request<Body> {
    Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": name,
                "value": value,
                "type": "token"
            }))
            .unwrap(),
        ))
        .unwrap()
}

#[tokio::test]
async fn create_secret_rejects_bootstrap_secret_names() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    for name in [EnvVars::SESSION_SECRET, EnvVars::FABRO_DEV_TOKEN] {
        let response = app
            .clone()
            .oneshot(create_token_secret_request(name, "secret-value"))
            .await
            .unwrap();
        let body = response_json!(response, StatusCode::BAD_REQUEST).await;

        assert_eq!(
            body["errors"][0]["detail"],
            format!("{name} is a bootstrap secret; configure it with process env or server.env")
        );
        assert!(state.stores.vault.get(name).await.unwrap().is_none());
    }
}

#[tokio::test]
async fn create_secret_allows_optional_vault_and_custom_secret_names() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    for (name, value) in [
        (EnvVars::GITHUB_APP_CLIENT_SECRET, "github-client-secret"),
        ("CUSTOM_WORKFLOW_TOKEN", "custom-secret"),
    ] {
        let response = app
            .clone()
            .oneshot(create_token_secret_request(name, value))
            .await
            .unwrap();

        assert_status!(response, StatusCode::OK).await;
        assert_eq!(
            state
                .stores
                .vault
                .get(name)
                .await
                .unwrap()
                .map(|entry| entry.value),
            Some(value.to_string())
        );
    }
}

#[tokio::test]
async fn github_webhook_rejects_missing_signature() {
    let app = webhook_test_app(crate::test_support::test_auth_mode());
    let body = br#"{"action":"opened"}"#;

    let response = app
        .oneshot(webhook_request(None, None, body))
        .await
        .unwrap();
    assert_status!(response, StatusCode::UNAUTHORIZED).await;
}

#[tokio::test]
async fn github_webhook_rejects_signature_signed_with_wrong_secret() {
    let app = webhook_test_app(crate::test_support::test_auth_mode());
    let body = br#"{"action":"opened"}"#;
    let bad_signature = compute_signature(b"wrong-secret", body);

    let response = app
        .oneshot(webhook_request(Some(&bad_signature), None, body))
        .await
        .unwrap();
    assert_status!(response, StatusCode::UNAUTHORIZED).await;
}

#[tokio::test]
async fn github_webhook_accepts_valid_signature_when_auth_disabled() {
    let body = br#"{"repository":{"full_name":"owner/repo"},"action":"opened"}"#;
    let signature = compute_signature(TEST_WEBHOOK_SECRET.as_bytes(), body);
    let app = webhook_test_app(crate::test_support::test_auth_mode());

    let response = app
        .oneshot(webhook_request(Some(&signature), None, body))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn github_webhook_accepts_valid_signature_without_bearer_token() {
    let body = br#"{"repository":{"full_name":"owner/repo"},"action":"opened"}"#;
    let signature = compute_signature(TEST_WEBHOOK_SECRET.as_bytes(), body);
    let app = webhook_test_app(dev_token_auth_mode());

    let response = app
        .oneshot(webhook_request(Some(&signature), None, body))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn github_webhook_accepts_valid_signature_with_wrong_bearer_token() {
    let body = br#"{"repository":{"full_name":"owner/repo"},"action":"opened"}"#;
    let signature = compute_signature(TEST_WEBHOOK_SECRET.as_bytes(), body);
    let app = webhook_test_app(dev_token_auth_mode());

    let response = app
        .oneshot(webhook_request(
            Some(&signature),
            Some(&format!("Bearer {WRONG_DEV_TOKEN}")),
            body,
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn create_secret_stores_valid_oauth_entries() {
    let state = TestAppStateBuilder::new().build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "OPENAI_CODEX",
                "value": openai_oauth_credential_json(),
                "type": "oauth"
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::OK).await;
    let listed = state.stores.vault.list().await.unwrap();
    assert_eq!(listed.len(), 1);
    assert_eq!(listed[0].name, "OPENAI_CODEX");
    assert_eq!(listed[0].secret_type, SecretType::Oauth);
    assert!(
        state
            .stores
            .vault
            .get("OPENAI_CODEX")
            .await
            .unwrap()
            .is_some()
    );
}

#[tokio::test]
async fn create_secret_rejects_under_scoped_daytona_api_key_and_leaves_vault_unchanged() {
    let server = MockServer::start_async().await;
    let auth = mock_daytona_auth_probe(&server).await;
    let current_key = mock_daytona_current_key(&server, vec![
        "delete:snapshots",
        "delete:sandboxes",
        "delete:volumes",
    ])
    .await;
    let base_url = server.base_url();
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        fabro_config::RunLayer::default(),
        5,
        move |name| match name {
            EnvVars::DAYTONA_API_URL => Some(base_url.clone()),
            _ => None,
        },
    );
    state
        .stores
        .vault
        .set(
            EnvVars::DAYTONA_API_KEY,
            "existing",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": EnvVars::DAYTONA_API_KEY,
                "value": "dtn_test",
                "type": "token"
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::UNPROCESSABLE_ENTITY).await;

    assert_eq!(
        body["errors"][0]["detail"],
        "API key 'delete-only' is missing required Daytona scopes: \
         write:snapshots, write:sandboxes. Regenerate the key with all \
         snapshot and sandbox scopes."
    );
    assert_eq!(
        state
            .stores
            .vault
            .get(EnvVars::DAYTONA_API_KEY)
            .await
            .unwrap()
            .map(|entry| entry.value),
        Some("existing".to_string())
    );
    auth.assert_async().await;
    current_key.assert_async().await;
}

#[tokio::test]
async fn diagnostics_reports_under_scoped_daytona_api_key() {
    let server = MockServer::start_async().await;
    let auth = mock_daytona_auth_probe(&server).await;
    let current_key = mock_daytona_current_key(&server, vec![
        "delete:snapshots",
        "delete:sandboxes",
        "delete:volumes",
    ])
    .await;
    let base_url = server.base_url();
    let settings = fabro_config::ServerSettingsBuilder::from_toml(
        r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.sandbox.providers.docker]
enabled = false
"#,
    )
    .expect("settings should parse");
    let state = test_app_state_with_env_lookup(
        settings,
        fabro_config::RunLayer::default(),
        5,
        move |name| match name {
            EnvVars::DAYTONA_API_URL => Some(base_url.clone()),
            _ => None,
        },
    );
    state
        .stores
        .vault
        .set(
            EnvVars::DAYTONA_API_KEY,
            "dtn_test",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();

    let report = crate::diagnostics::run_all(&state).await;
    let cloud_sandbox = report
        .sections
        .iter()
        .flat_map(|section| &section.checks)
        .find(|check| check.name == "Cloud Sandbox")
        .expect("cloud sandbox check should be present");

    assert_eq!(cloud_sandbox.status, CheckStatus::Error);
    assert_eq!(
        cloud_sandbox.summary,
        "Daytona API key is missing required scopes"
    );
    assert_eq!(
        cloud_sandbox.details[0].text,
        "missing: write:snapshots, write:sandboxes"
    );
    assert_eq!(
        cloud_sandbox.remediation.as_deref(),
        Some(
            "Regenerate the Daytona API key with scopes: write:snapshots, \
             delete:snapshots, write:sandboxes, delete:sandboxes, then \
             `fabro secret set DAYTONA_API_KEY`."
        )
    );
    auth.assert_async().await;
    current_key.assert_async().await;
}

#[tokio::test]
async fn resolve_llm_client_reads_openai_token_from_vault() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    state
        .stores
        .vault
        .set(
            "OPENAI_API_KEY",
            "vault-openai-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();

    let llm_result = state.resolve_llm_client().await.unwrap();

    assert_eq!(llm_result.client.provider_names(), vec!["openai"]);
    assert!(llm_result.auth_issues.is_empty());
}

#[tokio::test]
async fn resolve_llm_client_ignores_env_lookup_provider_tokens() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |name| (name == EnvVars::OPENAI_API_KEY).then(|| "env-openai-key".to_string()),
    );

    let llm_result = state.resolve_llm_client().await.unwrap();

    assert!(
        llm_result.client.provider_names().is_empty(),
        "server LLM credentials should come from vault only"
    );
    assert!(llm_result.auth_issues.is_empty());
}

struct FailingCredentialSource;

#[async_trait::async_trait]
impl CredentialSource for FailingCredentialSource {
    async fn resolve(
        &self,
        catalog: &fabro_model::Catalog,
    ) -> anyhow::Result<fabro_auth::ResolvedCredentials> {
        let _ = catalog;
        Err(anyhow::Error::new(std::io::Error::other("credential leaf"))
            .context("credential source context"))
    }

    async fn configured_providers(
        &self,
        catalog: &fabro_model::Catalog,
    ) -> Vec<fabro_model::ProviderId> {
        let _ = catalog;
        Vec::new()
    }
}

#[tokio::test]
async fn resolve_llm_client_from_source_preserves_credential_source_chain() {
    let catalog = state_test_catalog();
    let Err(err) = resolve_llm_client_from_source(&FailingCredentialSource, catalog).await else {
        panic!("expected credential resolution to fail");
    };
    let chain = err.chain().map(ToString::to_string).collect::<Vec<_>>();

    assert!(
        chain
            .iter()
            .any(|cause| cause == "credential source context"),
        "expected context in chain, got {chain:#?}"
    );
    assert!(
        chain.iter().any(|cause| cause == "credential leaf"),
        "expected source in chain, got {chain:#?}"
    );
}

#[tokio::test]
async fn llm_source_configured_providers_reads_openai_token_from_vault() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    state
        .stores
        .vault
        .set(
            "OPENAI_API_KEY",
            "vault-openai-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();

    let catalog = state.catalog();
    assert_eq!(
        state
            .llm_source
            .configured_providers(catalog.as_ref())
            .await,
        vec![ProviderId::openai()]
    );
}

#[tokio::test]
async fn resolve_llm_client_uses_vault_key_without_env_lookup_openai_settings() {
    let server = MockServer::start_async().await;
    let response_mock = server
        .mock_async(|when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", "Bearer vault-openai-key");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload("hello from vault key"));
        })
        .await;
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .env_lookup(|name| match name {
            "OPENAI_ORG_ID" => Some("env-org".to_string()),
            _ => None,
        })
        .provider_base_url("openai", server.url("/v1"))
        .build();
    state
        .stores
        .vault
        .set(
            "OPENAI_API_KEY",
            "vault-openai-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();

    let llm_result = state.resolve_llm_client().await.unwrap();
    let response = llm_result
        .client
        .complete(&LlmRequest {
            model:            "gpt-5.4".to_string(),
            messages:         vec![LlmMessage::user("Hello")],
            provider:         Some("openai".to_string()),
            tools:            None,
            tool_choice:      None,
            response_format:  None,
            temperature:      None,
            top_p:            None,
            max_tokens:       None,
            stop_sequences:   None,
            reasoning_effort: None,
            speed:            None,
            metadata:         None,
            provider_options: None,
        })
        .await
        .unwrap();

    assert_eq!(response.text(), "hello from vault key");
    response_mock.assert_async().await;
}

#[tokio::test]
async fn list_secrets_includes_oauth_metadata() {
    let state = test_app_state();
    state
        .stores
        .vault
        .set(
            "OPENAI_CODEX",
            &openai_oauth_credential_json(),
            SecretType::Oauth,
            Some("saved auth"),
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/secrets"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be an array");
    let entry = data
        .iter()
        .find(|entry| entry["name"] == "OPENAI_CODEX")
        .expect("oauth metadata should be listed");
    assert_eq!(entry["type"], "oauth");
    assert_eq!(entry["description"], "saved auth");
    assert!(entry.get("updated_at").is_some());
    assert!(entry.get("value").is_none());
}

#[tokio::test]
async fn create_secret_rejects_invalid_oauth_json() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "OPENAI_CODEX",
                "value": "{not-json",
                "type": "oauth"
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn create_secret_rejects_invalid_oauth_name() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "1OPENAI",
                "value": openai_oauth_credential_json(),
                "type": "oauth"
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn delete_secret_by_name_removes_file_secret() {
    let state = TestAppStateBuilder::new().build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let create_req = Request::builder()
        .method("POST")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "/tmp/test.pem",
                "value": "pem-data",
                "type": "file",
            }))
            .unwrap(),
        ))
        .unwrap();
    let create_response = app.clone().oneshot(create_req).await.unwrap();
    assert_status!(create_response, StatusCode::OK).await;

    let delete_req = Request::builder()
        .method("DELETE")
        .uri(api("/secrets"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "name": "/tmp/test.pem",
            }))
            .unwrap(),
        ))
        .unwrap();

    let delete_response = app.oneshot(delete_req).await.unwrap();
    assert_status!(delete_response, StatusCode::NO_CONTENT).await;
    assert!(state.stores.vault.list().await.unwrap().is_empty());
}

#[test]
fn server_secrets_resolve_bootstrap_process_env_before_server_env() {
    let dir = tempfile::tempdir().unwrap();
    std::fs::write(
        dir.path().join("server.env"),
        "SESSION_SECRET=file-value\nFABRO_DEV_TOKEN=file-dev-token\n",
    )
    .unwrap();

    let secrets = ServerSecrets::load(
        dir.path().join("server.env"),
        HashMap::from([("SESSION_SECRET".to_string(), "env-value".to_string())]),
    )
    .unwrap();

    assert_eq!(secrets.get("SESSION_SECRET").as_deref(), Some("env-value"));
    assert_eq!(
        secrets.get("FABRO_DEV_TOKEN").as_deref(),
        Some("file-dev-token")
    );
}

fn slack_app_state_with_secret_sources(
    vault_entries: &[(&str, &str, SecretType)],
    server_secret_env: HashMap<String, String>,
) -> Arc<AppState> {
    slack_app_state_with_settings_and_secret_sources(
        default_test_server_settings(),
        vault_entries,
        server_secret_env,
    )
}

fn slack_app_state_with_settings_and_secret_sources(
    settings: ServerSettings,
    vault_entries: &[(&str, &str, SecretType)],
    server_secret_env: HashMap<String, String>,
) -> Arc<AppState> {
    let (store, artifact_store) = test_store_bundle();
    let vault_path = test_secret_store_path();
    let server_env_path = vault_path.with_file_name("server.env");
    let mut vault = Vault::load(vault_path.clone()).unwrap();
    for (name, value, secret_type) in vault_entries {
        vault.set(name, value, *secret_type, None).unwrap();
    }
    build_app_state(AppStateConfig {
        resolved_settings: resolved_runtime_settings_for_tests(
            settings,
            RunLayer::default(),
            LlmCatalogSettings::default(),
        ),
        registry_factory_override: None,
        max_concurrent_runs: 5,
        store,
        artifact_store,
        db_pool: test_db_pool_for_vault_path(&vault_path).expect("test db pool should build"),
        preloaded_vault: vault,
        server_secrets: load_test_server_secrets(server_env_path, server_secret_env),
        env_lookup: default_env_lookup(),
        github_api_base_url: None,
        active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
        http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
        sandbox_provider_registry: None,
        shutdown: tokio_util::sync::CancellationToken::new(),
        worker_control_bus: None,
        worker_runtime: None,
        automation_materializer_override: None,
    })
    .expect("slack test app state should build")
}

fn slack_test_vault_tokens() -> [(&'static str, &'static str, SecretType); 2] {
    [
        (
            EnvVars::FABRO_SLACK_BOT_TOKEN,
            "xoxb-test",
            SecretType::Token,
        ),
        (
            EnvVars::FABRO_SLACK_APP_TOKEN,
            "xapp-test",
            SecretType::Token,
        ),
    ]
}

#[test]
fn slack_service_ignores_vault_tokens_when_config_is_absent() {
    let state = slack_app_state_with_secret_sources(&slack_test_vault_tokens(), HashMap::new());

    assert!(state.slack_service.is_none());
}

#[test]
fn slack_service_is_enabled_by_config_and_vault_tokens() {
    let state = slack_app_state_with_settings_and_secret_sources(
        server_settings_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.integrations.slack]
enabled = true
"#,
        ),
        &[
            (
                EnvVars::FABRO_SLACK_BOT_TOKEN,
                "xoxb-test",
                SecretType::Token,
            ),
            (
                EnvVars::FABRO_SLACK_APP_TOKEN,
                "xapp-test",
                SecretType::Token,
            ),
        ],
        HashMap::new(),
    );

    let service = state
        .slack_service
        .as_ref()
        .expect("slack service should be enabled by config and vault tokens");
    let connection = service.connection_status();
    assert_eq!(connection.kind, IntegrationConnectionKind::SocketMode);
    assert_eq!(connection.status, IntegrationConnectionState::Connecting);
    assert!(connection.last_connected_at.is_none());
    assert!(connection.last_error.is_none());
    assert!(service.default_channel.is_none());
}

#[test]
fn slack_service_receives_configured_default_channel_verbatim() {
    let state = slack_app_state_with_settings_and_secret_sources(
        server_settings_from_toml(
            r##"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.integrations.slack]
enabled = true
default_channel = "#releases"
"##,
        ),
        &slack_test_vault_tokens(),
        HashMap::new(),
    );

    let service = state
        .slack_service
        .as_ref()
        .expect("slack service should be enabled by config and vault tokens");
    assert_eq!(service.default_channel.as_deref(), Some("#releases"));
}

#[test]
fn slack_service_ignores_server_env_tokens() {
    let state = slack_app_state_with_secret_sources(
        &[],
        HashMap::from([
            (
                EnvVars::FABRO_SLACK_BOT_TOKEN.to_string(),
                "xoxb-server-env".to_string(),
            ),
            (
                EnvVars::FABRO_SLACK_APP_TOKEN.to_string(),
                "xapp-server-env".to_string(),
            ),
        ]),
    );

    assert!(state.slack_service.is_none());
}

#[test]
fn slack_service_respects_disabled_server_config_even_with_vault_tokens() {
    let mut settings = default_test_server_settings();
    settings.server.integrations.slack.enabled = false;
    let (store, artifact_store) = test_store_bundle();
    let vault_path = test_secret_store_path();
    let mut vault = Vault::load(vault_path.clone()).unwrap();
    vault
        .set(
            EnvVars::FABRO_SLACK_BOT_TOKEN,
            "xoxb-test",
            SecretType::Token,
            None,
        )
        .unwrap();
    vault
        .set(
            EnvVars::FABRO_SLACK_APP_TOKEN,
            "xapp-test",
            SecretType::Token,
            None,
        )
        .unwrap();

    let state = build_app_state(AppStateConfig {
        resolved_settings: resolved_runtime_settings_for_tests(
            settings,
            RunLayer::default(),
            LlmCatalogSettings::default(),
        ),
        registry_factory_override: None,
        max_concurrent_runs: 5,
        store,
        artifact_store,
        db_pool: test_db_pool_for_vault_path(&vault_path).expect("test db pool should build"),
        preloaded_vault: vault,
        server_secrets: load_test_server_secrets(
            tempfile::tempdir().unwrap().path().join("server.env"),
            HashMap::new(),
        ),
        env_lookup: default_env_lookup(),
        github_api_base_url: None,
        active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
        http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
        sandbox_provider_registry: None,
        shutdown: tokio_util::sync::CancellationToken::new(),
        worker_control_bus: None,
        worker_runtime: None,
        automation_materializer_override: None,
    })
    .expect("slack disabled test app state should build");

    assert!(state.slack_service.is_none());
}

#[cfg(unix)]
#[test]
fn worker_command_uses_null_stdin_and_token_env() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let cmd = worker_command(
        state.as_ref(),
        RunId::new(),
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_worker_command_passes_token_only_by_env(&cmd);
}

#[cfg(unix)]
#[test]
fn worker_command_sets_worker_args() {
    let storage_dir = tempfile::tempdir().unwrap();
    let run_dir = storage_dir.path().join("run-scratch");
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Resume,
        &run_dir,
        false,
    )
    .unwrap();

    let args = cmd
        .as_std()
        .get_args()
        .map(|arg| arg.to_string_lossy().into_owned())
        .collect::<Vec<_>>();
    assert_eq!(args, vec![
        "__run-worker".to_string(),
        "--server".to_string(),
        "http://127.0.0.1:32276".to_string(),
        "--storage-dir".to_string(),
        storage_dir.path().display().to_string(),
        "--run-dir".to_string(),
        run_dir.display().to_string(),
        "--run-id".to_string(),
        run_id.to_string(),
        "--mode".to_string(),
        "resume".to_string(),
    ]);
}

#[cfg(unix)]
#[test]
fn worker_command_default_token_omits_agent_run_tools_scope() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_worker_command_passes_token_only_by_env(&cmd);
    let claims = worker_token_claims(&cmd, state.as_ref());

    assert_eq!(claims.run_id, run_id.to_string());
    assert_eq!(claims.scope.split_whitespace().collect::<Vec<_>>(), vec![
        "run:worker"
    ]);
}

#[cfg(unix)]
#[test]
fn worker_command_opt_in_token_includes_agent_run_tools_scope() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        true,
    )
    .unwrap();

    assert_worker_command_passes_token_only_by_env(&cmd);
    let claims = worker_token_claims(&cmd, state.as_ref());

    assert_eq!(claims.run_id, run_id.to_string());
    assert_eq!(claims.scope.split_whitespace().collect::<Vec<_>>(), vec![
        "run:worker",
        "agent:run_tools"
    ]);
}

#[cfg(unix)]
#[test]
fn worker_command_forwards_github_app_private_key_from_vault() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let spec = worker_launch_spec(
        state.as_ref(),
        RunId::new(),
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
        Some("test-private-key".to_string()),
    )
    .unwrap();
    let cmd = LocalWorkerRuntime::command_for_spec(&spec);

    assert_eq!(
        command_env_value(&cmd, EnvVars::GITHUB_APP_PRIVATE_KEY),
        EnvOverride::Set("test-private-key".to_string())
    );
}

#[cfg(unix)]
#[test]
fn worker_command_omits_github_app_private_key_when_unset() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state(storage_dir.path(), &["dev-token"], Some(TEST_DEV_TOKEN));
    let cmd = worker_command(
        state.as_ref(),
        RunId::new(),
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_eq!(
        command_env_value(&cmd, EnvVars::GITHUB_APP_PRIVATE_KEY),
        EnvOverride::Unchanged
    );
}

#[cfg(unix)]
#[test]
fn worker_command_sets_fabro_log_from_server_logging_config() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state_with_extra_config(
        storage_dir.path(),
        &["dev-token"],
        Some(TEST_DEV_TOKEN),
        r#"
[server.logging]
level = "debug"
"#,
    );
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_eq!(
        command_env_value(&cmd, EnvVars::FABRO_LOG),
        EnvOverride::Set("debug".to_string())
    );
}

#[cfg(unix)]
#[test]
fn worker_command_sets_fabro_log_destination_from_server_logging_config() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state_with_extra_config(
        storage_dir.path(),
        &["dev-token"],
        Some(TEST_DEV_TOKEN),
        r#"
[server.logging]
destination = "stdout"
"#,
    );
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_eq!(
        command_env_value(&cmd, EnvVars::FABRO_LOG_DESTINATION),
        EnvOverride::Set("stdout".to_string())
    );
}

#[cfg(unix)]
#[test]
fn worker_command_sets_fabro_config_to_active_absolute_config_path() {
    let storage_dir = tempfile::tempdir().unwrap();
    let config_dir = tempfile::tempdir().unwrap();
    let active_config_path = config_dir.path().join("settings.toml");
    let state = worker_command_test_state_with_active_config_path(
        storage_dir.path(),
        &["dev-token"],
        Some(TEST_DEV_TOKEN),
        active_config_path.clone(),
    );
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert!(active_config_path.is_absolute());
    assert_eq!(
        command_env_value(&cmd, EnvVars::FABRO_CONFIG),
        EnvOverride::Set(active_config_path.display().to_string())
    );
    let worker_args = cmd
        .as_std()
        .get_args()
        .map(|arg| arg.to_string_lossy().into_owned())
        .collect::<Vec<_>>();
    assert!(
        !worker_args.iter().any(|arg| arg == "--config"),
        "__run-worker argument contract should not grow hidden config args: {worker_args:?}"
    );
}

#[cfg(unix)]
#[test]
fn worker_command_env_log_destination_overrides_server_logging_config() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state_with_extra_config_and_env_lookup(
        storage_dir.path(),
        &["dev-token"],
        Some(TEST_DEV_TOKEN),
        r#"
[server.logging]
destination = "file"
"#,
        &[],
        |name| (name == EnvVars::FABRO_LOG_DESTINATION).then(|| "stdout".to_string()),
    );
    let run_id = RunId::new();

    let cmd = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    )
    .unwrap();

    assert_eq!(
        command_env_value(&cmd, EnvVars::FABRO_LOG_DESTINATION),
        EnvOverride::Set("stdout".to_string())
    );
}

#[cfg(unix)]
#[test]
fn worker_command_rejects_invalid_env_log_destination() {
    let storage_dir = tempfile::tempdir().unwrap();
    let state = worker_command_test_state_with_extra_config_and_env_lookup(
        storage_dir.path(),
        &["dev-token"],
        Some(TEST_DEV_TOKEN),
        r#"
[server.logging]
destination = "file"
"#,
        &[],
        |name| (name == EnvVars::FABRO_LOG_DESTINATION).then(|| "stdot".to_string()),
    );
    let run_id = RunId::new();

    let Err(err) = worker_command(
        state.as_ref(),
        run_id,
        RunExecutionMode::Start,
        storage_dir.path(),
        false,
    ) else {
        panic!("invalid env destination should fail");
    };

    let message = err.to_string();
    assert!(message.contains(EnvVars::FABRO_LOG_DESTINATION));
    assert!(message.contains("stdot"));
}

#[test]
fn build_app_state_requires_session_secret_for_worker_tokens() {
    let server_settings = server_settings_from_toml(
        r#"
_version = 1

[server.auth]
methods = ["dev-token"]
"#,
    );
    let (store, artifact_store) = test_store_bundle();
    let vault_path = test_secret_store_path();
    let server_env_path = vault_path.with_file_name("server.env");
    let db_pool = test_db_pool_for_vault_path(&vault_path).expect("test db pool should build");
    let preloaded_vault = crate::test_support::test_secret_snapshot(db_pool.clone())
        .expect("test secret snapshot should build");
    let Err(err) = build_app_state(AppStateConfig {
        resolved_settings: resolved_runtime_settings_for_tests(
            server_settings,
            RunLayer::default(),
            LlmCatalogSettings::default(),
        ),
        registry_factory_override: None,
        max_concurrent_runs: 5,
        store,
        artifact_store,
        db_pool,
        preloaded_vault,
        server_secrets: ServerSecrets::load(server_env_path, HashMap::new()).unwrap(),
        env_lookup: default_env_lookup(),
        github_api_base_url: None,
        active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
        http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
        sandbox_provider_registry: None,
        shutdown: tokio_util::sync::CancellationToken::new(),
        worker_control_bus: None,
        worker_runtime: None,
        automation_materializer_override: None,
    }) else {
        panic!("build_app_state should require SESSION_SECRET")
    };

    assert!(err.to_string().contains(
        "Fabro server refuses to start: auth is configured but SESSION_SECRET is not set."
    ));
}

#[tokio::test]
async fn build_app_state_migrates_legacy_vault_file_on_boot() {
    let vault_path = test_secret_store_path();
    let timestamp = "2026-05-18T12:00:00Z";
    let legacy_api_key = json!({
        "provider": "anthropic",
        "type": "api_key",
        "key": "sk-ant-legacy",
    });
    let legacy_oauth = json!({
        "provider": "openai",
        "type": "codex_oauth",
        "tokens": {
            "access_token": "codex-access",
            "refresh_token": "codex-refresh",
            "expires_at": "2026-05-18T13:00:00Z",
        },
        "config": {
            "auth_url": "https://auth.openai.com",
            "token_url": "https://auth.openai.com/oauth/token",
            "client_id": "client",
            "scopes": ["openid", "offline_access"],
            "redirect_uri": "https://auth.openai.com/deviceauth/callback",
            "use_pkce": false,
        },
        "account_id": "acct_legacy",
    });
    let legacy_vault = json!({
        "anthropic": {
            "value": legacy_api_key.to_string(),
            "type": "credential",
            "created_at": timestamp,
            "updated_at": timestamp,
        },
        "openai_codex": {
            "value": legacy_oauth.to_string(),
            "type": "credential",
            "created_at": timestamp,
            "updated_at": timestamp,
        },
        "GITHUB_TOKEN": {
            "value": "ghp_legacy",
            "type": "environment",
            "created_at": timestamp,
            "updated_at": timestamp,
        },
        "/tmp/github.pem": {
            "value": "/tmp/github.pem",
            "type": "file",
            "created_at": timestamp,
            "updated_at": timestamp,
        },
    });
    std::fs::write(
        &vault_path,
        serde_json::to_vec_pretty(&legacy_vault).unwrap(),
    )
    .expect("legacy vault should be writable");

    let state = build_test_app_state_with_vault_path(&vault_path)
        .expect("legacy vault should not prevent server boot");

    let vault = state.stores.vault.snapshot().await.unwrap();
    let api_key_entry = vault
        .get_entry("ANTHROPIC_API_KEY")
        .expect("legacy provider credential should be migrated to token name");
    assert_eq!(api_key_entry.secret_type, SecretType::Token);
    assert_eq!(api_key_entry.value, "sk-ant-legacy");
    assert!(vault.get_entry("anthropic").is_none());

    let oauth_entry = vault
        .get_entry("OPENAI_CODEX")
        .expect("legacy Codex credential should be migrated to canonical OAuth name");
    assert_eq!(oauth_entry.secret_type, SecretType::Oauth);
    let oauth: fabro_auth::OAuthCredential =
        serde_json::from_str(&oauth_entry.value).expect("migrated OAuth JSON should parse");
    assert_eq!(oauth.tokens.access_token, "codex-access");
    assert_eq!(oauth.account_id.as_deref(), Some("acct_legacy"));
    assert!(vault.get_entry("openai_codex").is_none());

    assert_eq!(
        vault.get_entry("GITHUB_TOKEN").unwrap().secret_type,
        SecretType::Token
    );
    assert_eq!(
        vault.get_entry("/tmp/github.pem").unwrap().secret_type,
        SecretType::File
    );
}

fn build_test_app_state_with_vault_path(vault_path: &Path) -> anyhow::Result<Arc<AppState>> {
    let (store, artifact_store) = test_store_bundle();
    let db_pool = test_db_pool_for_vault_path(vault_path)?;
    let preloaded_vault = crate::test_support::test_secret_snapshot(db_pool.clone())?;
    build_app_state(AppStateConfig {
        resolved_settings: resolved_runtime_settings_for_tests(
            default_test_server_settings(),
            RunLayer::default(),
            LlmCatalogSettings::default(),
        ),
        registry_factory_override: None,
        max_concurrent_runs: 5,
        store,
        artifact_store,
        db_pool,
        preloaded_vault,
        server_secrets: load_test_server_secrets(
            vault_path.with_file_name("server.env"),
            HashMap::new(),
        ),
        env_lookup: default_env_lookup(),
        github_api_base_url: None,
        active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
        http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
        sandbox_provider_registry: None,
        shutdown: tokio_util::sync::CancellationToken::new(),
        worker_control_bus: None,
        worker_runtime: None,
        automation_materializer_override: None,
    })
}

fn test_worker_ref(pid: u32) -> WorkerRef {
    WorkerRef::Local { pid }
}

#[cfg(unix)]
fn worker_command(
    state: &AppState,
    run_id: RunId,
    mode: RunExecutionMode,
    run_dir: &Path,
    agent_fabro_tools_enabled: bool,
) -> anyhow::Result<Command> {
    let spec = worker_launch_spec(
        state,
        run_id,
        mode,
        run_dir,
        agent_fabro_tools_enabled,
        None,
    )?;
    Ok(LocalWorkerRuntime::command_for_spec(&spec))
}

fn worker_command_test_state(
    storage_dir: &Path,
    methods: &[&str],
    dev_token: Option<&str>,
) -> Arc<AppState> {
    worker_command_test_state_with_extra_config(storage_dir, methods, dev_token, "")
}

fn worker_command_test_state_with_extra_config(
    storage_dir: &Path,
    methods: &[&str],
    dev_token: Option<&str>,
    extra_config: &str,
) -> Arc<AppState> {
    worker_command_test_state_with_extra_config_and_env_lookup(
        storage_dir,
        methods,
        dev_token,
        extra_config,
        &[],
        |_| None,
    )
}

fn worker_command_test_state_with_extra_config_and_env_lookup(
    storage_dir: &Path,
    methods: &[&str],
    dev_token: Option<&str>,
    extra_config: &str,
    extra_server_secrets: &[(&str, &str)],
    env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
) -> Arc<AppState> {
    worker_command_test_state_inner(
        storage_dir,
        methods,
        dev_token,
        extra_config,
        extra_server_secrets,
        env_lookup,
        None,
    )
}

fn worker_command_test_state_with_active_config_path(
    storage_dir: &Path,
    methods: &[&str],
    dev_token: Option<&str>,
    active_config_path: PathBuf,
) -> Arc<AppState> {
    worker_command_test_state_inner(
        storage_dir,
        methods,
        dev_token,
        "",
        &[],
        |_| None,
        Some(active_config_path),
    )
}

fn worker_command_test_state_inner(
    storage_dir: &Path,
    methods: &[&str],
    dev_token: Option<&str>,
    extra_config: &str,
    extra_server_secrets: &[(&str, &str)],
    env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
    active_config_path: Option<PathBuf>,
) -> Arc<AppState> {
    let dev_token = dev_token.map(str::to_owned);
    std::fs::create_dir_all(storage_dir).unwrap();
    let source = format!(
        r#"
_version = 1

[server.storage]
root = "{}"

[server.auth]
methods = [{}]

[server.auth.github]
allowed_usernames = ["octocat"]
{extra_config}
"#,
        storage_dir.display(),
        methods
            .iter()
            .map(|method| format!("\"{method}\""))
            .collect::<Vec<_>>()
            .join(", ")
    );
    let runtime_directory = Storage::new(storage_dir).runtime_directory();
    ServerDaemon::new(
        std::process::id(),
        Bind::Tcp("127.0.0.1:32276".parse::<std::net::SocketAddr>().unwrap()),
        runtime_directory.log_path(),
    )
    .write(&runtime_directory)
    .unwrap();

    let mut server_secret_env: HashMap<String, String> = dev_token
        .map(|token| HashMap::from([("FABRO_DEV_TOKEN".to_string(), token)]))
        .unwrap_or_default();
    for (key, value) in extra_server_secrets {
        server_secret_env.insert((*key).to_string(), (*value).to_string());
    }
    let mut builder = TestAppStateBuilder::new()
        .runtime_settings(
            server_settings_from_toml(&source),
            manifest_run_defaults_from_toml(&source),
        )
        .max_concurrent_runs(5)
        .env_lookup(env_lookup)
        .server_secret_env(server_secret_env);
    if let Some(active_config_path) = active_config_path {
        builder = builder.active_config_path(active_config_path);
    }
    builder.build()
}

#[cfg(unix)]
#[derive(Debug, PartialEq, Eq)]
enum EnvOverride {
    Unchanged,
    Removed,
    Set(String),
}

#[cfg(unix)]
fn command_env_value(cmd: &Command, key: &str) -> EnvOverride {
    cmd.as_std()
        .get_envs()
        .find_map(|(name, value)| {
            (name.to_str() == Some(key)).then(|| match value {
                Some(value) => EnvOverride::Set(value.to_string_lossy().into_owned()),
                None => EnvOverride::Removed,
            })
        })
        .unwrap_or(EnvOverride::Unchanged)
}

#[cfg(unix)]
fn assert_worker_command_passes_token_only_by_env(cmd: &Command) {
    assert!(matches!(
        command_env_value(cmd, EnvVars::FABRO_WORKER_TOKEN),
        EnvOverride::Set(_)
    ));
    assert_eq!(
        command_env_value(cmd, EnvVars::FABRO_DEV_TOKEN),
        EnvOverride::Unchanged
    );
    let args = cmd
        .as_std()
        .get_args()
        .map(|arg| arg.to_string_lossy())
        .collect::<Vec<_>>();
    assert!(!args.iter().any(|arg| arg == "--artifact-upload-token"));
    assert!(!args.iter().any(|arg| arg == "--worker-token"));
}

#[cfg(unix)]
fn worker_token_claims(cmd: &Command, state: &AppState) -> crate::worker_token::WorkerTokenClaims {
    let EnvOverride::Set(token) = command_env_value(cmd, EnvVars::FABRO_WORKER_TOKEN) else {
        panic!("worker token env should be set");
    };

    jsonwebtoken::decode::<crate::worker_token::WorkerTokenClaims>(
        &token,
        state.worker_token_keys().decoding_key(),
        state.worker_token_keys().validation(),
    )
    .expect("worker token should decode")
    .claims
}

#[derive(Default)]
struct RecordingWorkerRuntime {
    requested:     StdMutex<Vec<WorkerRef>>,
    forced:        StdMutex<Vec<WorkerRef>>,
    alive:         AtomicBool,
    forced_notify: Notify,
}

impl RecordingWorkerRuntime {
    fn requested_refs(&self) -> Vec<WorkerRef> {
        self.requested
            .lock()
            .expect("requested lock poisoned")
            .clone()
    }

    fn forced_refs(&self) -> Vec<WorkerRef> {
        self.forced.lock().expect("forced lock poisoned").clone()
    }

    fn set_alive(&self, alive: bool) {
        self.alive.store(alive, Ordering::Relaxed);
    }

    async fn wait_for_forced_ref(&self, worker_ref: &WorkerRef) {
        tokio::time::timeout(std::time::Duration::from_secs(1), async {
            loop {
                let notified = self.forced_notify.notified();
                if self.forced_refs().contains(worker_ref) {
                    return;
                }
                notified.await;
            }
        })
        .await
        .expect("worker should be force-stopped after the cancellation grace period");
    }
}

#[async_trait::async_trait]
impl WorkerRuntime for RecordingWorkerRuntime {
    async fn start(&self, _spec: WorkerLaunchSpec) -> anyhow::Result<StartedWorker> {
        anyhow::bail!("recording runtime does not start workers")
    }

    async fn request_stop(&self, worker_ref: &WorkerRef) {
        self.requested
            .lock()
            .expect("requested lock poisoned")
            .push(worker_ref.clone());
    }

    async fn force_stop(&self, worker_ref: &WorkerRef) {
        self.forced
            .lock()
            .expect("forced lock poisoned")
            .push(worker_ref.clone());
        self.alive.store(false, Ordering::Relaxed);
        self.forced_notify.notify_one();
    }

    async fn is_alive(&self, _worker_ref: &WorkerRef) -> bool {
        self.alive.load(Ordering::Relaxed)
    }
}

async fn worker_transport_with_receiver(
    run_id: RunId,
) -> (RunAnswerTransport, WorkerControlReceiver) {
    let bus = StdArc::new(LocalWorkerControlBus::new());
    let receiver = bus
        .subscribe(run_id, WorkerControlCursor::Start)
        .await
        .expect("test worker bus should subscribe");
    // Ensure the subscription task is waiting before the test publishes.
    tokio::task::yield_now().await;
    let bus: StdArc<dyn WorkerControlBus> = bus;
    let transport = RunAnswerTransport::Worker { run_id, bus };
    (transport, receiver)
}

async fn recv_worker_control_envelope(
    receiver: &mut WorkerControlReceiver,
) -> WorkerControlEnvelope {
    receiver
        .recv()
        .await
        .expect("test worker control receiver should stay open")
        .expect("test worker control delivery should succeed")
        .envelope
}

#[tokio::test]
async fn worker_answer_transport_cancel_run_publishes_cancel_message() {
    let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;

    transport.cancel_run().await.unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::cancel_run()
    );
}

#[tokio::test]
async fn worker_answer_transport_steer_publishes_plain_steer_message() {
    let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;
    let actor = Principal::System {
        system_kind: SystemActorKind::Engine,
    };

    transport
        .steer("try again".to_string(), actor.clone())
        .await
        .unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::steer("try again", actor)
    );
}

#[tokio::test]
async fn worker_answer_transport_interrupt_publishes_interrupt_message() {
    let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;
    let actor = Principal::System {
        system_kind: SystemActorKind::Engine,
    };

    transport.interrupt(actor.clone()).await.unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::interrupt(actor)
    );
}

#[tokio::test]
async fn worker_answer_transport_interrupt_then_steer_publishes_single_combined_message() {
    let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;
    let actor = Principal::System {
        system_kind: SystemActorKind::Engine,
    };

    transport
        .interrupt_then_steer("try again".to_string(), actor.clone())
        .await
        .unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::interrupt_then_steer("try again", actor)
    );
}

#[tokio::test]
async fn worker_answer_transport_pair_commands_publish_control_messages() {
    let run_id = fixtures::RUN_1;
    let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;
    let pair_id = "01HZX6M29F1CD5YYMHT1F5D7WQ".parse().unwrap();
    let message_id = "01HZX6M4D7Y1QW0Q0P6V8Z4DR5".parse().unwrap();
    let actor = Principal::System {
        system_kind: SystemActorKind::Engine,
    };
    let target = pair_test_target();

    transport
        .start_pair(run_id, pair_id, target.clone(), actor.clone())
        .await
        .unwrap();
    transport
        .send_pair_message(
            pair_id,
            message_id,
            "inspect this".to_string(),
            Some("client-1".to_string()),
            actor.clone(),
        )
        .await
        .unwrap();
    transport.end_pair(pair_id, actor.clone()).await.unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor.clone())
    );
    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::pair_message(
            pair_id,
            message_id,
            "inspect this",
            Some("client-1".to_string()),
            actor.clone()
        )
    );
    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::end_pair(pair_id, actor)
    );
}

#[tokio::test]
async fn worker_answer_transport_pause_and_unpause_publish_control_messages() {
    let (transport, mut control_rx) = worker_transport_with_receiver(fixtures::RUN_1).await;

    transport.pause_run().await.unwrap();
    transport.unpause_run().await.unwrap();

    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::pause_run()
    );
    assert_eq!(
        recv_worker_control_envelope(&mut control_rx).await,
        WorkerControlEnvelope::unpause_run()
    );
}

#[tokio::test]
async fn in_process_answer_transport_cancel_run_cancels_pending_interviews() {
    let interviewer = Arc::new(ControlInterviewer::new());
    let emitter = Arc::new(fabro_workflow::event::Emitter::new(
        fabro_types::RunId::new(),
    ));
    let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(emitter));
    let transport = RunAnswerTransport::InProcess {
        interviewer:  Arc::clone(&interviewer),
        steering_hub: Arc::clone(&steering_hub),
    };
    let mut question = Question::new("Approve?", QuestionType::YesNo);
    question.id = "q-1".to_string();
    let ask_interviewer = Arc::clone(&interviewer);
    let answer_task = tokio::spawn(async move { ask_interviewer.ask(question).await });
    tokio::task::yield_now().await;

    transport.cancel_run().await.unwrap();

    let answer = answer_task.await.unwrap().answer;
    assert_eq!(answer.value, AnswerValue::Cancelled);
}

fn manifest_json(target_path: &str, dot_source: &str) -> serde_json::Value {
    serde_json::json!({
        "version": 1,
        "cwd": "/tmp",
        "target": {
            "path": target_path,
        },
        "workflows": {
            target_path: {
                "source": dot_source,
                "files": {},
            },
        },
    })
}

fn minimal_manifest_json(dot_source: &str) -> serde_json::Value {
    manifest_json("workflow.fabro", dot_source)
}

fn manifest_body(dot_source: &str) -> Body {
    Body::from(serde_json::to_string(&minimal_manifest_json(dot_source)).unwrap())
}

fn manifest_body_for(target_path: &str, dot_source: &str) -> Body {
    Body::from(serde_json::to_string(&manifest_json(target_path, dot_source)).unwrap())
}

async fn create_run(app: &Router, dot_source: &str) -> String {
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(dot_source))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    body["id"].as_str().unwrap().to_string()
}

#[tokio::test]
async fn create_run_response_includes_web_url_when_web_enabled() {
    let state = test_app_state_with_options(
        server_settings_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
enabled = true
url = "http://127.0.0.1:32276"
"#,
        ),
        RunLayer::default(),
        5,
    );
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header("content-type", "application/json")
                .body(manifest_body(MINIMAL_DOT))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let id = body["id"].as_str().expect("id should be a string");
    assert_eq!(
        body["links"]["web"].as_str(),
        Some(format!("http://127.0.0.1:32276/runs/{id}").as_str()),
    );
}

#[tokio::test]
async fn system_repair_runs_lists_catalog_entries_without_projection() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    state
        .stores
        .runs
        .catalog_index()
        .await
        .unwrap()
        .add(&run_id)
        .await
        .unwrap();

    let response = app
        .oneshot(
            Request::builder()
                .method(Method::GET)
                .uri(api("/system/repair/runs"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["total_count"], 1);
    assert_eq!(body["runs"][0]["run_id"], run_id.to_string());
    let created_at = body["runs"][0]["created_at"]
        .as_str()
        .unwrap()
        .parse::<chrono::DateTime<Utc>>()
        .unwrap();
    assert_eq!(created_at, run_id.created_at());
    assert!(
        body["runs"][0]["error"]
            .as_str()
            .unwrap()
            .contains("no events"),
        "got: {}",
        body["runs"][0]["error"]
    );
}

#[tokio::test]
async fn create_run_response_omits_web_url_when_web_disabled() {
    let state = test_app_state_with_options(
        server_settings_from_toml(
            r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.web]
enabled = false
url = "http://127.0.0.1:32276"
"#,
        ),
        RunLayer::default(),
        5,
    );
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header("content-type", "application/json")
                .body(manifest_body(MINIMAL_DOT))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    assert!(
        body.get("web_url").is_none() || body["web_url"].is_null(),
        "web_url should be absent or null when web is disabled, got {body}"
    );
}

#[tokio::test]
async fn create_run_without_explicit_title_returns_deterministic_then_updates_generated_title() {
    let llm = MockServer::start_async().await;
    let title_mock = mock_openai_title_response(&llm, "Generated deploy title", None).await;
    let state = TestAppStateBuilder::new()
        .provider_base_url("openai", llm.url("/v1"))
        .env_lookup(|_| None)
        .build();
    state
        .stores
        .vault
        .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let body = post_run_manifest(&app, minimal_manifest_json(MINIMAL_DOT)).await;
    let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap();

    assert_eq!(body["title"], "Test");
    wait_for_run_title(&state, run_id, "Generated deploy title").await;
    assert_eq!(title_update_event_count(&state, run_id).await, 1);
    title_mock.assert_async().await;
}

#[tokio::test]
async fn create_run_with_explicit_title_skips_generated_title_work() {
    let llm = MockServer::start_async().await;
    let title_mock = mock_openai_title_response(&llm, "Generated deploy title", None).await;
    let state = TestAppStateBuilder::new()
        .provider_base_url("openai", llm.url("/v1"))
        .env_lookup(|_| None)
        .build();
    state
        .stores
        .vault
        .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let mut manifest = minimal_manifest_json(MINIMAL_DOT);
    manifest["title"] = json!("Caller title");

    let body = post_run_manifest(&app, manifest).await;
    let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap();
    // The spawn gate is synchronous in `create_run`, so once the response
    // returns we know no title task was scheduled. No sleep needed.

    assert_eq!(
        state
            .stores
            .runs
            .get_cached_summary(&run_id, Utc::now())
            .await
            .unwrap()
            .unwrap()
            .title,
        "Caller title"
    );
    assert_eq!(title_update_event_count(&state, run_id).await, 0);
    title_mock.assert_calls_async(0).await;
}

#[tokio::test]
async fn create_run_without_ready_llm_provider_rejects_implicit_model_selection() {
    let state = TestAppStateBuilder::new().env_lookup(|_| None).build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header("content-type", "application/json")
                .body(Body::from(minimal_manifest_json(MINIMAL_DOT).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;

    assert!(
        body["errors"][0]["detail"]
            .as_str()
            .is_some_and(|detail| detail.contains("no default model is available")),
        "unexpected response: {body}"
    );
    assert!(state.runs.lock().expect("runs lock poisoned").is_empty());
}

#[tokio::test]
async fn generated_title_failure_leaves_deterministic_title_unchanged() {
    let llm = MockServer::start_async().await;
    let title_mock = llm
        .mock_async(|when, then| {
            when.method(POST).path("/v1/responses");
            then.status(500)
                .header("content-type", "application/json")
                .json_body(json!({"error": {"message": "boom"}}));
        })
        .await;
    let state = TestAppStateBuilder::new()
        .provider_base_url("openai", llm.url("/v1"))
        .env_lookup(|_| None)
        .build();
    state
        .stores
        .vault
        .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let body = post_run_manifest(&app, minimal_manifest_json(MINIMAL_DOT)).await;
    let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap();
    wait_for_mock_hits(&title_mock, 1).await;
    tokio::time::sleep(std::time::Duration::from_millis(25)).await;

    assert_eq!(
        state
            .stores
            .runs
            .get_cached_summary(&run_id, Utc::now())
            .await
            .unwrap()
            .unwrap()
            .title,
        "Test"
    );
    assert_eq!(title_update_event_count(&state, run_id).await, 0);
}

#[tokio::test]
async fn generated_title_does_not_overwrite_user_title_edit() {
    let llm = MockServer::start_async().await;
    let title_mock = mock_openai_title_response(
        &llm,
        "Generated deploy title",
        Some(std::time::Duration::from_millis(150)),
    )
    .await;
    let state = TestAppStateBuilder::new()
        .provider_base_url("openai", llm.url("/v1"))
        .env_lookup(|_| None)
        .build();
    state
        .stores
        .vault
        .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let body = post_run_manifest(&app, minimal_manifest_json(MINIMAL_DOT)).await;
    let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap();
    let patch = Request::builder()
        .method("PATCH")
        .uri(api(&format!("/runs/{run_id}")))
        .header("content-type", "application/json")
        .body(Body::from(json!({"title": "User title"}).to_string()))
        .unwrap();
    let response = app.clone().oneshot(patch).await.unwrap();
    response_json!(response, StatusCode::OK).await;

    wait_for_mock_hits(&title_mock, 1).await;
    tokio::time::sleep(std::time::Duration::from_millis(250)).await;

    assert_eq!(
        state
            .stores
            .runs
            .get_cached_summary(&run_id, Utc::now())
            .await
            .unwrap()
            .unwrap()
            .title,
        "User title"
    );
    assert_eq!(title_update_event_count(&state, run_id).await, 1);
}

async fn post_run_manifest(app: &Router, manifest: serde_json::Value) -> serde_json::Value {
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header("content-type", "application/json")
                .body(Body::from(manifest.to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    response_json!(response, StatusCode::CREATED).await
}

#[tokio::test]
async fn post_runs_ignores_removed_run_id_input() {
    let app = crate::test_support::build_test_router(test_app_state());
    let submitted_run_id = RunId::new();
    let mut manifest = minimal_manifest_json(MINIMAL_DOT);
    manifest["run_id"] = json!(submitted_run_id.to_string());

    let created = post_run_manifest(&app, manifest).await;
    let allocated_run_id = created["id"]
        .as_str()
        .expect("create response should contain an id")
        .parse::<RunId>()
        .expect("create response id should be valid");

    assert_ne!(allocated_run_id, submitted_run_id);
}

/// Old clients may still send the removed `target.identifier` and
/// `goal.path` manifest properties. The server must keep accepting such
/// bodies as unknown fields: the workflow is selected by `target.path`
/// and the goal comes from the resolved `goal.text`.
#[tokio::test]
async fn post_runs_accepts_legacy_manifest_metadata_properties() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let picked_dot = r"digraph PickedFlow {
        start [shape=Mdiamond]
        exit [shape=Msquare]
        start -> exit
    }";
    let decoy_dot = r"digraph DecoyFlow {
        start [shape=Mdiamond]
        exit [shape=Msquare]
        start -> exit
    }";
    let manifest = serde_json::json!({
        "version": 1,
        "cwd": "/tmp",
        "target": {
            // Legacy display metadata: deliberately names the decoy entry
            // to prove selection never reads it.
            "identifier": "decoy.fabro",
            "path": "picked.fabro",
        },
        "goal": {
            "type": "file",
            "text": "Goal text from the legacy body",
            "path": "/tmp/original/goal.md",
        },
        "workflows": {
            "picked.fabro": { "source": picked_dot, "files": {} },
            "decoy.fabro": { "source": decoy_dot, "files": {} },
        },
    });

    let created = post_run_manifest(&app, manifest).await;
    let run_id = created["id"]
        .as_str()
        .expect("create response should contain an id")
        .parse::<RunId>()
        .expect("create response id should be valid");

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let run_state = run_store.state().await.unwrap();
    assert_eq!(run_state.spec.graph.name, "PickedFlow");
    assert_eq!(
        run_state.spec.graph.goal(),
        "Goal text from the legacy body"
    );
}

#[tokio::test]
async fn post_runs_create_regression_keeps_api_behavior_without_automation_metadata() {
    let state = TestAppStateBuilder::new()
        .env_lookup(|_| None)
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let mut manifest = minimal_manifest_json(MINIMAL_DOT);
    manifest["title"] = json!("API title");

    let body = post_run_manifest(&app, manifest).await;
    let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap();

    assert_eq!(body["title"], "API title");
    assert!(body["automation"].is_null());
    assert_eq!(body["lifecycle"]["status"]["kind"], "submitted");
    let summary = state
        .stores
        .runs
        .get_cached_summary(&run_id, Utc::now())
        .await
        .unwrap()
        .unwrap();
    assert!(summary.automation.is_none());
}

#[tokio::test]
async fn create_run_from_manifest_helper_persists_without_automation_metadata() {
    let state = TestAppStateBuilder::new()
        .env_lookup(|_| None)
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .build();
    let manifest: RunManifest = serde_json::from_value(minimal_manifest_json(MINIMAL_DOT)).unwrap();
    let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();
    let run_id = RunId::new();

    let response = Box::pin(handler::runs::create_run_from_manifest(
        Arc::clone(&state),
        handler::runs::CreateRunFromManifestRequest {
            manifest,
            submitted_manifest_bytes,
            explicit_run_id: Some(run_id),
            explicit_title_supplied: false,
            actor: Principal::System {
                system_kind: SystemActorKind::Engine,
            },
            headers: HeaderMap::new(),
            automation: None,
        },
    ))
    .await;

    let body = response_json!(response, StatusCode::CREATED).await;
    assert_eq!(body["id"], run_id.to_string());
    assert!(body["automation"].is_null());
    let summary = state
        .stores
        .runs
        .get_cached_summary(&run_id, Utc::now())
        .await
        .unwrap()
        .unwrap();
    assert!(summary.automation.is_none());
}

#[tokio::test]
async fn create_run_from_manifest_helper_persists_automation_metadata() {
    let state = TestAppStateBuilder::new()
        .env_lookup(|_| None)
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .build();
    let manifest: RunManifest = serde_json::from_value(minimal_manifest_json(MINIMAL_DOT)).unwrap();
    let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();
    let run_id = RunId::new();
    let automation = fabro_types::AutomationRef {
        id:         "nightly".to_string(),
        name:       Some("Nightly".to_string()),
        trigger_id: Some("schedule".to_string()),
    };

    let response = Box::pin(handler::runs::create_run_from_manifest(
        Arc::clone(&state),
        handler::runs::CreateRunFromManifestRequest {
            manifest,
            submitted_manifest_bytes,
            explicit_run_id: Some(run_id),
            explicit_title_supplied: false,
            actor: Principal::System {
                system_kind: SystemActorKind::Engine,
            },
            headers: HeaderMap::new(),
            automation: Some(automation.clone()),
        },
    ))
    .await;

    let body = response_json!(response, StatusCode::CREATED).await;
    assert_eq!(body["automation"]["id"], automation.id);
    assert_eq!(
        body["automation"]["name"],
        automation.name.as_deref().unwrap()
    );
    assert_eq!(
        body["automation"]["trigger_id"],
        automation.trigger_id.as_deref().unwrap()
    );
    let summary = state
        .stores
        .runs
        .get_cached_summary(&run_id, Utc::now())
        .await
        .unwrap()
        .unwrap();
    assert_eq!(summary.automation, Some(automation));
}

#[tokio::test]
async fn create_run_from_manifest_pins_compiled_and_persisted_behavior() {
    let state = TestAppStateBuilder::new()
        .runtime_settings(
            default_test_server_settings(),
            manifest_run_defaults_from_toml(
                r#"
[run.metadata]
server-label = "server"
layer = "server"
"#,
            ),
        )
        .env_lookup(|_| None)
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .build();
    let run_id = RunId::new();
    let dot = r#"digraph CompilePin {
        graph [goal="Graph goal", target="{{ inputs.target }}"]
        start [shape=Mdiamond]
        work [prompt="Ship {{ inputs.target }}", model="gpt-5.4"]
        exit [shape=Msquare]
        start -> work -> exit
    }"#;
    let mut manifest_json = minimal_manifest_json(dot);
    manifest_json["title"] = json!("  Pinned create  ");
    manifest_json["goal"] = json!({
        "type": "value",
        "text": "Inline release goal"
    });
    manifest_json["args"] = json!({
        "model": "gpt-5.4",
        "input": ["target=payments"]
    });
    manifest_json["configs"] = json!([{
        "type": "project",
        "path": "/tmp/project/.fabro/project.toml",
        "source": r#"
_version = 1

[project]
name = "payments-project"

[run.metadata]
project-label = "project"
layer = "project"
"#
    }]);
    manifest_json["cwd"] = json!("/tmp/project");
    manifest_json["git"] = json!({
        "origin_url": "https://github.com/acme/payments.git",
        "branch": "feature/compiler",
        "sha": "0123456789abcdef",
        "dirty": "clean"
    });
    let manifest: RunManifest = serde_json::from_value(manifest_json).unwrap();
    let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();
    let mut headers = HeaderMap::new();
    headers.insert(
        header::USER_AGENT,
        "fabro-cli/9.8.7".parse().expect("user agent should parse"),
    );

    let response = Box::pin(handler::runs::create_run_from_manifest(
        Arc::clone(&state),
        handler::runs::CreateRunFromManifestRequest {
            manifest,
            submitted_manifest_bytes: submitted_manifest_bytes.clone(),
            explicit_run_id: Some(run_id),
            explicit_title_supplied: true,
            actor: Principal::System {
                system_kind: SystemActorKind::Engine,
            },
            headers,
            automation: None,
        },
    ))
    .await;

    let body = response_json!(response, StatusCode::CREATED).await;
    assert_eq!(body["id"], run_id.to_string());
    assert_eq!(body["title"], "Pinned create");
    assert_eq!(body["lifecycle"]["status"]["kind"], "submitted");

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let events = run_store.list_events().await.unwrap();
    assert_eq!(
        events
            .iter()
            .map(|envelope| envelope.event.event_name())
            .collect::<Vec<_>>(),
        vec!["run.created", "run.submitted"]
    );
    let run_state = run_store.state().await.unwrap();
    let spec = &run_state.spec;
    assert_eq!(spec.run_id, run_id);
    assert_eq!(spec.graph.goal(), "Inline release goal");
    assert_eq!(
        spec.graph.attrs.get("target").and_then(AttrValue::as_str),
        Some("{{ inputs.target }}")
    );
    assert_eq!(
        spec.graph.nodes["work"]
            .attrs
            .get("prompt")
            .and_then(AttrValue::as_str),
        Some("Ship payments")
    );
    assert_eq!(
        spec.graph.nodes["work"]
            .attrs
            .get("model")
            .and_then(AttrValue::as_str),
        Some("gpt-5.4")
    );
    assert_eq!(
        spec.graph.nodes["work"]
            .attrs
            .get("provider")
            .and_then(AttrValue::as_str),
        Some("openai")
    );
    assert_eq!(spec.settings.run.model.name.as_deref(), Some("gpt-5.4"));
    assert_eq!(spec.settings.run.model.provider.as_deref(), Some("openai"));
    assert_eq!(
        spec.settings.run.inputs.get("target"),
        Some(&toml::Value::String("payments".to_string()))
    );
    assert_eq!(
        spec.settings.project.name.as_deref(),
        Some("payments-project")
    );
    assert_eq!(
        spec.labels.get("project-label").map(String::as_str),
        Some("project")
    );
    assert_eq!(
        spec.labels.get("layer").map(String::as_str),
        Some("project")
    );
    assert_eq!(
        spec.git.as_ref().map(|git| git.origin_url.as_str()),
        Some("https://github.com/acme/payments.git")
    );

    let created = events[0].event.to_value().unwrap();
    assert_eq!(created["properties"]["title"], "Pinned create");
    assert_eq!(created["properties"]["labels"]["project-label"], "project");
    assert_eq!(
        created["properties"]["provenance"]["client"]["user_agent"],
        "fabro-cli/9.8.7"
    );
    assert_eq!(
        created["properties"]["provenance"]["subject"]["kind"],
        "system"
    );
    let manifest_blob = created["properties"]["manifest_blob"]
        .as_str()
        .expect("run.created should carry the submitted source blob")
        .parse::<BlobHash>()
        .unwrap();
    let persisted_manifest = run_store
        .read_blob(&manifest_blob)
        .await
        .unwrap()
        .expect("submitted source blob should exist");
    assert_eq!(persisted_manifest.as_ref(), submitted_manifest_bytes);
}

#[tokio::test]
async fn create_run_from_manifest_pins_compiler_http_error_mappings() {
    let cases = [
        (
            {
                let mut manifest = minimal_manifest_json(MINIMAL_DOT);
                manifest["version"] = json!(2);
                manifest
            },
            "unsupported manifest version 2",
        ),
        (
            minimal_manifest_json(
                r#"digraph Test {
                    graph [goal="Test"]
                    start [shape=Mdiamond]
                    work [prompt="Use {{ vars.MISSING }}"]
                    exit [shape=Msquare]
                    start -> work -> exit
                }"#,
            ),
            "Validation failed",
        ),
        (
            {
                let mut manifest = minimal_manifest_json(
                    r#"digraph Test {
                        graph [goal="Test"]
                        start [shape=Mdiamond]
                        work [prompt="Do work", model="gpt-5.4", provider="missing-provider"]
                        exit [shape=Msquare]
                        start -> work -> exit
                    }"#,
                );
                manifest["args"] = json!({
                    "model": "gpt-5.4",
                    "provider": "missing-provider"
                });
                manifest
            },
            "Model selection failed: unknown model provider 'missing-provider'",
        ),
    ];

    for (manifest_json, expected_detail) in cases {
        let state = TestAppStateBuilder::new()
            .env_lookup(|_| None)
            .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
            .build();
        let manifest: RunManifest = serde_json::from_value(manifest_json).unwrap();
        let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();

        let response = Box::pin(handler::runs::create_run_from_manifest(
            state,
            handler::runs::CreateRunFromManifestRequest {
                manifest,
                submitted_manifest_bytes,
                explicit_run_id: Some(RunId::new()),
                explicit_title_supplied: true,
                actor: Principal::System {
                    system_kind: SystemActorKind::Engine,
                },
                headers: HeaderMap::new(),
                automation: None,
            },
        ))
        .await;

        let body = response_json!(response, StatusCode::BAD_REQUEST).await;
        assert_eq!(body["errors"][0]["detail"], expected_detail);
    }
}

#[tokio::test]
async fn create_run_from_manifest_preserves_competing_preparation_error_precedence() {
    let mut workflow_before_title = minimal_manifest_json(MINIMAL_DOT);
    workflow_before_title["workflows"]["workflow.fabro"]["config"] = json!({
        "path": "workflow.toml",
        "source": "_version = 1\n[run.unknown]\nvalue = true\n",
    });
    workflow_before_title["title"] = json!("   ");

    let mut project_parse_before_later_path = minimal_manifest_json(MINIMAL_DOT);
    project_parse_before_later_path["configs"] = json!([
        {
            "type": "project",
            "path": "/tmp/.fabro/project.toml",
            "source": "_version = 1\n[run.unknown]\nvalue = true\n",
        },
        {
            "type": "project",
            "source": "_version = 1\n",
        },
    ]);

    for (manifest_json, expected_detail) in [
        (workflow_before_title, "Failed to parse run config TOML"),
        (
            project_parse_before_later_path,
            "Failed to parse run config TOML",
        ),
    ] {
        let state = TestAppStateBuilder::new().build();
        let manifest: RunManifest = serde_json::from_value(manifest_json).unwrap();
        let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();

        let response = Box::pin(handler::runs::create_run_from_manifest(
            state,
            handler::runs::CreateRunFromManifestRequest {
                manifest,
                submitted_manifest_bytes,
                explicit_run_id: None,
                explicit_title_supplied: true,
                actor: Principal::System {
                    system_kind: SystemActorKind::Engine,
                },
                headers: HeaderMap::new(),
                automation: None,
            },
        ))
        .await;

        let body = response_json!(response, StatusCode::BAD_REQUEST).await;
        assert_eq!(body["errors"][0]["detail"], expected_detail);
    }
}

#[tokio::test]
async fn create_run_from_manifest_resolves_generated_id_after_variable_snapshot() {
    let state = TestAppStateBuilder::new()
        .env_lookup(|_| None)
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .build();
    let variable = state
        .stores
        .variables
        .set("OWNER", "payments", None)
        .await
        .expect("test variable should persist");
    let manifest: RunManifest = serde_json::from_value(minimal_manifest_json(
        r#"digraph Test {
            graph [goal="Test"]
            start [shape=Mdiamond]
            work [prompt="Ship {{ vars.OWNER }}"]
            exit [shape=Msquare]
            start -> work -> exit
        }"#,
    ))
    .unwrap();
    let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap();

    let response = Box::pin(handler::runs::create_run_from_manifest(
        state,
        handler::runs::CreateRunFromManifestRequest {
            manifest,
            submitted_manifest_bytes,
            explicit_run_id: None,
            explicit_title_supplied: false,
            actor: Principal::System {
                system_kind: SystemActorKind::Engine,
            },
            headers: HeaderMap::new(),
            automation: None,
        },
    ))
    .await;

    let body = response_json!(response, StatusCode::CREATED).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();
    // RunId is a ULID whose timestamp only has millisecond precision, so
    // truncate the variable timestamp to milliseconds before comparing.
    assert!(run_id.created_at() >= variable.updated_at.trunc_subsecs(3));
}

#[tokio::test]
async fn fake_automation_materializer_injection_captures_input_and_returns_manifest() {
    let materialized_manifest: RunManifest =
        serde_json::from_value(minimal_manifest_json(MINIMAL_DOT)).unwrap();
    let fake = TestAutomationRunMaterializer::succeed(
        materialized_manifest.clone(),
        b"{\"fake\":true}".to_vec(),
    );
    let state = TestAppStateBuilder::new()
        .automation_materializer(fake.clone())
        .build();
    let run_id = RunId::new();
    let user_settings_path = PathBuf::from("/tmp/fabro/settings.toml");
    let temp_root = PathBuf::from("/tmp/fabro/automation");
    let target = AutomationTarget {
        repository:   "fabro-sh/fabro".to_string(),
        ref_selector: "main".to_string(),
        workflow:     "demo".to_string(),
    };

    let output = state
        .materialize_automation_run(AutomationRunMaterializeInput {
            automation_id: AutomationId::new("nightly").unwrap(),
            target: target.clone(),
            run_id,
            user_settings_path: user_settings_path.clone(),
            temp_root: temp_root.clone(),
        })
        .await
        .expect("fake materializer should succeed");

    assert_eq!(
        serde_json::to_value(&output.manifest).unwrap(),
        serde_json::to_value(&materialized_manifest).unwrap()
    );
    assert_eq!(output.submitted_manifest_bytes, b"{\"fake\":true}".to_vec());
    let captured = fake.captured_inputs();
    assert_eq!(captured.len(), 1);
    assert_eq!(captured[0].automation_id.as_str(), "nightly");
    assert_eq!(captured[0].target, target);
    assert_eq!(captured[0].run_id, run_id);
    assert_eq!(captured[0].user_settings_path, user_settings_path);
    assert_eq!(captured[0].temp_root, temp_root);
}

async fn mock_openai_title_response<'a>(
    server: &'a MockServer,
    title: &str,
    delay: Option<std::time::Duration>,
) -> httpmock::Mock<'a> {
    let title = title.to_string();
    server
        .mock_async(move |when, then| {
            when.method(POST).path("/v1/responses");
            let then = then
                .status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload(
                    &json!({ "title": title }).to_string(),
                ));
            if let Some(delay) = delay {
                then.delay(delay);
            }
        })
        .await
}

async fn wait_for_run_title(state: &AppState, run_id: RunId, expected: &str) {
    for _ in 0..50 {
        let title = state
            .stores
            .runs
            .get_cached_summary(&run_id, Utc::now())
            .await
            .unwrap()
            .unwrap()
            .title;
        if title == expected {
            return;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }
    panic!("run {run_id} title did not become {expected:?}");
}

async fn wait_for_mock_hits(mock: &httpmock::Mock<'_>, expected: usize) {
    for _ in 0..50 {
        if mock.calls_async().await >= expected {
            return;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }
    panic!("mock did not receive {expected} request(s)");
}

/// Poll `GET /runs/{id}/pull_request/creation` until the creation leaves
/// `pending`, returning the terminal creation body.
async fn wait_for_pull_request_creation(app: &Router, run_id: RunId) -> serde_json::Value {
    for _ in 0..150 {
        let response = app
            .clone()
            .oneshot(
                Request::builder()
                    .method("GET")
                    .uri(api(&format!("/runs/{run_id}/pull_request/creation")))
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap();
        let body = response_json!(response, StatusCode::OK).await;
        if body["status"] != "pending" {
            return body;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }
    panic!("pull request creation for run {run_id} did not finish");
}

async fn title_update_event_count(state: &AppState, run_id: RunId) -> usize {
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    run_store
        .list_events()
        .await
        .unwrap()
        .into_iter()
        .filter(|event| event.event.event_name() == "run.title.updated")
        .count()
}

#[tokio::test]
async fn validate_endpoint_returns_workflow_summary_without_preflight_checks() {
    let app = test_app_with();
    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/validate"))
                .header("content-type", "application/json")
                .body(manifest_body(MINIMAL_DOT))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["ok"], true);
    assert_eq!(body["workflow"]["name"], "Test");
    assert_eq!(body["workflow"]["nodes"], 2);
    assert_eq!(body["workflow"]["edges"], 1);
    assert!(body.get("checks").is_none());
}

#[tokio::test]
async fn validate_endpoint_uses_app_state_catalog_for_model_diagnostics() {
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "https://api.acme.test/v1"

[providers.acme.auth]
credentials = ["env:ACME_API_KEY"]

[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true

[models."acme-large".limits]
context_window = 128000

[models."acme-large".features]
tools = true
vision = false
reasoning = false
"#,
    )
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    let app = crate::test_support::build_test_router(state);
    let dot = r#"digraph Test {
        graph [goal="Test"]
        start [shape=Mdiamond]
        work [model="acme-large", provider="acme", prompt="Do it"]
        exit  [shape=Msquare]
        start -> work -> exit
    }"#;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/validate"))
                .header("content-type", "application/json")
                .body(manifest_body(dot))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let diagnostics = body["workflow"]["diagnostics"].as_array().unwrap();

    assert!(
        diagnostics
            .iter()
            .all(|diagnostic| diagnostic["rule"] != "node_model_known"),
        "custom model/provider should validate against app-state catalog: {body}"
    );
}

#[tokio::test]
async fn validate_endpoint_returns_template_source_coordinates() {
    let app = test_app_with();
    let dot = r#"digraph ValidatePlan {
        start [shape=Mdiamond, label="Start"]
        exit  [shape=Msquare, label="Exit"]
        test_imported_prompt [label="moo" prompt="@test.md"]
        start -> test_imported_prompt -> exit
    }"#;
    let manifest = serde_json::json!({
        "version": 1,
        "cwd": "/tmp",
        "target": {
            "path": "workflow.fabro",
        },
        "workflows": {
            "workflow.fabro": {
                "source": dot,
                "files": {
                    "test.md": {
                        "content": "{{ inputs.foo }}",
                        "ref": {
                            "type": "file_inline",
                            "original": "test.md",
                            "from": "workflow.fabro",
                        },
                    },
                },
            },
        },
    });

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/validate"))
                .header("content-type", "application/json")
                .body(Body::from(serde_json::to_string(&manifest).unwrap()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let diagnostics = body["workflow"]["diagnostics"].as_array().unwrap();
    let diagnostic = diagnostics
        .iter()
        .find(|diagnostic| diagnostic["rule"] == "template_undefined_variable")
        .expect("expected template diagnostic");

    assert_eq!(diagnostic["source_path"], "test.md");
    assert_eq!(diagnostic["line"], 1);
    assert_eq!(diagnostic["column"], 4);
    assert!(
        diagnostic["node_id"]
            .as_str()
            .unwrap()
            .contains("test_imported_prompt")
    );
}

async fn create_run_for_target(app: &Router, target_path: &str, dot_source: &str) -> String {
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body_for(target_path, dot_source))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    body["id"].as_str().unwrap().to_string()
}

async fn create_run_for_target_with_workflow_name(
    app: &Router,
    target_path: &str,
    dot_source: &str,
    workflow_name: &str,
) -> String {
    let mut manifest = manifest_json(target_path, dot_source);
    manifest["workflows"][target_path]["config"] = serde_json::json!({
        "path": "workflow.toml",
        "source": format!("_version = 1\n\n[workflow]\nname = {workflow_name:?}\n"),
    });
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(Body::from(serde_json::to_string(&manifest).unwrap()))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    body["id"].as_str().unwrap().to_string()
}

fn named_workflow_dot(name: &str, goal: &str) -> String {
    format!(
        r#"digraph {name} {{
    graph [goal="{goal}"]
    start [shape=Mdiamond]
    exit  [shape=Msquare]
    start -> exit
}}"#
    )
}

fn multipart_body(
    boundary: &str,
    manifest: &serde_json::Value,
    files: &[(&str, &str, &[u8])],
) -> Body {
    let mut body = Vec::new();
    body.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
    body.extend_from_slice(b"Content-Disposition: form-data; name=\"manifest\"\r\n");
    body.extend_from_slice(b"Content-Type: application/json\r\n\r\n");
    body.extend_from_slice(serde_json::to_string(manifest).unwrap().as_bytes());
    body.extend_from_slice(b"\r\n");

    for (part, filename, bytes) in files {
        body.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
        body.extend_from_slice(
            format!("Content-Disposition: form-data; name=\"{part}\"; filename=\"{filename}\"\r\n")
                .as_bytes(),
        );
        body.extend_from_slice(b"Content-Type: application/octet-stream\r\n\r\n");
        body.extend_from_slice(bytes);
        body.extend_from_slice(b"\r\n");
    }

    body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
    Body::from(body)
}

/// Create a run via POST /runs, then start it via POST /runs/{id}/start.
/// Returns the run_id string.
async fn create_and_start_run(app: &Router, dot_source: &str) -> String {
    let run_id = create_run(app, dot_source).await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/start")))
        .body(Body::empty())
        .unwrap();
    app.clone().oneshot(req).await.unwrap();

    run_id
}

async fn create_durable_run_with_events(
    state: &Arc<AppState>,
    run_id: RunId,
    events: &[workflow_event::Event],
) {
    let run_store = state.stores.runs.create_run(&run_id).await.unwrap();
    if !matches!(
        events.first(),
        Some(workflow_event::Event::RunCreated { .. })
    ) {
        append_default_run_created(&run_store, run_id).await;
    }
    let needs_running = events.iter().any(|event| {
        matches!(
            event,
            workflow_event::Event::WorkflowRunCompleted { .. }
                | workflow_event::Event::WorkflowRunFailed { .. }
        )
    });
    let has_starting = events
        .iter()
        .any(|event| matches!(event, workflow_event::Event::RunStarting));
    let has_runnable = events
        .iter()
        .any(|event| matches!(event, workflow_event::Event::RunRunnable { .. }));
    let has_running = events
        .iter()
        .any(|event| matches!(event, workflow_event::Event::RunRunning));
    let mut inserted_runnable = has_runnable;
    let mut inserted_starting = has_starting;
    for event in events {
        if !inserted_runnable
            && matches!(
                event,
                workflow_event::Event::RunStarting
                    | workflow_event::Event::RunRunning
                    | workflow_event::Event::RunBlocked { .. }
                    | workflow_event::Event::RunPaused
                    | workflow_event::Event::WorkflowRunCompleted { .. }
                    | workflow_event::Event::WorkflowRunFailed { .. }
            )
        {
            workflow_event::append_event(
                &run_store,
                &run_id,
                &workflow_event::Event::RunRunnable {
                    source: fabro_types::RunRunnableSource::StartRequested,
                    actor:  None,
                },
            )
            .await
            .unwrap();
            inserted_runnable = true;
        }
        if !inserted_starting
            && matches!(
                event,
                workflow_event::Event::RunRunning
                    | workflow_event::Event::RunBlocked { .. }
                    | workflow_event::Event::RunPaused
                    | workflow_event::Event::WorkflowRunCompleted { .. }
                    | workflow_event::Event::WorkflowRunFailed { .. }
            )
        {
            workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
                .await
                .unwrap();
            inserted_starting = true;
        }
        if needs_running
            && !has_running
            && matches!(
                event,
                workflow_event::Event::WorkflowRunCompleted { .. }
                    | workflow_event::Event::WorkflowRunFailed { .. }
            )
        {
            workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
                .await
                .unwrap();
        }
        workflow_event::append_event(&run_store, &run_id, event)
            .await
            .unwrap();
    }
}

fn stage_started_event(node_id: &str, handler_type: &str) -> workflow_event::Event {
    workflow_event::Event::StageStarted {
        graph_visit:           None,
        resumed_from_stage_id: None,
        node_id:               node_id.to_string(),
        name:                  node_id.to_string(),
        index:                 1,
        handler_type:          handler_type.to_string(),
        attempt:               1,
        max_attempts:          1,
    }
}

fn command_started_event(node_id: &str) -> workflow_event::Event {
    workflow_event::Event::CommandStarted {
        node_id:    node_id.to_string(),
        script:     "echo ok".to_string(),
        command:    "echo ok".to_string(),
        language:   "shell".to_string(),
        timeout_ms: None,
    }
}

fn agent_session_activated_event(node_id: &str, visit: u32) -> workflow_event::Event {
    workflow_event::Event::AgentSessionActivated {
        node_id: node_id.to_string(),
        visit,
        session_id: "session-1".to_string(),
        thread_id: None,
        provider: Some("openai".to_string()),
        model: Some("gpt-5.4".to_string()),
        reasoning_effort: None,
        speed: None,
        permission_level: None,
        capabilities: Vec::new(),
    }
}

fn stage_completed_event(node_id: &str) -> workflow_event::Event {
    workflow_event::Event::StageCompleted {
        node_id: node_id.to_string(),
        name: node_id.to_string(),
        index: 1,
        timing: StageTiming::wall_only(42),
        status: "succeeded".to_string(),
        preferred_label: None,
        suggested_next_ids: Vec::new(),
        billing: None,
        failure: None,
        notes: None,
        files_touched: Vec::new(),
        context_updates: None,
        jump_to_node: None,
        context_values: None,
        node_visits: None,
        loop_failure_signatures: None,
        restart_failure_signatures: None,
        response: None,
        attempt: 1,
        max_attempts: 1,
    }
}

fn context_window_event(
    stage: &str,
    visit: u32,
    context_window: StageContextWindowProjection,
) -> workflow_event::Event {
    workflow_event::Event::Agent {
        stage: stage.to_string(),
        visit,
        event: fabro_agent::AgentEvent::AssistantMessage {
            text:            "assistant response".to_string(),
            model:           ModelRef {
                provider: ProviderId::openai(),
                model_id: "gpt-5.4".into(),
                speed:    None,
            },
            usage:           TokenCounts::default(),
            cost_usd:        None,
            cost_source:     None,
            tool_call_count: 0,
            context_window:  Some(context_window),
            reasoning:       None,
        },
        session_id: Some("session-1".to_string()),
        parent_session_id: None,
        tool_call_id: None,
    }
}

fn context_window_snapshot(
    input_tokens: u64,
    warnings: Vec<StageContextWindowWarning>,
) -> StageContextWindowProjection {
    StageContextWindowProjection {
        provider: "openai".to_string(),
        model: "gpt-5.4".to_string(),
        context_window_tokens: 400_000,
        input_tokens,
        usage_percent: input_tokens as f64 * 100.0 / 400_000.0,
        count_method: StageContextWindowCountMethod::ResponseUsageScaledBreakdown,
        staleness: StageContextWindowStaleness::Live,
        generated_at: Utc::now(),
        event_seq: None,
        breakdown: vec![StageContextWindowBreakdownItem {
            category:      StageContextWindowCategory::Conversation,
            tokens:        input_tokens,
            usage_percent: input_tokens as f64 * 100.0 / 400_000.0,
        }],
        warnings,
    }
}

async fn append_default_run_created(run_store: &fabro_store::RunDatabase, run_id: RunId) {
    workflow_event::append_event(run_store, &run_id, &workflow_event::Event::RunCreated {
        run_id,
        title: None,
        settings: serde_json::to_value(WorkflowSettings::default()).unwrap(),
        graph: serde_json::to_value(Graph::new("test")).unwrap(),
        workflow_source: None,
        labels: std::collections::BTreeMap::default(),
        source_directory: None,
        workflow_slug: None,
        workflow_version_id: None,
        automation: None,
        provenance: test_support::test_run_provenance(),
        manifest_blob: None,
        spec_blob: None,
        git: None,
        fork_source_ref: None,
        retried_from: None,
        parent_id: None,
        web_url: None,
    })
    .await
    .unwrap();
}

fn workflow_settings_with_run_notifications(
    run_toml: &str,
    workflow_name: Option<&str>,
) -> WorkflowSettings {
    let mut settings = WorkflowSettingsBuilder::new()
        .server_manifest_defaults(RunLayer::default(), test_environment_defaults())
        .workflow_toml(run_toml)
        .expect("run notification settings should parse")
        .build()
        .expect("run notification settings should resolve");
    settings.workflow.name = workflow_name.map(str::to_string);
    settings
}

fn test_environment_defaults() -> MergeMap<EnvironmentLayer> {
    MergeMap::from(HashMap::from([("default".to_string(), EnvironmentLayer {
        provider: Some("local".to_string()),
        ..EnvironmentLayer::default()
    })]))
}

async fn create_slack_notification_run(
    state: &Arc<AppState>,
    run_id: RunId,
    settings: WorkflowSettings,
    graph_name: &str,
    workflow_slug: Option<&str>,
) -> fabro_store::RunDatabase {
    let run_store = state.stores.runs.create_run(&run_id).await.unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunCreated {
        run_id,
        title: None,
        settings: serde_json::to_value(settings).unwrap(),
        graph: serde_json::to_value(Graph::new(graph_name)).unwrap(),
        workflow_source: None,
        labels: std::collections::BTreeMap::default(),
        source_directory: None,
        workflow_slug: workflow_slug.map(str::to_string),
        workflow_version_id: None,
        automation: None,
        provenance: test_support::test_run_provenance(),
        manifest_blob: None,
        spec_blob: None,
        git: None,
        fork_source_ref: None,
        retried_from: None,
        parent_id: None,
        web_url: None,
    })
    .await
    .unwrap();
    run_store
}

async fn append_slack_notification_event(
    run_store: &fabro_store::RunDatabase,
    run_id: RunId,
    event: &workflow_event::Event,
) -> EventEnvelope {
    workflow_event::append_event(run_store, &run_id, event)
        .await
        .unwrap();
    run_store
        .list_events()
        .await
        .unwrap()
        .last()
        .expect("appended event should be present")
        .clone()
}

async fn mock_slack_post<'a>(
    server: &'a MockServer,
    body_includes: Vec<String>,
    ts: &'static str,
) -> httpmock::Mock<'a> {
    server
        .mock_async(move |when, then| {
            let mut when = when
                .method(POST)
                .path("/chat.postMessage")
                .header("authorization", "Bearer xoxb-test");
            for part in body_includes {
                when = when.body_includes(part);
            }
            then.status(200)
                .header("content-type", "application/json")
                .json_body(json!({
                    "ok": true,
                    "channel": "C123",
                    "ts": ts,
                }));
        })
        .await
}

fn slack_lifecycle_service(base_url: String, default_channel: Option<&str>) -> SlackService {
    SlackService {
        client:          fabro_slack::client::SlackClient::with_api_base_and_http(
            "xoxb-test".to_string(),
            base_url,
            fabro_http::test_http_client().expect("test HTTP client should build"),
        ),
        app_token:       "xapp-test".to_string(),
        default_channel: default_channel.map(str::to_string),
        posted_messages: StdArc::new(StdMutex::new(HashMap::new())),
        thread_registry: StdArc::new(ThreadRegistry::new()),
        connection:      StdArc::new(StdMutex::new(SlackConnectionRuntimeState::default())),
    }
}

fn workflow_run_started_event(run_id: RunId) -> workflow_event::Event {
    workflow_event::Event::WorkflowRunStarted {
        name: "run.started event name".to_string(),
        run_id,
        base_branch: None,
        base_sha: None,
        run_branch: None,
        worktree_dir: None,
        goal: None,
    }
}

#[tokio::test]
async fn slack_lifecycle_run_started_posts_for_matching_enabled_route() {
    let server = MockServer::start_async().await;
    let post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#deploys""##.to_string(),
            "Fabro run started".to_string(),
            "Deploy workflow".to_string(),
            "Open in Fabro".to_string(),
        ],
        "100.1",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.deploys]
enabled = true
provider = "slack"
events = ["run.started", "run.completed", "run.failed"]

[run.notifications.deploys.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store =
        create_slack_notification_run(&state, run_id, settings, "deploy-graph", Some("deploy"))
            .await;
    let envelope =
        append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id))
            .await;

    service
        .handle_event(
            state.as_ref(),
            &envelope,
            Some("https://fabro.example/runs/run-1"),
        )
        .await;

    post.assert_async().await;
    assert!(
        service
            .posted_messages
            .lock()
            .expect("posted messages lock poisoned")
            .is_empty(),
        "lifecycle posts must not use interview message state"
    );
}

#[tokio::test]
async fn slack_lifecycle_run_completed_posts_result_and_duration() {
    let server = MockServer::start_async().await;
    let post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#deploys""##.to_string(),
            "Fabro run completed".to_string(),
            "succeeded — completed".to_string(),
            "1m 5s".to_string(),
        ],
        "100.2",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.deploys]
enabled = true
provider = "slack"
events = ["run.completed"]

[run.notifications.deploys.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    })
    .await
    .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
        .await
        .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
        .await
        .unwrap();
    let envelope = append_slack_notification_event(
        &run_store,
        run_id,
        &workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(65_432),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    )
    .await;

    service.handle_event(state.as_ref(), &envelope, None).await;

    post.assert_async().await;
}

#[tokio::test]
async fn slack_lifecycle_run_failed_posts_failure_result_message_and_duration() {
    let server = MockServer::start_async().await;
    let post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#deploys""##.to_string(),
            "Fabro run failed".to_string(),
            "workflow_error — command &lt;failed&gt; &amp; exited".to_string(),
            "1.2s".to_string(),
        ],
        "100.3",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.deploys]
enabled = true
provider = "slack"
events = ["run.failed"]

[run.notifications.deploys.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    })
    .await
    .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
        .await
        .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
        .await
        .unwrap();
    let envelope = append_slack_notification_event(
        &run_store,
        run_id,
        &workflow_event::Event::WorkflowRunFailed {
            failure:              fabro_types::RunFailure {
                reason: fabro_types::FailureReason::WorkflowError,
                detail: FailureDetail::new(
                    "command <failed> & exited",
                    FailureCategory::Deterministic,
                ),
            },
            timing:               fabro_types::RunTiming::wall_only(1_234),
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    )
    .await;

    service.handle_event(state.as_ref(), &envelope, None).await;

    post.assert_async().await;
}

#[tokio::test]
async fn slack_lifecycle_skips_non_matching_events_and_disabled_routes() {
    let server = MockServer::start_async().await;
    let unexpected = mock_slack_post(&server, Vec::new(), "100.4").await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.disabled]
enabled = false
provider = "slack"
events = ["run.started"]

[run.notifications.disabled.slack]
channel = "#deploys"

[run.notifications.stage]
enabled = true
provider = "slack"
events = ["stage.completed"]

[run.notifications.stage.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    let envelope =
        append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id))
            .await;

    service.handle_event(state.as_ref(), &envelope, None).await;

    unexpected.assert_calls_async(0).await;
}

#[tokio::test]
async fn slack_lifecycle_missing_channel_is_skipped_without_blocking_other_routes() {
    let server = MockServer::start_async().await;
    let post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#ops""##.to_string(),
            "Fabro run started".to_string(),
        ],
        "100.5",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.missing]
enabled = true
provider = "slack"
events = ["run.started"]

[run.notifications.unresolved]
enabled = true
provider = "slack"
events = ["run.started"]

[run.notifications.unresolved.slack]
channel = "{{ env.MISSING_SLACK_CHANNEL }}"

[run.notifications.valid]
enabled = true
provider = "slack"
events = ["run.started"]

[run.notifications.valid.slack]
channel = "#ops"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    let envelope =
        append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id))
            .await;

    service.handle_event(state.as_ref(), &envelope, None).await;

    post.assert_async().await;
}

#[tokio::test]
async fn slack_lifecycle_uses_prior_pull_request_created_details() {
    let server = MockServer::start_async().await;
    let post = mock_slack_post(
        &server,
        vec![
            "Fabro run completed".to_string(),
            "https://github.com/fabro-sh/fabro/pull/42".to_string(),
            "#42".to_string(),
            "Ship &lt;prod&gt; &amp; notify".to_string(),
        ],
        "100.6",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), None);
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.deploys]
enabled = true
provider = "slack"
events = ["run.completed"]

[run.notifications.deploys.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    })
    .await
    .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
        .await
        .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
        .await
        .unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::PullRequestCreated {
            pr_url:      "https://github.com/fabro-sh/fabro/pull/42".to_string(),
            pr_number:   42,
            owner:       "fabro-sh".to_string(),
            repo:        "fabro".to_string(),
            base_branch: "main".to_string(),
            head_branch: "fabro/run/test".to_string(),
            head_sha:    Some("final-sha".to_string()),
            title:       "Ship <prod> & notify".to_string(),
            draft:       false,
        },
    )
    .await
    .unwrap();
    let envelope = append_slack_notification_event(
        &run_store,
        run_id,
        &workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    )
    .await;

    service.handle_event(state.as_ref(), &envelope, None).await;

    post.assert_async().await;
}

#[tokio::test]
async fn slack_interviews_keep_state_separate_from_lifecycle_notifications() {
    let server = MockServer::start_async().await;
    let interview_post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#reviews""##.to_string(),
            "Answer deploy question".to_string(),
        ],
        "200.1",
    )
    .await;
    let lifecycle_post = mock_slack_post(
        &server,
        vec![
            r##""channel":"#deploys""##.to_string(),
            "Fabro run started".to_string(),
        ],
        "200.2",
    )
    .await;
    let state = test_app_state();
    let service = slack_lifecycle_service(server.base_url(), Some("#reviews"));
    let run_id = fixtures::RUN_1;
    let settings = workflow_settings_with_run_notifications(
        r##"
[run.notifications.deploys]
enabled = true
provider = "slack"
events = ["run.started"]

[run.notifications.deploys.slack]
channel = "#deploys"
"##,
        Some("Deploy workflow"),
    );
    let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await;
    let lifecycle_envelope =
        append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id))
            .await;
    let interview_envelope = append_slack_notification_event(
        &run_store,
        run_id,
        &workflow_event::Event::InterviewStarted {
            question_id:     "q-1".to_string(),
            question:        "Answer deploy question".to_string(),
            stage:           "review".to_string(),
            question_type:   "freeform".to_string(),
            options:         Vec::new(),
            allow_freeform:  true,
            timeout_seconds: None,
            context_display: None,
            review_target:   None,
        },
    )
    .await;

    service
        .handle_event(state.as_ref(), &lifecycle_envelope, None)
        .await;
    assert!(
        service
            .posted_messages
            .lock()
            .expect("posted messages lock poisoned")
            .is_empty(),
        "lifecycle notification should not record interview metadata"
    );
    assert!(
        service.thread_registry.resolve("200.2").is_none(),
        "lifecycle notification should not register answer threads"
    );

    service
        .handle_event(state.as_ref(), &interview_envelope, None)
        .await;

    lifecycle_post.assert_async().await;
    interview_post.assert_async().await;
    assert!(
        service
            .posted_messages
            .lock()
            .expect("posted messages lock poisoned")
            .contains_key(&(run_id, "q-1".to_string())),
        "interview posts should retain interview state"
    );
    assert!(
        service.thread_registry.resolve("200.1").is_some(),
        "freeform interview posts should register reply threads"
    );
}

#[tokio::test]
async fn persist_cancelled_run_status_ignores_already_terminal_runs() {
    let state = test_app_state();
    let run_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    persist_cancelled_run_status(state.as_ref(), run_id)
        .await
        .unwrap();

    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    let projection = run_store.state().await.unwrap();
    assert_eq!(projection.status, RunStatus::Succeeded {
        reason: SuccessReason::Completed,
    });
    assert!(!run_store.list_events().await.unwrap().iter().any(|event| {
        matches!(
            event.event.body,
            EventBody::RunFailed(ref props) if props.failure.reason == FailureReason::Cancelled
        )
    }));
}

#[tokio::test]
async fn delete_terminal_managed_run_does_not_send_cancel_signal() {
    let state = test_app_state();
    let run_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let temp = tempfile::tempdir().unwrap();
    let run_dir = temp.path().join("run");
    std::fs::create_dir_all(&run_dir).unwrap();
    let cancel_token = CancellationToken::new();
    let mut run = managed_run(
        MINIMAL_DOT.to_string(),
        RunStatus::Running,
        Utc::now(),
        run_dir,
        RunExecutionMode::Start,
    );
    run.cancel_token = Some(cancel_token.clone());
    let (cancel_tx, _cancel_rx) = oneshot::channel();
    run.cancel_tx = Some(cancel_tx);
    state
        .runs
        .lock()
        .expect("runs lock poisoned")
        .insert(run_id, run);

    delete_run_internal(state.as_ref(), run_id, true)
        .await
        .unwrap();

    assert!(!cancel_token.is_cancelled());
}

/// Append a stage lifecycle event with an explicit `StageScope`, so the
/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare
/// [`workflow_event::append_event`] helper only writes `node_id` because
/// stage lifecycle variants don't carry visit in their payload — production
/// always emits via `Emitter::emit_scoped`.
async fn append_scoped_stage_event(
    state: &Arc<AppState>,
    run_id: RunId,
    node_id: &str,
    visit: u32,
    event: &workflow_event::Event,
) {
    let scope = fabro_workflow::event::StageScope {
        node_id: node_id.to_string(),
        visit,
        parallel_group_id: None,
        parallel_branch_id: None,
    };
    append_event_with_scope(state, run_id, event, &scope).await;
}

async fn append_event_with_scope(
    state: &Arc<AppState>,
    run_id: RunId,
    event: &workflow_event::Event,
    scope: &fabro_workflow::event::StageScope,
) {
    let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(scope));
    let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap();
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    run_store.append_event(&payload).await.unwrap();
}

fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str {
    body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|stage| stage["id"] == id)
        .and_then(|stage| stage["status"].as_str())
        .unwrap()
}

#[tokio::test]
async fn list_run_stages_projects_retrying_until_completion() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "setup",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "setup".to_string(),
            name:                  "Setup".to_string(),
            index:                 0,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "setup",
        1,
        &workflow_event::Event::StageCompleted {
            node_id: "setup".to_string(),
            name: "Setup".to_string(),
            index: 0,
            timing: fabro_types::StageTiming::wall_only(5),
            status: "succeeded".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 1,
            max_attempts: 1,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          3,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageFailed {
            node_id:    "work".to_string(),
            name:       "Work".to_string(),
            index:      1,
            failure:    FailureDetail::new("try again", FailureCategory::TransientInfra),
            will_retry: true,
            timing:     fabro_types::StageTiming::wall_only(10),
            billing:    None,
            actor:      None,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageRetrying {
            node_id:      "work".to_string(),
            name:         "Work".to_string(),
            index:        1,
            attempt:      2,
            max_attempts: 3,
            delay_ms:     100,
        },
    )
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(stage_status(&body, "setup@1"), "succeeded");
    assert_eq!(stage_status(&body, "work@1"), "retrying");

    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageCompleted {
            node_id: "work".to_string(),
            name: "Work".to_string(),
            index: 1,
            timing: fabro_types::StageTiming::wall_only(25),
            status: "partially_succeeded".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 2,
            max_attempts: 3,
        },
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(stage_status(&body, "work@1"), "partially_succeeded");
}

#[tokio::test]
async fn list_run_stages_projects_running_stage_as_cancelled_after_cancelled_run_failure() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 1,
            handler_type:          "agent".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::WorkflowRunFailed {
            failure:              fabro_types::RunFailure {
                reason: fabro_types::FailureReason::Cancelled,
                detail: FailureDetail::new("cancelled", FailureCategory::Canceled),
            },
            timing:               fabro_types::RunTiming::wall_only(100),
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    )
    .await
    .unwrap();

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(stage_status(&body, "work@1"), "cancelled");
}

fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value {
    body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|stage| stage["id"] == id)
        .unwrap_or_else(|| panic!("stage {id} not found in {body:#?}"))
}

#[tokio::test]
async fn list_run_stages_includes_stage_model_usage() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "prompt",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "prompt".to_string(),
            name:                  "Prompt".to_string(),
            index:                 0,
            handler_type:          "prompt".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "prompt",
        1,
        &workflow_event::Event::Prompt {
            stage:            "prompt".to_string(),
            visit:            1,
            text:             "Summarize".to_string(),
            mode:             Some(StageModelUsage::MODE_PROMPT.to_string()),
            provider:         Some("openai".to_string()),
            model:            Some("gpt-5.5".to_string()),
            reasoning_effort: Some(ReasoningEffort::High),
            speed:            Some(Speed::Fast),
        },
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        stage_entry(&body, "prompt@1")["provider_used"],
        json!({
            "mode": "prompt",
            "provider": "openai",
            "model": "gpt-5.5",
            "reasoning_effort": "high",
            "speed": "fast"
        })
    );
}

fn test_billed_usage(
    model_id: &str,
    input_tokens: i64,
    output_tokens: i64,
) -> fabro_model::BilledModelUsage {
    serde_json::from_value(json!({
        "input": {
            "usage": {
                "model": {
                    "provider": "openai",
                    "model_id": model_id
                },
                "tokens": {
                    "input_tokens": input_tokens,
                    "output_tokens": output_tokens
                }
            },
            "facts": { "algorithm": "openai" }
        },
        "total_usd_micros": input_tokens + output_tokens
    }))
    .unwrap()
}

async fn create_billed_retry_run(state: &Arc<AppState>, run_id: RunId) {
    create_durable_run_with_events(state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    append_scoped_stage_event(
        state,
        run_id,
        "verify",
        1,
        &workflow_event::Event::StageFailed {
            node_id:    "verify".to_string(),
            name:       "Verify".to_string(),
            index:      1,
            failure:    FailureDetail::new("try again", FailureCategory::TransientInfra),
            will_retry: true,
            timing:     fabro_types::StageTiming::wall_only(1200),
            billing:    Some(test_billed_usage("gpt-old", 100, 10)),
            actor:      None,
        },
    )
    .await;
    append_scoped_stage_event(
        state,
        run_id,
        "verify",
        2,
        &workflow_event::Event::StageCompleted {
            node_id: "verify".to_string(),
            name: "Verify".to_string(),
            index: 1,
            timing: fabro_types::StageTiming::wall_only(800),
            status: "succeeded".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: Some(test_billed_usage("gpt-new", 200, 20)),
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 2,
            max_attempts: 2,
        },
    )
    .await;
}

#[tokio::test]
async fn list_run_stages_distinguishes_visits() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let mut graph = Graph::new("test");
    let mut verify = Node::new("verify");
    verify
        .attrs
        .insert("type".to_string(), AttrValue::String("command".to_string()));
    graph.nodes.insert("verify".to_string(), verify);

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunCreated {
            run_id,
            title: None,
            settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(),
            graph: serde_json::to_value(&graph).unwrap(),
            workflow_source: None,
            labels: std::collections::BTreeMap::default(),
            source_directory: None,
            workflow_slug: Some("test".to_string()),
            workflow_version_id: None,
            automation: None,
            provenance: test_support::test_run_provenance(),
            manifest_blob: None,
            spec_blob: None,
            git: None,
            fork_source_ref: None,
            retried_from: None,
            parent_id: None,
            web_url: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    // First visit of `verify` — failed.
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "verify".to_string(),
            name:                  "Verify".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        1,
        &workflow_event::Event::StageCompleted {
            node_id: "verify".to_string(),
            name: "Verify".to_string(),
            index: 1,
            timing: fabro_types::StageTiming::wall_only(1500),
            status: "failed".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 1,
            max_attempts: 1,
        },
    )
    .await;

    // Second visit of `verify` — running.
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        2,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "verify".to_string(),
            name:                  "Verify".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let data = body["data"].as_array().unwrap();
    let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect();
    assert_eq!(verify_entries.len(), 2, "expected two verify visits");

    let first = stage_entry(&body, "verify@1");
    assert_eq!(first["node_id"], "verify");
    assert_eq!(first["visit"], 1);
    assert_eq!(first["handler"], "command");
    assert_eq!(first["status"], "failed");
    assert_eq!(first["wall_time_ms"], 1500);

    let second = stage_entry(&body, "verify@2");
    assert_eq!(second["node_id"], "verify");
    assert_eq!(second["visit"], 2);
    assert_eq!(second["handler"], "command");
    assert_eq!(second["status"], "running");

    // Old `dot_id` field must be gone.
    assert!(first.get("dot_id").is_none(), "dot_id should be removed");
}

#[tokio::test]
async fn list_run_stages_exposes_execution_identity_for_resumed_stage() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let mut graph = Graph::new("test");
    let mut work = Node::new("work");
    work.attrs
        .insert("type".to_string(), AttrValue::String("agent".to_string()));
    graph.nodes.insert("work".to_string(), work);

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunCreated {
            run_id,
            title: None,
            settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(),
            graph: serde_json::to_value(&graph).unwrap(),
            workflow_source: None,
            labels: std::collections::BTreeMap::default(),
            source_directory: None,
            workflow_slug: Some("test".to_string()),
            workflow_version_id: None,
            automation: None,
            provenance: test_support::test_run_provenance(),
            manifest_blob: None,
            spec_blob: None,
            git: None,
            fork_source_ref: None,
            retried_from: None,
            parent_id: None,
            web_url: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    // Legacy-shaped first execution without identity metadata.
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 1,
            handler_type:          "agent".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    // Reexecution after cancel/resume: same graph visit, next ordinal.
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        2,
        &workflow_event::Event::StageStarted {
            graph_visit:           Some(1),
            resumed_from_stage_id: Some(fabro_types::StageId::new("work", 1)),
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 1,
            handler_type:          "agent".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let first = stage_entry(&body, "work@1");
    assert!(
        first.get("graph_visit").is_none(),
        "legacy stage should omit graph_visit"
    );
    assert!(
        first.get("resumed_from_stage_id").is_none(),
        "legacy stage should omit resumed_from_stage_id"
    );

    let second = stage_entry(&body, "work@2");
    assert_eq!(second["visit"], 2);
    assert_eq!(second["graph_visit"], 1);
    assert_eq!(second["resumed_from_stage_id"], "work@1");
}

#[tokio::test]
async fn list_run_stages_exposes_parallel_branch_identity() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "ordinary",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           Some(1),
            resumed_from_stage_id: None,
            node_id:               "ordinary".to_string(),
            name:                  "Ordinary".to_string(),
            index:                 0,
            handler_type:          "agent".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;

    let parallel_group_id = StageId::new("review_fork", 2);
    let parallel_branch_id = ParallelBranchId::new(parallel_group_id.clone(), 4);
    let branch_event = workflow_event::Event::ParallelBranchStarted {
        parallel_group_id:     parallel_group_id.clone(),
        parallel_branch_id:    parallel_branch_id.clone(),
        branch:                "review_glm".to_string(),
        index:                 4,
        item_label:            None,
        graph_visit:           Some(3),
        resumed_from_stage_id: None,
    };
    let branch_scope = workflow_event::StageScope::for_parallel_branch(
        "review_glm",
        3,
        parallel_group_id,
        parallel_branch_id,
    );
    append_event_with_scope(&state, run_id, &branch_event, &branch_scope).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let branch = stage_entry(&body, "review_glm@3");
    assert_eq!(branch["parallel_group_id"], "review_fork@2");
    assert_eq!(branch["parallel_branch_index"], 4);

    let ordinary = stage_entry(&body, "ordinary@1");
    assert!(ordinary.get("parallel_group_id").is_none());
    assert!(ordinary.get("parallel_branch_index").is_none());
}

#[tokio::test]
async fn run_billing_includes_live_stage_timing_in_rows_and_totals() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_run_started_event(run_id),
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &stage_started_event("work", "command"),
    )
    .await;

    tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let stages = body["stages"].as_array().unwrap();

    assert_eq!(stages.len(), 1);
    let row_timing = &stages[0]["timing"];
    assert!(row_timing["active_time_ms"].as_u64().unwrap() > 0);
    assert_eq!(row_timing["tool_time_ms"], row_timing["active_time_ms"]);
    assert_eq!(&body["totals"]["timing"], row_timing);
}

/// `checkpoint.completed_nodes` records every visit, so a looped node appears
/// once per re-entry. Billing must dedup so a retried node renders as one row
/// and `runtime_secs` is summed across all visits exactly once.
#[tokio::test]
async fn run_billing_dedups_retried_nodes_and_sums_their_durations() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    // Visit 1 of `verify` — completed in 1.5s.
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        1,
        &workflow_event::Event::StageCompleted {
            node_id: "verify".to_string(),
            name: "Verify".to_string(),
            index: 1,
            timing: fabro_types::StageTiming::wall_only(1500),
            status: "failed".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 1,
            max_attempts: 1,
        },
    )
    .await;

    // Visit 2 of `verify` — completed in 0.8s.
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        2,
        &workflow_event::Event::StageCompleted {
            node_id: "verify".to_string(),
            name: "Verify".to_string(),
            index: 1,
            timing: fabro_types::StageTiming::wall_only(800),
            status: "succeeded".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 1,
            max_attempts: 1,
        },
    )
    .await;

    // Checkpoint records `verify` twice (once per visit) — this is what makes
    // the dedup necessary.
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::CheckpointCompleted {
            graph_visit: None,
            resumed_from_stage_id: None,
            node_id: "verify".to_string(),
            status: "running".to_string(),
            current_node: "verify".to_string(),
            completed_nodes: vec!["verify".to_string(), "verify".to_string()],
            node_retries: std::collections::BTreeMap::new(),
            context_values: std::collections::BTreeMap::new(),
            node_outcomes: std::collections::BTreeMap::from([(
                "verify".to_string(),
                Outcome::default(),
            )]),
            next_node_id: Some("done".to_string()),
            git_commit_sha: None,
            loop_failure_signatures: std::collections::BTreeMap::new(),
            restart_failure_signatures: std::collections::BTreeMap::new(),
            node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]),
            diff: None,
            diff_summary: None,
        },
    )
    .await
    .unwrap();

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let stages = body["stages"].as_array().unwrap();
    assert_eq!(
        stages.len(),
        1,
        "expected one row for the retried verify node"
    );
    assert_eq!(stages[0]["stage"]["id"], "verify");
    // Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s).
    assert!(
        stages[0]["timing"]["wall_time_ms"].as_u64().unwrap() == 2300,
        "row runtime_secs should sum visits, got {}",
        stages[0]["timing"]["wall_time_ms"]
    );

    // Totals must not double-count: a single 2.3s, not 4.6s.
    assert!(
        body["totals"]["timing"]["wall_time_ms"].as_u64().unwrap() == 2300,
        "totals.runtime_secs should sum visits exactly once, got {}",
        body["totals"]["timing"]["wall_time_ms"]
    );
}

#[tokio::test]
async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_billed_retry_run(&state, run_id).await;
    let success_usage = test_billed_usage("gpt-new", 200, 20);
    let mut latest_outcome: Outcome<Option<fabro_model::BilledModelUsage>> = Outcome::success();
    latest_outcome.usage = Some(success_usage);
    latest_outcome.timing = Some(fabro_types::StageTiming::wall_only(800));
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::CheckpointCompleted {
            graph_visit: None,
            resumed_from_stage_id: None,
            node_id: "verify".to_string(),
            status: "running".to_string(),
            current_node: "verify".to_string(),
            completed_nodes: vec!["verify".to_string(), "verify".to_string()],
            node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]),
            context_values: std::collections::BTreeMap::new(),
            node_outcomes: std::collections::BTreeMap::from([(
                "verify".to_string(),
                latest_outcome,
            )]),
            next_node_id: None,
            git_commit_sha: None,
            loop_failure_signatures: std::collections::BTreeMap::new(),
            restart_failure_signatures: std::collections::BTreeMap::new(),
            node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]),
            diff: None,
            diff_summary: None,
        },
    )
    .await
    .unwrap();

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let stages = body["stages"].as_array().unwrap();
    assert_eq!(stages.len(), 1);
    assert_eq!(stages[0]["stage"]["id"], "verify");
    assert_eq!(stages[0]["model"]["provider"], "openai");
    assert_eq!(stages[0]["model"]["model_id"], "gpt-new");
    assert_eq!(stages[0]["billing"]["input_tokens"], 300);
    assert_eq!(stages[0]["billing"]["output_tokens"], 30);
    assert_eq!(stages[0]["billing"]["total_usd_micros"], 330);
    assert!(stages[0]["timing"]["wall_time_ms"].as_u64().unwrap() == 2000);

    assert_eq!(body["totals"]["input_tokens"], 300);
    assert_eq!(body["totals"]["output_tokens"], 30);
    assert_eq!(body["totals"]["total_usd_micros"], 330);
    assert!(body["totals"]["timing"]["wall_time_ms"].as_u64().unwrap() == 2000);

    let by_model = body["by_model"].as_array().unwrap();
    assert_eq!(by_model.len(), 2);
    let old_model = by_model
        .iter()
        .find(|entry| entry["model"]["model_id"] == "gpt-old")
        .unwrap();
    let new_model = by_model
        .iter()
        .find(|entry| entry["model"]["model_id"] == "gpt-new")
        .unwrap();
    assert_eq!(old_model["model"]["provider"], "openai");
    assert_eq!(new_model["model"]["provider"], "openai");
    assert_eq!(old_model["stages"], 1);
    assert_eq!(old_model["billing"]["input_tokens"], 100);
    assert_eq!(new_model["stages"], 1);
    assert_eq!(new_model["billing"]["input_tokens"], 200);
}

/// The stage popover reads `billing` off the stages list, so it must be scoped
/// to one visit — unlike the Billing tab's rows, which sum every visit of a
/// node.
#[tokio::test]
async fn list_run_stages_reports_billing_per_visit() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_billed_retry_run(&state, run_id).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let first = stage_entry(&body, "verify@1");
    assert_eq!(first["billing"]["input_tokens"], 100);
    assert_eq!(first["billing"]["output_tokens"], 10);
    assert_eq!(first["billing"]["total_usd_micros"], 110);

    let second = stage_entry(&body, "verify@2");
    assert_eq!(second["billing"]["input_tokens"], 200);
    assert_eq!(second["billing"]["output_tokens"], 20);
    assert_eq!(second["billing"]["total_usd_micros"], 220);
}

#[tokio::test]
async fn list_run_stages_reports_zero_billing_for_a_stage_that_called_no_model() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    let started = stage_started_event("script", "command");
    append_scoped_stage_event(&state, run_id, "script", 1, &started).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    let billing = &stage_entry(&body, "script@1")["billing"];
    assert_eq!(billing["input_tokens"], 0);
    assert_eq!(billing["output_tokens"], 0);
    // No model ran, so there is nothing to price — not a $0.00 cost.
    assert!(billing.get("total_usd_micros").is_none());
}

#[tokio::test]
async fn list_run_stages_shows_retrying_after_failed_event() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 0,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          3,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageFailed {
            node_id:    "work".to_string(),
            name:       "Work".to_string(),
            index:      0,
            failure:    FailureDetail::new("flake", FailureCategory::TransientInfra),
            will_retry: true,
            timing:     fabro_types::StageTiming::wall_only(5),
            billing:    None,
            actor:      None,
        },
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageRetrying {
            node_id:      "work".to_string(),
            name:         "Work".to_string(),
            index:        0,
            attempt:      2,
            max_attempts: 3,
            delay_ms:     50,
        },
    )
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(stage_status(&body, "work@1"), "retrying");
}

#[tokio::test]
async fn list_run_stages_shows_retrying_when_failed_will_retry() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 0,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          3,
        },
    )
    .await;
    // Only StageFailed, no StageRetrying yet — should still render retrying
    // because props.will_retry is true.
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &workflow_event::Event::StageFailed {
            node_id:    "work".to_string(),
            name:       "Work".to_string(),
            index:      0,
            failure:    FailureDetail::new("flake", FailureCategory::TransientInfra),
            will_retry: true,
            timing:     fabro_types::StageTiming::wall_only(5),
            billing:    None,
            actor:      None,
        },
    )
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(stage_status(&body, "work@1"), "retrying");
}

#[tokio::test]
async fn run_billing_retried_node_then_succeeded_emits_one_row_with_final_attempt_duration() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 0,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          3,
        },
        workflow_event::Event::StageFailed {
            node_id:    "work".to_string(),
            name:       "Work".to_string(),
            index:      0,
            failure:    FailureDetail::new("transient", FailureCategory::TransientInfra),
            will_retry: true,
            timing:     fabro_types::StageTiming::wall_only(10),
            billing:    None,
            actor:      None,
        },
        workflow_event::Event::StageRetrying {
            node_id:      "work".to_string(),
            name:         "Work".to_string(),
            index:        0,
            attempt:      2,
            max_attempts: 3,
            delay_ms:     0,
        },
        workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "work".to_string(),
            name:                  "Work".to_string(),
            index:                 0,
            handler_type:          "command".to_string(),
            attempt:               2,
            max_attempts:          3,
        },
        workflow_event::Event::StageCompleted {
            node_id: "work".to_string(),
            name: "Work".to_string(),
            index: 0,
            timing: fabro_types::StageTiming::wall_only(25),
            status: "succeeded".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 2,
            max_attempts: 3,
        },
    ])
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let stages = body["stages"].as_array().unwrap();
    assert_eq!(stages.len(), 1, "retry collapses to one row per node_id");
    let row = &stages[0];
    assert_eq!(row["stage"]["id"], "work");
    assert_eq!(
        row["state"], "succeeded",
        "final state mirrors the latest StageCompleted"
    );
    let runtime = row["timing"]["wall_time_ms"].as_u64().unwrap();
    assert_eq!(
        runtime, 25,
        "runtime should equal final attempt's 25ms, got {runtime}"
    );
}

fn revisit_test_started(node_id: &str) -> workflow_event::Event {
    workflow_event::Event::StageStarted {
        graph_visit:           None,
        resumed_from_stage_id: None,
        node_id:               node_id.to_string(),
        name:                  node_id.to_string(),
        index:                 0,
        handler_type:          "command".to_string(),
        attempt:               1,
        max_attempts:          1,
    }
}

fn revisit_test_completed_with_visit(
    node_id: &str,
    duration_ms: u64,
    visit: usize,
) -> workflow_event::Event {
    let mut node_visits = std::collections::BTreeMap::new();
    node_visits.insert(node_id.to_string(), visit);
    workflow_event::Event::StageCompleted {
        node_id: node_id.to_string(),
        name: node_id.to_string(),
        index: 0,
        timing: fabro_types::StageTiming::wall_only(duration_ms),
        status: "succeeded".to_string(),
        preferred_label: None,
        suggested_next_ids: Vec::new(),
        billing: None,
        failure: None,
        notes: None,
        files_touched: Vec::new(),
        context_updates: None,
        jump_to_node: None,
        context_values: None,
        node_visits: Some(node_visits),
        loop_failure_signatures: None,
        restart_failure_signatures: None,
        response: None,
        attempt: 1,
        max_attempts: 1,
    }
}

#[tokio::test]
async fn run_billing_revisited_node_collapses_to_two_rows_with_summed_visit_duration() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        // A → B → A loop. Per-visit `node_visits` payload steers the reducer
        // to attribute each StageCompleted to the right visit.
        revisit_test_started("a"),
        revisit_test_completed_with_visit("a", 1, 1),
        revisit_test_started("b"),
        revisit_test_completed_with_visit("b", 2, 1),
        revisit_test_started("a"),
        revisit_test_completed_with_visit("a", 99, 2),
    ])
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let stages = body["stages"].as_array().unwrap();
    assert_eq!(stages.len(), 2, "two distinct node_ids → two rows");
    assert_eq!(
        stages[0]["stage"]["id"], "a",
        "A appeared first → A's row first"
    );
    assert_eq!(stages[1]["stage"]["id"], "b");
    let a_runtime = stages[0]["timing"]["wall_time_ms"].as_u64().unwrap();
    assert_eq!(
        a_runtime, 100,
        "A should sum both visit durations (1ms + 99ms), got {a_runtime}"
    );
    let b_runtime = stages[1]["timing"]["wall_time_ms"].as_u64().unwrap();
    assert_eq!(
        b_runtime, 2,
        "B should carry its single visit's duration (2ms), got {b_runtime}"
    );
}

async fn append_raw_run_event(
    state: &Arc<AppState>,
    run_id: RunId,
    seq_hint: &str,
    ts: &str,
    event: &str,
    properties: serde_json::Value,
    node_id: Option<&str>,
) {
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    let payload = fabro_store::EventPayload::new(
        json!({
            "id": format!("evt-{seq_hint}"),
            "ts": ts,
            "run_id": run_id,
            "event": event,
            "node_id": node_id,
            "properties": properties,
        }),
        &run_id,
    )
    .unwrap();
    run_store.append_event(&payload).await.unwrap();
}

async fn create_unreadable_durable_run(state: &Arc<AppState>, run_id: RunId) {
    let run_store = state.stores.runs.create_run(&run_id).await.unwrap();
    append_default_run_created(&run_store, run_id).await;
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    })
    .await
    .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
        .await
        .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
        .await
        .unwrap();
    let seq = run_store.last_event_seq().await.unwrap().unwrap() + 1;
    let completed = workflow_event::to_run_event_at(
        &run_id,
        &workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1),
            artifact_count:       0,
            status:               "legacy-status".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
        "2026-05-05T20:46:33Z".parse().unwrap(),
        None,
    );
    let payload = workflow_event::build_redacted_event_payload(&completed, &run_id).unwrap();
    fabro_store::test_support::put_unvalidated_run_event(
        &state.stores.runs,
        &run_id,
        seq,
        payload.as_value(),
    )
    .await
    .unwrap();
    let err = run_store
        .state()
        .await
        .expect_err("poison event should make the run projection unreadable");
    assert!(
        err.to_string().contains("invalid completed stage status"),
        "unexpected projection error: {err}"
    );
}

fn github_token_settings() -> ServerSettings {
    ServerSettingsBuilder::from_toml(
        r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[server.integrations.github]
strategy = "token"
"#,
    )
    .expect("github token settings fixture should resolve")
}

fn create_github_token_app_state(
    token: Option<&str>,
    github_api_base_url: Option<String>,
) -> Arc<AppState> {
    create_github_token_app_state_with_env_lookup(token, github_api_base_url, |_| None)
}

fn create_github_token_app_state_with_env_lookup(
    token: Option<&str>,
    github_api_base_url: Option<String>,
    env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
) -> Arc<AppState> {
    create_github_token_app_state_with_env_lookup_and_llm_catalog_settings(
        token,
        github_api_base_url,
        env_lookup,
        LlmCatalogSettings::default(),
    )
}

fn create_github_token_app_state_with_env_lookup_and_llm_catalog_settings(
    token: Option<&str>,
    github_api_base_url: Option<String>,
    env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
    llm_catalog_settings: LlmCatalogSettings,
) -> Arc<AppState> {
    let (store, artifact_store) = test_store_bundle();
    let vault_path = test_secret_store_path();
    let server_env_path = vault_path.with_file_name("server.env");
    let active_config_path = vault_path.with_file_name("settings.toml");
    if let Some(token) = token {
        Vault::load(vault_path.clone())
            .expect("test vault should load")
            .set("GITHUB_TOKEN", token, SecretType::Token, None)
            .expect("test github token should be writable");
    }
    Vault::load(vault_path.clone())
        .expect("test vault should load")
        .set(
            EnvVars::OPENAI_API_KEY,
            "test-openai-api-key",
            SecretType::Token,
            None,
        )
        .expect("test OpenAI credential should be writable");
    let db_pool = test_db_pool_for_vault_path(&vault_path).expect("test db pool should build");
    let preloaded_vault = crate::test_support::test_secret_snapshot(db_pool.clone())
        .expect("test secret snapshot should build");
    let config = AppStateConfig {
        resolved_settings: resolved_runtime_settings_for_tests(
            github_token_settings(),
            RunLayer::default(),
            llm_catalog_settings,
        ),
        registry_factory_override: None,
        max_concurrent_runs: 5,
        store,
        artifact_store,
        db_pool,
        preloaded_vault,
        server_secrets: load_test_server_secrets(server_env_path, HashMap::new()),
        env_lookup: Arc::new(env_lookup),
        github_api_base_url,
        active_config_path,
        http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
        sandbox_provider_registry: None,
        shutdown: tokio_util::sync::CancellationToken::new(),
        worker_control_bus: None,
        worker_runtime: None,
        automation_materializer_override: None,
    };
    build_app_state(config).expect("test app state should build")
}

#[tokio::test]
async fn github_token_strategy_ignores_process_env_token() {
    let state = create_github_token_app_state_with_env_lookup(None, None, |name| match name {
        EnvVars::GITHUB_TOKEN => Some("ghu_from_env".to_string()),
        _ => None,
    });
    let settings = state.server_settings();

    let err = state
        .github_credentials(&settings.server.integrations.github)
        .await
        .expect_err("server runtime should ignore env-backed GitHub tokens");

    assert_eq!(
        err.to_string(),
        "GITHUB_TOKEN not configured -- run fabro install or run fabro secret set GITHUB_TOKEN"
    );
}

#[tokio::test]
async fn github_token_strategy_ignores_gh_token_alias() {
    let state = create_github_token_app_state_with_env_lookup(None, None, |name| match name {
        EnvVars::GH_TOKEN => Some("ghu_from_env_alias".to_string()),
        _ => None,
    });
    state
        .stores
        .vault
        .set(
            EnvVars::GH_TOKEN,
            "ghu_from_vault_alias",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let settings = state.server_settings();

    let err = state
        .github_credentials(&settings.server.integrations.github)
        .await
        .expect_err("server runtime should ignore GH_TOKEN in env and vault");

    assert_eq!(
        err.to_string(),
        "GITHUB_TOKEN not configured -- run fabro install or run fabro secret set GITHUB_TOKEN"
    );
}

#[tokio::test]
async fn github_token_strategy_reads_github_token_from_vault() {
    let state = create_github_token_app_state(Some("ghu_test"), None);
    let settings = state.server_settings();

    let credentials = state
        .github_credentials(&settings.server.integrations.github)
        .await
        .expect("vault GitHub token should resolve")
        .expect("vault GitHub token should produce credentials");

    assert!(
        matches!(credentials, fabro_github::GitHubCredentials::Pat(token) if token == "ghu_test")
    );
}

/// Build the (state, router, run_id) triple every PR-endpoint test
/// needs. Use this instead of repeating the
/// state/build_router/fixtures::RUN_1 incantation per test.
fn pr_test_app(
    token: Option<&str>,
    github_api_base_url: Option<String>,
) -> (Arc<AppState>, Router, RunId) {
    let state = create_github_token_app_state(token, github_api_base_url);
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    (state, app, fixtures::RUN_1)
}

/// Same as [`pr_test_app`] but creates a fresh minimal run via the
/// HTTP create-run endpoint instead of using fixtures::RUN_1. For
/// tests that exercise endpoints expecting a real on-disk run rather
/// than a synthetic fixture id.
async fn pr_test_app_with_minimal_run(
    token: Option<&str>,
    github_api_base_url: Option<String>,
) -> (Arc<AppState>, Router, String) {
    let state = create_github_token_app_state(token, github_api_base_url);
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT).await;
    (state, app, run_id)
}

/// Same as [`pr_test_app`] but the run is set up as a completed
/// workflow ready for `POST /runs/{id}/pull_request`. The branches
/// and diff are fixed defaults; only the origin URL varies per
/// test (None to test missing-origin rejection, gitlab.com to test
/// non-github rejection, etc.).
async fn pr_test_app_with_completed_run(
    token: Option<&str>,
    github_api_base_url: Option<String>,
    repo_origin_url: Option<&str>,
) -> (Arc<AppState>, Router, RunId) {
    let (state, app, run_id) = pr_test_app(token, github_api_base_url);
    Box::pin(create_completed_run_ready_for_pull_request(
        &state,
        run_id,
        repo_origin_url,
        Some("main"),
        Some("fabro/run/42"),
        "diff --git a/src/lib.rs b/src/lib.rs\n+fn shipped() {}\n",
    ))
    .await;
    (state, app, run_id)
}

async fn create_run_with_pull_request_record(
    state: &Arc<AppState>,
    run_id: RunId,
    pr_url: &str,
    pr_number: u64,
    title: &str,
) {
    create_durable_run_with_events(state, run_id, &[
        workflow_event::Event::PullRequestCreated {
            pr_url: pr_url.to_string(),
            pr_number,
            owner: "acme".to_string(),
            repo: "widgets".to_string(),
            base_branch: "main".to_string(),
            head_branch: "feature".to_string(),
            head_sha: Some("final-sha".to_string()),
            title: title.to_string(),
            draft: false,
        },
    ])
    .await;
}

async fn create_run_with_linked_pull_request_record(
    state: &Arc<AppState>,
    run_id: RunId,
    pull_request: PullRequestLink,
) {
    create_durable_run_with_events(state, run_id, &[workflow_event::Event::PullRequestLinked {
        pull_request,
    }])
    .await;
}

async fn create_completed_run_ready_for_pull_request(
    state: &Arc<AppState>,
    run_id: RunId,
    repo_origin_url: Option<&str>,
    base_branch: Option<&str>,
    run_branch: Option<&str>,
    final_patch: &str,
) {
    let mut graph = Graph::new("test");
    graph.attrs.insert(
        "goal".to_string(),
        AttrValue::String("Ship the server-side PR".to_string()),
    );
    let git = match (repo_origin_url, base_branch) {
        (Some(origin), Some(branch)) => Some(fabro_types::GitContext {
            origin_url: origin.to_string(),
            branch:     branch.to_string(),
            sha:        None,
            dirty:      fabro_types::DirtyStatus::Clean,
        }),
        _ => None,
    };
    let run_spec = RunSpec {
        run_id,
        settings: fabro_types::WorkflowSettings::default(),
        graph,
        graph_source: None,
        workflow_slug: Some("test".to_string()),
        workflow_version_id: None,
        automation: None,
        source_directory: Some("/tmp/project".to_string()),
        git: git.clone(),
        labels: HashMap::new(),
        provenance: test_support::test_run_provenance(),
        manifest_blob: None,
        definition_blob: None,
        spec_blob: None,
        fork_source_ref: None,
    };

    create_durable_run_with_events(state, run_id, &[
        workflow_event::Event::RunCreated {
            run_id,
            title: None,
            settings: serde_json::to_value(&run_spec.settings).unwrap(),
            graph: serde_json::to_value(&run_spec.graph).unwrap(),
            workflow_source: None,
            labels: run_spec.labels.clone().into_iter().collect(),
            source_directory: run_spec.source_directory.clone(),
            workflow_slug: run_spec.workflow_slug.clone(),
            workflow_version_id: run_spec.workflow_version_id,
            automation: None,
            provenance: run_spec.provenance.clone(),
            manifest_blob: None,
            spec_blob: None,
            git,
            fork_source_ref: None,
            retried_from: None,
            parent_id: None,
            web_url: None,
        },
        workflow_event::Event::WorkflowRunStarted {
            name: "test".to_string(),
            run_id,
            base_branch: base_branch.map(str::to_string),
            base_sha: None,
            run_branch: run_branch.map(str::to_string),
            worktree_dir: None,
            goal: Some("Ship the server-side PR".to_string()),
        },
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: Some("final-sha".to_string()),
            final_patch:          Some(final_patch.to_string()),
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;
}

fn test_event_envelope(seq: u32, run_id: RunId, body: EventBody) -> EventEnvelope {
    EventEnvelope {
        seq,
        event: RunEvent {
            id: format!("evt-{seq}"),
            ts: Utc::now(),
            run_id,
            node_id: None,
            node_label: None,
            stage_id: None,
            parallel_group_id: None,
            parallel_branch_id: None,
            session_id: None,
            parent_session_id: None,
            tool_call_id: None,
            actor: None,
            body,
        },
    }
}

#[tokio::test]
async fn test_model_unknown_returns_404() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/models/nonexistent-model-xyz/test"))
        .header("content-type", "application/json")
        .body(Body::empty())
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/models/sonnet/test?provider=anthropic"))
        .header("content-type", "application/json")
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["model_id"], "claude-sonnet-5");
    assert_eq!(body["provider"], "anthropic");
    assert_eq!(body["status"], "skip");
}

#[tokio::test]
async fn test_model_unqualified_known_alias_requires_a_ready_provider() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/models/sonnet/test"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn model_api_keeps_duplicate_ids_provider_scoped_and_selects_ready_priority() {
    let direct_upstream = MockServer::start();
    let aggregator_upstream = MockServer::start();
    let direct_probe = direct_upstream.mock(|when, then| {
        when.method(POST)
            .path("/chat/completions")
            .json_body_includes(r#"{"model":"portable-model"}"#);
        then.status(200)
            .header("content-type", "application/json")
            .json_body(json!({
                "id": "chatcmpl-direct",
                "model": "portable-model",
                "choices": [{
                    "message": {"role": "assistant", "content": "OK"},
                    "finish_reason": "stop"
                }],
                "usage": {
                    "prompt_tokens": 1,
                    "completion_tokens": 1,
                    "total_tokens": 2
                }
            }));
    });
    let aggregator_probe = aggregator_upstream.mock(|when, then| {
        when.method(POST)
            .path("/chat/completions")
            .json_body_includes(r#"{"model":"vendor/portable-model"}"#);
        then.status(200)
            .header("content-type", "application/json")
            .json_body(json!({
                "id": "chatcmpl-aggregator",
                "model": "vendor/portable-model",
                "choices": [{
                    "message": {"role": "assistant", "content": "OK"},
                    "finish_reason": "stop"
                }],
                "usage": {
                    "prompt_tokens": 1,
                    "completion_tokens": 1,
                    "total_tokens": 2
                }
            }));
    });
    let settings: LlmCatalogSettings = toml::from_str(&format!(
        r#"
[providers.direct]
display_name = "Direct"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
priority = 120

[providers.direct.auth]
credentials = ["vault:DIRECT_API_KEY"]

[providers.direct.models.portable-model]
display_name = "Portable (direct)"
family = "portable"
aliases = ["portable"]
default = true

[providers.direct.models.portable-model.limits]
context_window = 1000

[providers.direct.models.portable-model.features]
tools = false
vision = false
reasoning = false

[providers.aggregator]
display_name = "Aggregator"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
priority = 110

[providers.aggregator.auth]
credentials = ["vault:AGGREGATOR_API_KEY"]

[providers.aggregator.models.portable-model]
api_id = "vendor/portable-model"
display_name = "Portable (aggregator)"
family = "portable"
aliases = ["portable"]
default = true

[providers.aggregator.models.portable-model.limits]
context_window = 1000

[providers.aggregator.models.portable-model.features]
tools = false
vision = false
reasoning = false
"#,
        direct_upstream.base_url(),
        aggregator_upstream.base_url(),
    ))
    .unwrap();
    let state = TestAppStateBuilder::new()
        .llm_catalog_settings(settings)
        .vault_entries([
            ("DIRECT_API_KEY", "direct-test-key"),
            ("AGGREGATOR_API_KEY", "aggregator-test-key"),
        ])
        .build();
    let app = crate::test_support::build_test_router(state);

    let list = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/models?query=portable-model"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let list = response_json!(list, StatusCode::OK).await;
    let rows = list["data"].as_array().unwrap();
    assert_eq!(rows.len(), 2);
    assert_eq!(
        rows.iter()
            .map(|row| row["provider"].as_str().unwrap())
            .collect::<std::collections::BTreeSet<_>>(),
        std::collections::BTreeSet::from(["aggregator", "direct"])
    );
    assert!(
        rows.iter()
            .all(|row| row["id"] == "portable-model" && row["configured"] == true)
    );

    let filtered = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/models?provider=aggregator&query=portable-model"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let filtered = response_json!(filtered, StatusCode::OK).await;
    assert_eq!(filtered["data"].as_array().unwrap().len(), 1);
    assert_eq!(filtered["data"][0]["provider"], "aggregator");

    for (query, expected_provider) in [
        ("?provider=direct", "direct"),
        ("?provider=aggregator", "aggregator"),
        ("", "direct"),
    ] {
        let response = app
            .clone()
            .oneshot(
                Request::builder()
                    .method("POST")
                    .uri(api(&format!("/models/portable/test{query}")))
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap();
        let body = response_json!(response, StatusCode::OK).await;
        assert_eq!(body["model_id"], "portable-model");
        assert_eq!(body["provider"], expected_provider);
        assert_eq!(body["status"], "ok");
    }

    direct_probe.assert_calls(2);
    aggregator_probe.assert_calls(1);
}

#[tokio::test]
async fn test_model_invalid_mode_returns_400() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/models/claude-opus-4-6/test?mode=bogus"))
        .header("content-type", "application/json")
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn test_provider_credentials_uses_app_state_catalog() {
    let upstream = MockServer::start();
    let completion = upstream.mock(|when, then| {
        when.method(POST)
            .path("/chat/completions")
            .header("authorization", "Bearer sk-test");
        then.status(200)
            .header("content-type", "application/json")
            .json_body(json!({
                "id": "chatcmpl_test",
                "object": "chat.completion",
                "created": 1_700_000_000,
                "model": "test-model",
                "choices": [{
                    "index": 0,
                    "message": {"role": "assistant", "content": "OK"},
                    "finish_reason": "stop"
                }],
                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
            }));
    });
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(&format!(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
priority = 120

[providers.acme.auth]
credentials = ["vault:ACME_API_KEY"]

[models."acme-probe"]
provider = "acme"
api_id = "test-model"
display_name = "Acme Probe"
family = "acme"
default = true
probe = true

[models."acme-probe".limits]
context_window = 128000

[models."acme-probe".features]
tools = false
vision = false
reasoning = false
"#,
        upstream.base_url()
    ))
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/acme/credentials/test"))
        .header("content-type", "application/json")
        .body(Body::from(json!({ "api_key": "sk-test" }).to_string()))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["ok"], true);
    completion.assert();
}

#[tokio::test]
async fn list_models_filters_by_provider() {
    let app = test_app_with();

    let req = Request::builder()
        .method("GET")
        .uri(api("/models?provider=anthropic"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();
    assert!(!models.is_empty());
    assert!(
        models
            .iter()
            .all(|model| model["provider"] == serde_json::Value::String("anthropic".into()))
    );
}

#[tokio::test]
async fn list_models_exposes_reasoning_effort_controls() {
    let app = test_app_with();

    let req = Request::builder()
        .method("GET")
        .uri(api("/models?provider=moonshot"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();
    let kimi_k3 = models
        .iter()
        .find(|model| model["id"] == "kimi-k3")
        .expect("Kimi K3 should be listed");
    let kimi_k2_5 = models
        .iter()
        .find(|model| model["id"] == "kimi-k2.5")
        .expect("Kimi K2.5 should be listed");

    assert_eq!(
        kimi_k3["controls"]["reasoning_effort"],
        json!(["low", "high", "max"])
    );
    assert_eq!(kimi_k2_5["controls"]["reasoning_effort"], json!([]));
}

#[tokio::test]
async fn list_models_marks_configured_true_when_provider_has_credential_material() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    state
        .stores
        .vault
        .set(
            EnvVars::ANTHROPIC_API_KEY,
            "test-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/models"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();

    assert!(models.iter().any(|model| model["provider"] != "anthropic"));
    assert!(models.iter().any(|model| model["provider"] == "anthropic"));
    assert!(
        models
            .iter()
            .filter(|model| model["provider"] == "anthropic")
            .all(|model| model["configured"].as_bool() == Some(true))
    );
    assert!(
        models
            .iter()
            .filter(|model| model["provider"] != "anthropic")
            .all(|model| model["configured"].as_bool() == Some(false))
    );
}

#[tokio::test]
async fn list_models_marks_configured_false_when_provider_cannot_register() {
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
agent_profile = "openai"
priority = 120

[providers.acme.auth]
credentials = ["env:ACME_API_KEY"]

[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true

[models."acme-large".limits]
context_window = 128000

[models."acme-large".features]
tools = true
vision = false
reasoning = false
"#,
    )
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .env_lookup(|name| (name == "ACME_API_KEY").then(|| "acme-key".to_string()))
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/models?provider=acme"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();

    assert_eq!(models.len(), 1);
    assert_eq!(models[0]["id"], "acme-large");
    assert_eq!(models[0]["configured"].as_bool(), Some(false));
}

#[tokio::test]
async fn list_models_marks_configured_false_when_no_credential_material() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/models"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();

    assert!(!models.is_empty());
    assert!(
        models
            .iter()
            .all(|model| model["configured"].as_bool() == Some(false))
    );
}

#[tokio::test]
async fn list_models_unknown_provider_returns_empty_page() {
    let app = test_app_with();

    let req = Request::builder()
        .method("GET")
        .uri(api("/models?provider=missing-provider"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["data"].as_array().unwrap().len(), 0);
    assert_eq!(body["meta"]["has_more"].as_bool(), Some(false));
}

#[tokio::test]
async fn list_models_uses_app_state_catalog_overrides() {
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "https://api.acme.test/v1"
priority = 120

[providers.acme.auth]
credentials = ["env:ACME_API_KEY"]

[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true

[models."acme-large".limits]
context_window = 128000

[models."acme-large".features]
tools = true
vision = false
reasoning = false
"#,
    )
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/models?provider=acme"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let models = body["data"].as_array().unwrap();
    assert_eq!(models.len(), 1);
    assert_eq!(models[0]["id"], "acme-large");
    assert_eq!(models[0]["provider"], "acme");
}

#[tokio::test]
async fn list_providers_marks_configured_per_provider_and_omits_secrets() {
    // Only `ANTHROPIC_API_KEY` is supplied in the vault, so anthropic resolves as
    // configured while every other catalog provider does not.
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    state
        .stores
        .vault
        .set(
            EnvVars::ANTHROPIC_API_KEY,
            "test-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/providers"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let providers = body["data"].as_array().unwrap();

    assert!(
        providers.len() >= 2,
        "builtin catalog should expose multiple providers"
    );

    let anthropic = providers
        .iter()
        .find(|provider| provider["id"] == "anthropic")
        .expect("anthropic provider should be present");
    assert_eq!(anthropic["configured"].as_bool(), Some(true));

    // `model_count` and `default_model` must reflect the catalog truth for
    // this exact provider, not merely be populated.
    let catalog = Catalog::builtin();
    let expected_model_count = catalog.list(Some(&ProviderId::anthropic())).len();
    assert_eq!(
        anthropic["model_count"].as_u64(),
        Some(expected_model_count as u64),
        "anthropic model_count should match the catalog"
    );
    let expected_default = catalog
        .default_for_provider(&ProviderId::anthropic())
        .expect("anthropic should have a catalog default model");
    assert_eq!(
        anthropic["default_model"].as_str(),
        Some(expected_default.id.as_str()),
        "anthropic default_model should match the catalog"
    );

    assert!(
        providers
            .iter()
            .filter(|provider| provider["id"] != "anthropic")
            .all(|provider| provider["configured"].as_bool() == Some(false)),
        "providers without supplied credentials should be unconfigured"
    );

    // Internal-only catalog fields and the injected credential value must
    // never reach the wire.
    let serialized = body["data"].to_string();
    assert!(!serialized.contains("\"auth\""), "leaked `auth`");
    assert!(
        !serialized.contains("\"extra_headers\""),
        "leaked `extra_headers`"
    );
    assert!(
        !serialized.contains("\"billing_policy\""),
        "leaked `billing_policy`"
    );
    assert!(
        !serialized.contains("\"agent_profile\""),
        "leaked `agent_profile`"
    );
    assert!(
        !serialized.contains("test-key"),
        "leaked the injected credential value"
    );
}

#[tokio::test]
async fn list_providers_marks_all_unconfigured_without_credentials() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("GET")
        .uri(api("/providers"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let providers = body["data"].as_array().unwrap();

    assert!(!providers.is_empty());
    assert!(
        providers
            .iter()
            .all(|provider| provider["configured"].as_bool() == Some(false)),
        "no provider should be configured when no credentials are supplied"
    );
}

#[tokio::test]
async fn test_providers_no_configured_providers_returns_error_summary() {
    let state = test_app_state_with_env_lookup(
        default_test_server_settings(),
        RunLayer::default(),
        5,
        |_| None,
    );
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["data"].as_array().unwrap().len(), 0);
    assert_eq!(body["summary"]["status"], "error");
    assert_eq!(body["summary"]["total"], 0);
    assert_eq!(body["summary"]["passed"], 0);
    assert_eq!(body["summary"]["failed"], 0);
}

#[tokio::test]
async fn test_providers_successful_probe_returns_probe_model() {
    let server = MockServer::start_async().await;
    let response_mock = server
        .mock_async(|when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", "Bearer vault-openai-key");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload("OK"));
        })
        .await;
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .provider_base_url("openai", server.url("/v1"))
        .build();
    state
        .stores
        .vault
        .set(
            EnvVars::OPENAI_API_KEY,
            "vault-openai-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let results = body["data"].as_array().unwrap();

    assert_eq!(results.len(), 1);
    assert_eq!(results[0]["provider"], "openai");
    assert_eq!(results[0]["model_id"], "gpt-5.4-mini");
    assert_eq!(results[0]["status"], "ok");
    assert!(results[0]["error_message"].is_null());
    assert_eq!(body["summary"]["status"], "ok");
    assert_eq!(body["summary"]["total"], 1);
    assert_eq!(body["summary"]["passed"], 1);
    assert_eq!(body["summary"]["failed"], 0);
    response_mock.assert_async().await;
}

#[tokio::test]
async fn test_providers_auth_issue_returns_error_without_upstream_call() {
    let server = MockServer::start_async().await;
    let upstream = server
        .mock_async(|when, then| {
            when.method(POST).path("/v1/responses");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload("unexpected"));
        })
        .await;
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .provider_base_url("openai", server.url("/v1"))
        .build();
    let mut credential = openai_oauth_credential();
    credential.tokens.expires_at = Utc::now() - ChronoDuration::hours(1);
    credential.tokens.refresh_token = None;
    state
        .stores
        .vault
        .set(
            "OPENAI_CODEX",
            &serde_json::to_string(&credential).unwrap(),
            SecretType::Oauth,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let results = body["data"].as_array().unwrap();

    assert_eq!(results.len(), 1);
    assert_eq!(results[0]["provider"], "openai");
    assert!(results[0]["model_id"].is_null());
    assert_eq!(results[0]["status"], "error");
    assert!(
        results[0]["error_message"]
            .as_str()
            .unwrap()
            .contains("requires re-authentication")
    );
    assert_eq!(body["summary"]["status"], "error");
    assert_eq!(body["summary"]["total"], 1);
    assert_eq!(body["summary"]["passed"], 0);
    assert_eq!(body["summary"]["failed"], 1);
    upstream.assert_calls_async(0).await;
}

#[tokio::test]
async fn test_providers_registration_issue_returns_error_without_probe() {
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"

[providers.acme.auth]
credentials = ["vault:ACME_API_KEY"]

[models."acme-probe"]
provider = "acme"
display_name = "Acme Probe"
family = "acme"
default = true
probe = true

[models."acme-probe".limits]
context_window = 128000

[models."acme-probe".features]
tools = true
vision = false
reasoning = false
"#,
    )
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    state
        .stores
        .vault
        .set("ACME_API_KEY", "acme-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let results = body["data"].as_array().unwrap();

    assert_eq!(results.len(), 1);
    assert_eq!(results[0]["provider"], "acme");
    assert!(results[0]["model_id"].is_null());
    assert_eq!(results[0]["status"], "error");
    assert!(
        results[0]["error_message"]
            .as_str()
            .unwrap()
            .contains("does not configure base_url")
    );
    assert_eq!(body["summary"]["status"], "error");
    assert_eq!(body["summary"]["total"], 1);
    assert_eq!(body["summary"]["passed"], 0);
    assert_eq!(body["summary"]["failed"], 1);
}

#[tokio::test]
async fn test_providers_mixed_results_preserve_catalog_order_and_counts() {
    let server = MockServer::start_async().await;
    let alpha_mock = server
        .mock_async(|when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", "Bearer alpha-key");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload("OK"));
        })
        .await;
    let zeta_mock = server
        .mock_async(|when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", "Bearer zeta-key");
            then.status(401)
                .header("content-type", "application/json")
                .json_body(json!({
                    "error": {
                        "message": "invalid api key",
                        "type": "invalid_request_error"
                    }
                }));
        })
        .await;
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(&format!(
        r#"
[providers.zeta]
display_name = "Zeta"
adapter = "openai"
base_url = "{base_url}"

[providers.zeta.auth]
credentials = ["vault:ZETA_API_KEY"]

[providers.alpha]
display_name = "Alpha"
adapter = "openai"
base_url = "{base_url}"

[providers.alpha.auth]
credentials = ["vault:ALPHA_API_KEY"]

[models."zeta-probe"]
provider = "zeta"
display_name = "Zeta Probe"
family = "zeta"
default = true
probe = true

[models."zeta-probe".limits]
context_window = 128000

[models."zeta-probe".features]
tools = true
vision = false
reasoning = false

[models."alpha-probe"]
provider = "alpha"
display_name = "Alpha Probe"
family = "alpha"
default = true
probe = true

[models."alpha-probe".limits]
context_window = 128000

[models."alpha-probe".features]
tools = true
vision = false
reasoning = false
"#,
        base_url = server.url("/v1")
    ))
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .llm_catalog_settings(llm_catalog_settings)
        .build();
    state
        .stores
        .vault
        .set("ALPHA_API_KEY", "alpha-key", SecretType::Token, None)
        .await
        .unwrap();
    state
        .stores
        .vault
        .set("ZETA_API_KEY", "zeta-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let results = body["data"].as_array().unwrap();

    assert_eq!(results.len(), 2);
    assert_eq!(results[0]["provider"], "alpha");
    assert_eq!(results[0]["model_id"], "alpha-probe");
    assert_eq!(results[0]["status"], "ok");
    assert_eq!(results[1]["provider"], "zeta");
    assert_eq!(results[1]["model_id"], "zeta-probe");
    assert_eq!(results[1]["status"], "error");
    assert_eq!(body["summary"]["status"], "error");
    assert_eq!(body["summary"]["total"], 2);
    assert_eq!(body["summary"]["passed"], 1);
    assert_eq!(body["summary"]["failed"], 1);
    alpha_mock.assert_async().await;
    zeta_mock.assert_async().await;
}

#[tokio::test]
async fn test_providers_response_does_not_leak_api_keys() {
    let leaked_key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789";
    let server = MockServer::start_async().await;
    let response_mock = server
        .mock_async(move |when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", format!("Bearer {leaked_key}"));
            then.status(401)
                .header("content-type", "application/json")
                .json_body(json!({
                    "error": {
                        "message": format!("invalid api key {leaked_key}"),
                        "type": "invalid_request_error"
                    }
                }));
        })
        .await;
    let state = TestAppStateBuilder::new()
        .runtime_settings(default_test_server_settings(), RunLayer::default())
        .max_concurrent_runs(5)
        .provider_base_url("openai", server.url("/v1"))
        .build();
    state
        .stores
        .vault
        .set(EnvVars::OPENAI_API_KEY, leaked_key, SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let serialized = body.to_string();

    assert!(
        !serialized.contains(leaked_key),
        "provider test response leaked API key: {serialized}"
    );
    assert!(
        serialized.contains("REDACTED"),
        "provider test response should include a redacted error: {serialized}"
    );
    response_mock.assert_async().await;
}

#[tokio::test]
async fn test_providers_requires_user_auth() {
    let app = build_router(test_app_state(), test_auth_mode());

    let req = Request::builder()
        .method("POST")
        .uri(api("/providers/test"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::UNAUTHORIZED).await;
}

#[tokio::test]
async fn auth_login_github_redirects_to_github() {
    let source = r#"
_version = 1

[server.auth]
methods = ["github"]

[server.web]
enabled = true
url = "http://localhost:3000"

[server.auth.github]
allowed_usernames = ["octocat"]

[server.integrations.github]
app_id = "123"
client_id = "Iv1.testclient"
slug = "fabro"
"#;
    let app = build_router(
        test_app_state_with_session_key(
            server_settings_from_toml(source),
            manifest_run_defaults_from_toml(source),
            Some("github-redirect-test-key-0123456789"),
        ),
        AuthMode::Enabled(ConfiguredAuth {
            methods:    vec![ServerAuthMethod::Github],
            dev_token:  None,
            jwt_key:    None,
            jwt_issuer: None,
        }),
    );

    let response = app
        .oneshot(
            Request::builder()
                .uri("/auth/login/github")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let response = checked_response!(response, StatusCode::SEE_OTHER).await;
    let location = response
        .headers()
        .get(axum::http::header::LOCATION)
        .and_then(|value| value.to_str().ok())
        .unwrap();
    assert!(location.starts_with("https://github.com/login/oauth/authorize?"));
}

#[tokio::test]
async fn logout_redirects_to_login_page() {
    let app = test_app_with();

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/auth/logout")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let response = checked_response!(response, StatusCode::SEE_OTHER).await;
    assert_eq!(
        response
            .headers()
            .get(axum::http::header::LOCATION)
            .and_then(|value| value.to_str().ok()),
        Some("/login")
    );
}

#[tokio::test]
async fn static_favicon_is_served() {
    let app = test_app_with();

    let response = app
        .oneshot(
            Request::builder()
                .uri("/images/favicon.svg")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let response = checked_response!(response, StatusCode::OK).await;
    assert_eq!(
        response
            .headers()
            .get(axum::http::header::CONTENT_TYPE)
            .and_then(|value| value.to_str().ok()),
        Some("image/svg+xml")
    );
}

#[tokio::test]
async fn post_runs_starts_run_and_returns_id() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    assert!(body["id"].is_string());
    assert!(!body["id"].as_str().unwrap().is_empty());
}

#[tokio::test]
async fn post_runs_invalid_dot_returns_bad_request() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body("not a graph"))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn get_run_status_returns_status() {
    let state = test_app_state();
    let app = test_app_with_scheduler(state);

    let run_id = create_and_start_run(&app, MINIMAL_DOT).await;

    // Give run a moment to start
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;

    // Check status
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_id(&body).unwrap(), run_id);
    assert_eq!(body["goal"].as_str().unwrap(), "Test");
    assert_eq!(body["title"].as_str().unwrap(), "Test");
    assert!(body["repository"].is_object());
    assert!(!body["repository"]["name"].as_str().unwrap().is_empty());
    assert!(body["timestamps"]["created_at"].is_string());
    assert!(body["labels"].is_object());
}

#[tokio::test]
async fn get_run_status_advances_live_active_timing_between_events() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_run_started_event(run_id),
    ])
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "work",
        1,
        &stage_started_event("work", "command"),
    )
    .await;

    // The SQLite summary stores timing at the StageStarted event. A later
    // detail read must overlay the in-flight command's active time from the
    // projection even though no newer event has arrived.
    tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let timing = &body["timing"];

    assert!(timing["active_time_ms"].as_u64().unwrap() > 0);
    assert_eq!(timing["tool_time_ms"], timing["active_time_ms"]);
    assert!(timing["wall_time_ms"].as_u64().unwrap() >= timing["active_time_ms"].as_u64().unwrap());
}

#[tokio::test]
async fn get_run_status_not_found() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{missing_run_id}")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn resolve_run_returns_unique_run_id_prefix_match() {
    let app = test_app_with();
    let run_id = create_run(&app, MINIMAL_DOT).await;
    let selector = &run_id[..8];

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/resolve?selector={selector}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_id(&body), Some(run_id.as_str()));
}

#[tokio::test]
async fn resolve_run_returns_bad_request_for_ambiguous_prefix() {
    let app = test_app_with();
    let run_id_a = create_run(&app, MINIMAL_DOT).await;
    let run_id_b = create_run(&app, MINIMAL_DOT).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs/resolve?selector=0"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::BAD_REQUEST).await;
    let detail = body["errors"][0]["detail"]
        .as_str()
        .expect("error detail should be present");
    assert!(
        detail.contains(&run_id_a),
        "detail should mention first run: {detail}"
    );
    assert!(
        detail.contains(&run_id_b),
        "detail should mention second run: {detail}"
    );
    assert!(
        detail.contains("created_at="),
        "detail should include creation timestamps: {detail}"
    );
    assert!(
        detail.contains("workflow="),
        "detail should include workflow names: {detail}"
    );
    assert!(
        detail.contains("origin="),
        "detail should include origin URLs: {detail}"
    );
}

#[tokio::test]
async fn resolve_run_prefers_most_recent_exact_workflow_slug_match() {
    let app = test_app_with();
    let older_id = create_run_for_target(
        &app,
        "ship-feature.fabro",
        &named_workflow_dot("ShipFeatureAlpha", "older"),
    )
    .await;
    let newer_id = create_run_for_target(
        &app,
        "ship-feature.fabro",
        &named_workflow_dot("ShipFeatureBeta", "newer"),
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs/resolve?selector=ship-feature"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_id(&body), Some(newer_id.as_str()));
    assert_ne!(run_json_id(&body), Some(older_id.as_str()));
}

#[tokio::test]
async fn resolve_run_prefers_most_recent_collapsed_workflow_name_match() {
    let app = test_app_with();
    let older_id = create_run_for_target_with_workflow_name(
        &app,
        "nightly-alpha.fabro",
        &named_workflow_dot("OlderNightlyGraph", "older"),
        "Nightly_Build",
    )
    .await;
    let newer_id = create_run_for_target_with_workflow_name(
        &app,
        "nightly-beta.fabro",
        &named_workflow_dot("NewerNightlyGraph", "newer"),
        "Nightly_Build",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs/resolve?selector=nightlybuild"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_id(&body), Some(newer_id.as_str()));
    assert_ne!(run_json_id(&body), Some(older_id.as_str()));
}

#[tokio::test]
async fn resolve_run_returns_not_found_for_unknown_selector() {
    let app = test_app_with();

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs/resolve?selector=missing-run"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_questions_returns_empty_list() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Start a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    // Get questions (should be empty for a run without wait.human nodes)
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/questions")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert!(body["data"].is_array());
    assert_eq!(body["meta"]["has_more"], false);
}

#[tokio::test]
async fn submit_answer_not_found_run() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{missing_run_id}/questions/q1/answer")))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({"kind": "yes"})).unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn submit_pending_interview_answer_rejects_invalid_answer_shape() {
    let state = test_app_state();
    let pending = LoadedPendingInterview {
        run_id:   fixtures::RUN_1,
        qid:      "q-1".to_string(),
        question: InterviewQuestionRecord {
            id:              "q-1".to_string(),
            text:            "Approve deploy?".to_string(),
            stage:           "gate".to_string(),
            question_type:   QuestionType::MultipleChoice,
            options:         vec![fabro_types::run_event::InterviewOption {
                key:         "approve".to_string(),
                label:       "Approve".to_string(),
                description: None,
                preview:     None,
            }],
            allow_freeform:  false,
            timeout_seconds: None,
            context_display: None,
            review_target:   None,
        },
    };

    let response = submit_pending_interview_answer(
        state.as_ref(),
        &pending,
        AnswerSubmission::system(
            Answer::text("not a valid multiple choice answer"),
            SystemActorKind::Engine,
        ),
    )
    .await
    .unwrap_err();

    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[test]
fn validate_answer_for_question_accepts_no_for_confirmation() {
    let question = InterviewQuestionRecord {
        id:              "q-1".to_string(),
        text:            "Continue?".to_string(),
        stage:           "gate".to_string(),
        question_type:   QuestionType::Confirmation,
        options:         vec![],
        allow_freeform:  false,
        timeout_seconds: None,
        context_display: None,
        review_target:   None,
    };

    let result = validate_answer_for_question(&question, &Answer::no());

    assert!(result.is_ok());
}

#[test]
fn answer_from_typed_yes_request_maps_to_yes_answer() {
    let question = InterviewQuestionRecord {
        id:              "q-1".to_string(),
        text:            "Continue?".to_string(),
        stage:           "gate".to_string(),
        question_type:   QuestionType::YesNo,
        options:         vec![],
        allow_freeform:  false,
        timeout_seconds: None,
        context_display: None,
        review_target:   None,
    };
    let req: SubmitAnswerRequest = serde_json::from_value(json!({ "kind": "yes" })).unwrap();

    let answer = answer_from_request(req, &question).unwrap();

    assert_eq!(answer.value, AnswerValue::Yes);
}

#[test]
fn answer_from_typed_no_request_maps_to_no_answer() {
    let question = InterviewQuestionRecord {
        id:              "q-1".to_string(),
        text:            "Continue?".to_string(),
        stage:           "gate".to_string(),
        question_type:   QuestionType::YesNo,
        options:         vec![],
        allow_freeform:  false,
        timeout_seconds: None,
        context_display: None,
        review_target:   None,
    };
    let req: SubmitAnswerRequest = serde_json::from_value(json!({ "kind": "no" })).unwrap();

    let answer = answer_from_request(req, &question).unwrap();

    assert_eq!(answer.value, AnswerValue::No);
}

#[test]
fn answer_from_typed_selected_request_validates_and_attaches_option() {
    let question = InterviewQuestionRecord {
        id:              "q-1".to_string(),
        text:            "Choose one.".to_string(),
        stage:           "gate".to_string(),
        question_type:   QuestionType::MultipleChoice,
        options:         vec![fabro_types::run_event::InterviewOption {
            key:         "approve".to_string(),
            label:       "Approve".to_string(),
            description: None,
            preview:     None,
        }],
        allow_freeform:  false,
        timeout_seconds: None,
        context_display: None,
        review_target:   None,
    };
    let req: SubmitAnswerRequest =
        serde_json::from_value(json!({ "kind": "selected", "option_key": "approve" })).unwrap();

    let answer = answer_from_request(req, &question).unwrap();

    assert_eq!(answer.value, AnswerValue::Selected("approve".to_string()));
    assert_eq!(
        answer
            .selected_option
            .as_ref()
            .map(|option| option.label.as_str()),
        Some("Approve")
    );
}

#[test]
fn answer_from_typed_multi_selected_request_validates_option_keys() {
    let question = InterviewQuestionRecord {
        id:              "q-1".to_string(),
        text:            "Choose many.".to_string(),
        stage:           "gate".to_string(),
        question_type:   QuestionType::MultiSelect,
        options:         vec![
            fabro_types::run_event::InterviewOption {
                key:         "approve".to_string(),
                label:       "Approve".to_string(),
                description: None,
                preview:     None,
            },
            fabro_types::run_event::InterviewOption {
                key:         "notify".to_string(),
                label:       "Notify".to_string(),
                description: None,
                preview:     None,
            },
        ],
        allow_freeform:  false,
        timeout_seconds: None,
        context_display: None,
        review_target:   None,
    };
    let req: SubmitAnswerRequest = serde_json::from_value(json!({
        "kind": "multi_selected",
        "option_keys": ["approve", "notify"],
    }))
    .unwrap();

    let answer = answer_from_request(req, &question).unwrap();

    assert_eq!(
        answer.value,
        AnswerValue::MultiSelected(vec!["approve".to_string(), "notify".to_string()])
    );
}

#[tokio::test]
async fn get_events_not_found() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{missing_run_id}/events")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_state_returns_projection() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/state")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert!(body["stages"].is_object());
}

#[tokio::test]
async fn get_run_logs_returns_per_run_log_file() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;
    let log_path = Storage::new(state.server_storage_dir())
        .run_scratch(&run_id)
        .runtime_dir()
        .join("server.log");
    tokio::fs::create_dir_all(log_path.parent().unwrap())
        .await
        .unwrap();
    tokio::fs::write(&log_path, b"worker log line\nsecond line\n")
        .await
        .unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/logs")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let content_type = response
        .headers()
        .get(header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .map(str::to_owned);
    let body = response_bytes!(response, StatusCode::OK).await;

    assert_eq!(content_type.as_deref(), Some("text/plain; charset=utf-8"));
    assert_eq!(&body[..], b"worker log line\nsecond line\n");
}

#[tokio::test]
async fn get_run_logs_returns_not_found_for_missing_run() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(state);
    let missing_run_id = RunId::new();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{missing_run_id}/logs")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_logs_returns_not_found_when_log_file_is_missing() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/logs")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_stage_command_log_returns_scratch_slice() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let stage_id = StageId::new("script_node", 1);
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "script_node".to_string(),
            name:                  "Script".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
        workflow_event::Event::CommandStarted {
            node_id:    "script_node".to_string(),
            script:     "echo hello world".to_string(),
            command:    "echo hello world".to_string(),
            language:   "shell".to_string(),
            timeout_ms: None,
        },
    ])
    .await;
    let run_dir = Storage::new(state.server_storage_dir())
        .run_scratch(&run_id)
        .root()
        .to_path_buf();
    let log_path = command_log_path(&run_dir, &stage_id);
    tokio::fs::create_dir_all(log_path.parent().unwrap())
        .await
        .unwrap();
    tokio::fs::write(&log_path, b"hello world").await.unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/logs/output?offset=6&limit=5"
        )))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let bytes = BASE64_STANDARD
        .decode(body["bytes_base64"].as_str().unwrap())
        .unwrap();

    assert!(body.get("stream").is_none());
    assert_eq!(body["offset"], 6);
    assert_eq!(body["next_offset"], 11);
    assert_eq!(body["total_bytes"], 11);
    assert_eq!(bytes, b"world");
    assert_eq!(body["eof"], false);
    assert_eq!(body["cas_ref"], serde_json::Value::Null);
    assert_eq!(body["live_streaming"], true);
}

#[tokio::test]
async fn get_run_stage_command_log_returns_cas_slice() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let run_store = state.stores.runs.create_run(&run_id).await.unwrap();
    append_default_run_created(&run_store, run_id).await;
    let output_blob = run_store
        .write_blob(&serde_json::to_vec("hello world").unwrap())
        .await
        .unwrap();
    let output_ref = format!("blob://sha256/{output_blob}");
    for event in [
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "script_node".to_string(),
            name:                  "Script".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
        workflow_event::Event::CommandCompleted {
            node_id:        "script_node".to_string(),
            output:         output_ref.clone(),
            exit_code:      Some(0),
            duration_ms:    5,
            termination:    CommandTermination::Exited,
            output_bytes:   11,
            live_streaming: false,
        },
    ] {
        workflow_event::append_event(&run_store, &run_id, &event)
            .await
            .unwrap();
    }

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/script_node@1/logs/output?offset=6&limit=5"
        )))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let bytes = BASE64_STANDARD
        .decode(body["bytes_base64"].as_str().unwrap())
        .unwrap();

    assert!(body.get("stream").is_none());
    assert_eq!(body["offset"], 6);
    assert_eq!(body["next_offset"], 11);
    assert_eq!(body["total_bytes"], 11);
    assert_eq!(bytes, b"world");
    assert_eq!(body["eof"], true);
    assert_eq!(body["cas_ref"], output_ref);
    assert_eq!(body["live_streaming"], false);
}

#[tokio::test]
async fn get_run_stage_command_log_prefers_scratch_when_cas_ref_exists() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let stage_id = StageId::new("script_node", 1);
    let run_store = state.stores.runs.create_run(&run_id).await.unwrap();
    append_default_run_created(&run_store, run_id).await;
    let output_blob = run_store
        .write_blob(&serde_json::to_vec("cas log").unwrap())
        .await
        .unwrap();
    let output_ref = format!("blob://sha256/{output_blob}");
    for event in [
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "script_node".to_string(),
            name:                  "Script".to_string(),
            index:                 1,
            handler_type:          "command".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
        workflow_event::Event::CommandCompleted {
            node_id:        "script_node".to_string(),
            output:         output_ref.clone(),
            exit_code:      Some(0),
            duration_ms:    5,
            termination:    CommandTermination::Exited,
            output_bytes:   7,
            live_streaming: false,
        },
    ] {
        workflow_event::append_event(&run_store, &run_id, &event)
            .await
            .unwrap();
    }

    let run_dir = Storage::new(state.server_storage_dir())
        .run_scratch(&run_id)
        .root()
        .to_path_buf();
    let log_path = command_log_path(&run_dir, &stage_id);
    tokio::fs::create_dir_all(log_path.parent().unwrap())
        .await
        .unwrap();
    tokio::fs::write(&log_path, b"scratch log").await.unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/logs/output?offset=0&limit=64"
        )))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let bytes = BASE64_STANDARD
        .decode(body["bytes_base64"].as_str().unwrap())
        .unwrap();

    assert!(body.get("stream").is_none());
    assert_eq!(body["offset"], 0);
    assert_eq!(body["next_offset"], 11);
    assert_eq!(body["total_bytes"], 11);
    assert_eq!(bytes, b"scratch log");
    assert_eq!(body["eof"], true);
    assert_eq!(body["cas_ref"], output_ref);
    assert_eq!(body["live_streaming"], false);
}

#[tokio::test]
async fn get_run_stage_command_log_returns_not_found_for_missing_stage() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/stages/missing@1/logs/output")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_stage_context_window_returns_not_found_for_missing_run() {
    let app = crate::test_support::build_test_router(test_app_state_with_isolated_storage());
    let run_id = RunId::new();

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/agent@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_stage_context_window_returns_not_found_for_missing_stage() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/missing@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_run_stage_context_window_returns_unavailable_for_non_agent_stage() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        stage_started_event("script_node", "command"),
        command_started_event("script_node"),
    ])
    .await;

    let body = response_json!(
        app.oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/script_node@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap(),
        StatusCode::OK
    )
    .await;

    assert_eq!(body["available"], false);
    assert_eq!(body["unavailable_reason"], "not_agent_stage");
    assert_eq!(body["breakdown"], json!([]));
    assert_eq!(body["staleness"], "unavailable");
}

#[tokio::test]
async fn get_run_stage_context_window_returns_not_observed_for_agent_stage_without_snapshot() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        agent_session_activated_event("agent_node", 1),
    ])
    .await;

    let body = response_json!(
        app.oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/agent_node@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap(),
        StatusCode::OK
    )
    .await;

    assert_eq!(body["available"], false);
    assert_eq!(body["unavailable_reason"], "not_observed");
    assert_eq!(body["input_tokens"], serde_json::Value::Null);
    assert!(!body["warnings"].as_array().unwrap().is_empty());
}

#[tokio::test]
async fn get_run_stage_context_window_returns_live_projected_snapshot() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        stage_started_event("agent_node", "agent"),
        context_window_event(
            "agent_node",
            1,
            context_window_snapshot(123_456, Vec::new()),
        ),
    ])
    .await;

    let body = response_json!(
        app.oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/agent_node@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap(),
        StatusCode::OK
    )
    .await;

    assert_eq!(body["stage_id"], "agent_node@1");
    assert_eq!(body["available"], true);
    assert_eq!(body["provider"], "openai");
    assert_eq!(body["count_method"], "response_usage_scaled_breakdown");
    assert_eq!(body["staleness"], "live");
    assert_eq!(body["input_tokens"], 123_456);
    assert_eq!(body["breakdown"][0]["category"], "conversation");
}

#[tokio::test]
async fn get_run_stage_context_window_marks_completed_stage_snapshot_stored() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        stage_started_event("agent_node", "agent"),
        context_window_event("agent_node", 1, context_window_snapshot(100, Vec::new())),
        stage_completed_event("agent_node"),
    ])
    .await;

    let body = response_json!(
        app.oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/agent_node@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap(),
        StatusCode::OK
    )
    .await;

    assert_eq!(body["available"], true);
    assert_eq!(body["staleness"], "stored");
    assert_eq!(body["input_tokens"], 100);
}

#[tokio::test]
async fn get_run_stage_context_window_returns_projected_warnings() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        stage_started_event("agent_node", "agent"),
        context_window_event(
            "agent_node",
            1,
            context_window_snapshot(100, vec![StageContextWindowWarning {
                code:    "provider_token_count_failed".to_string(),
                message: "provider input token counting failed; returned local estimate"
                    .to_string(),
            }]),
        ),
    ])
    .await;

    let body = response_json!(
        app.oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/stages/agent_node@1/context-window"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap(),
        StatusCode::OK
    )
    .await;

    assert_eq!(body["warnings"][0]["code"], "provider_token_count_failed");
}

#[tokio::test]
async fn get_run_pull_request_returns_live_detail_from_github() {
    let github = MockServer::start();
    let github_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/pulls/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body(
                json!({
                    "number": 42,
                    "title": "Fix the bug",
                    "body": "Detailed description",
                    "state": "closed",
                    "draft": false,
                    "merged": true,
                    "merged_at": "2026-04-23T15:45:00Z",
                    "mergeable": false,
                    "additions": 10,
                    "deletions": 3,
                    "changed_files": 2,
                    "html_url": "https://github.com/acme/widgets/pull/42",
                    "user": { "login": "testuser" },
                    "head": { "ref": "feature" },
                    "base": { "ref": "main" },
                    "created_at": "2026-04-23T15:40:00Z",
                    "updated_at": "2026-04-23T15:45:00Z"
                })
                .to_string(),
            );
    });
    let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url()));

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["data"]["link"]["number"], 42);
    assert_eq!(body["data"]["link"]["owner"], "acme");
    assert_eq!(body["data"]["details"]["state"], "closed");
    assert_eq!(body["data"]["details"]["merged"], true);
    assert_eq!(body["data"]["details"]["head_branch"], "feature");
    assert_eq!(body["data"]["details"]["base_branch"], "main");
    assert_eq!(body["meta"]["details_status"], "available");
    github_mock.assert();
}

#[tokio::test]
async fn get_run_pull_request_returns_not_found_when_record_missing() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::NOT_FOUND).await;

    assert_eq!(body["errors"][0]["code"], "no_stored_record");
}

#[tokio::test]
async fn link_run_pull_request_links_github_pr_from_any_repo_and_updates_state() {
    let (_state, app, run_id) = pr_test_app_with_minimal_run(None, None).await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("PUT")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "html_url": "https://github.com/other/repo/pull/987"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["html_url"], "https://github.com/other/repo/pull/987");
    assert_eq!(body["owner"], "other");
    assert_eq!(body["repo"], "repo");
    assert_eq!(body["number"], 987);

    let state_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/state")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let state_body = response_json!(state_response, StatusCode::OK).await;
    assert_eq!(
        state_body["pull_request"]["html_url"],
        "https://github.com/other/repo/pull/987"
    );
}

#[tokio::test]
async fn link_run_pull_request_rejects_non_github_url() {
    let (_state, app, run_id) = pr_test_app_with_minimal_run(None, None).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("PUT")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "html_url": "https://gitlab.com/acme/widgets/-/merge_requests/42"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;

    assert_eq!(
        body["errors"][0]["code"],
        "unsupported_pull_request_provider"
    );
}

#[tokio::test]
async fn unlink_run_pull_request_appends_event_and_clears_projected_state() {
    let (state, app, run_id) = pr_test_app_with_minimal_run(None, None).await;
    let link_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("PUT")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "html_url": "https://github.com/acme/widgets/pull/42"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    response_json!(link_response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("DELETE")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["html_url"], "https://github.com/acme/widgets/pull/42");

    let state_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/state")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let state_body = response_json!(state_response, StatusCode::OK).await;
    assert!(state_body["pull_request"].is_null());

    let run_id = run_id.parse::<RunId>().unwrap();
    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let events = run_store.list_events().await.unwrap();
    assert!(events.iter().any(|event| {
        event.event.event_name() == "pull_request.unlinked"
            && event.event.properties().unwrap()["pull_request"]["html_url"]
                == "https://github.com/acme/widgets/pull/42"
    }));
}

#[tokio::test]
async fn get_run_pull_request_returns_stored_github_association_without_github_credentials() {
    let (state, app, run_id) = pr_test_app(None, None);

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["data"]["link"]["number"], 42);
    assert_eq!(
        body["data"]["link"]["html_url"],
        "https://github.com/acme/widgets/pull/42"
    );
    assert!(body["data"]["details"].is_null());
    assert_eq!(body["meta"]["details_status"], "unavailable");
    assert_eq!(
        body["meta"]["details_unavailable_reason"],
        "integration_unavailable"
    );
}

#[tokio::test]
async fn get_run_pull_request_returns_stored_github_association_when_github_pr_is_missing() {
    let github = MockServer::start();
    let github_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/pulls/42")
            .header("authorization", "Bearer ghu_test");
        then.status(404)
            .header("content-type", "application/json")
            .body(json!({ "message": "Not Found" }).to_string());
    });
    let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url()));

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["data"]["link"]["number"], 42);
    assert_eq!(
        body["data"]["link"]["html_url"],
        "https://github.com/acme/widgets/pull/42"
    );
    assert!(body["data"]["details"].is_null());
    assert_eq!(body["meta"]["details_status"], "unavailable");
    assert_eq!(body["meta"]["details_unavailable_reason"], "not_found");
    github_mock.assert();
}

#[tokio::test]
async fn create_run_pull_request_creates_and_persists_record() {
    let github = MockServer::start();
    let branch_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/branches/fabro/run/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body(json!({ "commit": { "sha": "final-sha" } }).to_string());
    });
    let create_mock = github.mock(|when, then| {
        when.method("POST")
            .path("/repos/acme/widgets/pulls")
            .header("authorization", "Bearer ghu_test");
        then.status(201)
            .header("content-type", "application/json")
            .body(
                json!({
                    "html_url": "https://github.com/acme/widgets/pull/42",
                    "number": 42,
                    "node_id": "PR_kwDOAA"
                })
                .to_string(),
            );
    });
    let find_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/pulls")
            .query_param("state", "open")
            .query_param("base", "main")
            .query_param("head", "acme:fabro/run/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body("[]");
    });
    let llm = MockServer::start_async().await;
    let response_mock = llm
        .mock_async(|when, then| {
            when.method(POST)
                .path("/v1/responses")
                .header("authorization", "Bearer openai-key");
            then.status(200)
                .header("content-type", "application/json")
                .json_body(openai_responses_payload(
                    &serde_json::to_string(&json!({
                        "title": "Mock title",
                        "body": "Narrative from mock.",
                    }))
                    .unwrap(),
                ));
        })
        .await;
    let state = create_github_token_app_state_with_env_lookup_and_llm_catalog_settings(
        Some("ghu_test"),
        Some(github.base_url()),
        |_| None,
        llm_catalog_settings_with_provider_base_url("openai", llm.url("/v1")),
    );
    state
        .stores
        .vault
        .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None)
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    Box::pin(create_completed_run_ready_for_pull_request(
        &state,
        run_id,
        Some("git@github.com:acme/widgets.git"),
        Some("main"),
        Some("fabro/run/42"),
        "diff --git a/src/lib.rs b/src/lib.rs\n+fn shipped() {}\n",
    ))
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "force": false,
                        "model": "gpt-5.4"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(
        response.headers().get(header::LOCATION).unwrap(),
        &format!("/api/v1/runs/{run_id}/pull_request/creation")
    );
    let body = response_json!(response, StatusCode::ACCEPTED).await;

    assert_eq!(body["status"], "pending");
    assert_eq!(body["model"], "gpt-5.4");

    // Starting the supervisor after the request simulates server recovery:
    // the durable pending event is enough to resume the operation.
    let supervisor = spawn_pull_request_creation_supervisor(Arc::clone(&state));

    let creation_body = wait_for_pull_request_creation(&app, run_id).await;

    assert_eq!(creation_body["status"], "succeeded");
    assert_eq!(creation_body["pull_request"]["number"], 42);
    assert_eq!(creation_body["pull_request"]["owner"], "acme");
    assert_eq!(creation_body["pull_request"]["repo"], "widgets");
    assert_eq!(
        creation_body["pull_request"]["html_url"],
        "https://github.com/acme/widgets/pull/42"
    );

    let state_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/state")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let state_body = response_json!(state_response, StatusCode::OK).await;
    assert_eq!(state_body["pull_request"]["number"], 42);
    assert_eq!(state_body["pull_request"]["owner"], "acme");
    assert_eq!(state_body["pull_request"]["repo"], "widgets");

    response_mock.assert_async().await;
    branch_mock.assert();
    find_mock.assert();
    create_mock.assert();
    state.shutdown_token().cancel();
    supervisor.await.unwrap();
}

#[tokio::test]
async fn create_run_pull_request_returns_the_active_durable_request() {
    let github = MockServer::start();
    let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run(
        Some("ghu_test"),
        Some(github.base_url()),
        Some("https://github.com/acme/widgets.git"),
    ))
    .await;

    let configured_provider_ids = state.ready_llm_provider_ids().await;
    let expected_default_model = state
        .catalog()
        .default_for_configured_ids(&configured_provider_ids)
        .id
        .to_string();
    let request_body = json!({
        "force": false,
        "model": null
    })
    .to_string();
    let first = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(request_body.clone()))
                .unwrap(),
        )
        .await
        .unwrap();
    let first_body = response_json!(first, StatusCode::ACCEPTED).await;

    let second = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(request_body))
                .unwrap(),
        )
        .await
        .unwrap();
    let second_body = response_json!(second, StatusCode::ACCEPTED).await;

    assert_eq!(first_body["id"], second_body["id"]);
    assert_eq!(first_body["model"], expected_default_model);
    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let events = run_store.list_events().await.unwrap();
    assert_eq!(
        events
            .iter()
            .filter(|event| event.event.event_name() == "pull_request.creation_requested")
            .count(),
        1
    );
}

#[tokio::test]
async fn create_run_pull_request_persists_generation_failure() {
    let github = MockServer::start();
    let branch_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/branches/fabro/run/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body(json!({ "commit": { "sha": "final-sha" } }).to_string());
    });
    let find_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/pulls")
            .query_param("state", "open")
            .query_param("base", "main")
            .query_param("head", "acme:fabro/run/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body("[]");
    });
    let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run(
        Some("ghu_test"),
        Some(github.base_url()),
        Some("https://github.com/acme/widgets.git"),
    ))
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({ "force": false, "model": "gpt-5.4" }).to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    response_json!(response, StatusCode::ACCEPTED).await;
    let supervisor = spawn_pull_request_creation_supervisor(Arc::clone(&state));

    let creation = wait_for_pull_request_creation(&app, run_id).await;

    assert_eq!(creation["status"], "failed");
    // The unconfigured LLM is what fails this fixture; pin the error to the
    // generation step so the test cannot pass on an earlier validation error.
    assert!(
        creation["error"]
            .as_str()
            .is_some_and(|error| error.contains("LLM generation failed")),
        "unexpected error: {:?}",
        creation["error"]
    );
    assert!(creation["pull_request"].is_null());
    branch_mock.assert();
    find_mock.assert();
    state.shutdown_token().cancel();
    supervisor.await.unwrap();
}

#[tokio::test]
async fn create_run_pull_request_returns_conflict_when_record_exists() {
    let (state, app, run_id) = pr_test_app(None, None);

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({ "force": false, "model": null }).to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CONFLICT).await;

    assert_eq!(body["errors"][0]["code"], "pull_request_exists");
    assert!(
        body["errors"][0]["detail"]
            .as_str()
            .unwrap()
            .contains("https://github.com/acme/widgets/pull/42")
    );
}

#[tokio::test]
async fn create_run_pull_request_rejects_missing_repo_origin() {
    let (_state, app, run_id) = Box::pin(pr_test_app_with_completed_run(None, None, None)).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "force": false,
                        "model": "claude-sonnet-4-6"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;

    assert_eq!(body["errors"][0]["code"], "missing_repo_origin");
}

#[tokio::test]
async fn create_run_pull_request_returns_service_unavailable_without_github_credentials() {
    let (_state, app, run_id) = Box::pin(pr_test_app_with_completed_run(
        None,
        None,
        Some("https://github.com/acme/widgets.git"),
    ))
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "force": false,
                        "model": "claude-sonnet-4-6"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await;

    assert_eq!(body["errors"][0]["code"], "integration_unavailable");
}

#[tokio::test]
async fn create_run_pull_request_rejects_non_github_origin_url() {
    let (_state, app, run_id) = Box::pin(pr_test_app_with_completed_run(
        Some("ghu_test"),
        None,
        Some("https://gitlab.com/acme/widgets.git"),
    ))
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({
                        "force": false,
                        "model": "claude-sonnet-4-6"
                    })
                    .to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;

    assert_eq!(body["errors"][0]["code"], "unsupported_host");
}

#[tokio::test]
async fn pull_request_endpoints_use_github_base_url_captured_at_startup() {
    let github = MockServer::start();
    let captured_mock = github.mock(|when, then| {
        when.method("GET")
            .path("/repos/acme/widgets/pulls/42")
            .header("authorization", "Bearer ghu_test");
        then.status(200)
            .header("content-type", "application/json")
            .body(
                json!({
                    "number": 42,
                    "title": "Captured",
                    "body": "",
                    "state": "open",
                    "draft": false,
                    "merged": false,
                    "mergeable": true,
                    "additions": 1,
                    "deletions": 0,
                    "changed_files": 1,
                    "html_url": "https://github.com/acme/widgets/pull/42",
                    "user": { "login": "octocat" },
                    "head": { "ref": "feature" },
                    "base": { "ref": "main" },
                    "created_at": "2026-04-23T12:00:00Z",
                    "updated_at": "2026-04-23T12:00:00Z"
                })
                .to_string(),
            );
    });
    let state = create_github_token_app_state(Some("ghu_test"), Some(github.base_url()));
    assert_eq!(state.github_api_base_url, github.base_url());

    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Captured",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/pull_request")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    response_json!(response, StatusCode::OK).await;

    // If the handler read GITHUB_BASE_URL at request time instead of using the
    // value captured at AppState construction, the outbound call would miss
    // this mock — no other server is running at the captured URL, and the
    // process env default points elsewhere.
    captured_mock.assert();
}

#[tokio::test]
async fn merge_run_pull_request_returns_not_found_when_record_missing() {
    let (_state, app, run_id) = pr_test_app_with_minimal_run(Some("ghu_test"), None).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/merge")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "method": "squash" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::NOT_FOUND).await;

    assert_eq!(body["errors"][0]["code"], "no_stored_record");
}

#[tokio::test]
async fn merge_run_pull_request_rejects_invalid_method() {
    let (state, app, run_id) = pr_test_app(Some("ghu_test"), None);

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/merge")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "method": "bogus" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
}

#[tokio::test]
async fn merge_run_pull_request_returns_service_unavailable_without_github_credentials() {
    let (state, app, run_id) = pr_test_app(None, None);

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/merge")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "method": "squash" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await;

    assert_eq!(body["errors"][0]["code"], "integration_unavailable");
}

#[tokio::test]
async fn merge_run_pull_request_uses_stored_link_coordinates() {
    let github = MockServer::start();
    let github_mock = github.mock(|when, then| {
        when.method("PUT")
            .path("/repos/acme/widgets/pulls/42/merge")
            .header("authorization", "Bearer ghu_test")
            .json_body(json!({ "merge_method": "squash" }));
        then.status(200)
            .header("content-type", "application/json")
            .body(json!({}).to_string());
    });
    let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url()));

    create_run_with_linked_pull_request_record(&state, run_id, PullRequestLink {
        owner:  "acme".to_string(),
        repo:   "widgets".to_string(),
        number: 42,
    })
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/merge")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "method": "squash" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["number"], 42);
    assert_eq!(body["html_url"], "https://github.com/acme/widgets/pull/42");
    github_mock.assert();
}

#[tokio::test]
async fn close_run_pull_request_returns_not_found_when_record_missing() {
    let (_state, app, run_id) = pr_test_app_with_minimal_run(Some("ghu_test"), None).await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/close")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::NOT_FOUND).await;

    assert_eq!(body["errors"][0]["code"], "no_stored_record");
}

#[tokio::test]
async fn close_run_pull_request_returns_service_unavailable_without_github_credentials() {
    let (state, app, run_id) = pr_test_app(None, None);

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/close")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await;

    assert_eq!(body["errors"][0]["code"], "integration_unavailable");
}

#[tokio::test]
async fn close_run_pull_request_returns_bad_gateway_when_github_pr_is_missing() {
    let github = MockServer::start();
    let github_mock = github.mock(|when, then| {
        when.method("PATCH")
            .path("/repos/acme/widgets/pulls/42")
            .header("authorization", "Bearer ghu_test");
        then.status(404)
            .header("content-type", "application/json")
            .body(json!({ "message": "Not Found" }).to_string());
    });
    let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url()));

    create_run_with_pull_request_record(
        &state,
        run_id,
        "https://github.com/acme/widgets/pull/42",
        42,
        "Fix the bug",
    )
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/pull_request/close")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::BAD_GATEWAY).await;

    assert_eq!(body["errors"][0]["code"], "github_not_found");
    github_mock.assert();
}

#[tokio::test]
async fn get_run_state_exposes_pending_interviews() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "pending-question",
        "2026-04-19T12:00:00Z",
        "interview.started",
        json!({
            "question_id": "q-1",
            "question": "Approve deploy?",
            "stage": "gate",
            "question_type": "multiple_choice",
            "options": [],
            "allow_freeform": false,
            "context_display": null,
            "timeout_seconds": null,
            "review_target": {
                "label": "Quarry review exercise",
                "url": "https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef",
                "kind": "document"
            },
        }),
        Some("gate"),
    )
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/state")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        body["pending_interviews"]["q-1"]["question"]["text"].as_str(),
        Some("Approve deploy?")
    );
    assert_eq!(
        body["pending_interviews"]["q-1"]["question"]["stage"].as_str(),
        Some("gate")
    );
}

#[tokio::test]
async fn cache_backed_run_endpoints_reflect_events_appended_after_warmup() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    state.stores.runs.warm_projection_cache().await.unwrap();

    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    })
    .await
    .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting)
        .await
        .unwrap();
    workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning)
        .await
        .unwrap();
    append_scoped_stage_event(
        &state,
        run_id,
        "review",
        1,
        &workflow_event::Event::StageStarted {
            graph_visit:           None,
            resumed_from_stage_id: None,
            node_id:               "review".to_string(),
            name:                  "Review".to_string(),
            index:                 0,
            handler_type:          "human".to_string(),
            attempt:               1,
            max_attempts:          1,
        },
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "cache-question",
        "2026-04-19T12:00:00Z",
        "interview.started",
        json!({
            "question_id": "q-cache",
            "question": "Approve cached deploy?",
            "stage": "review",
            "question_type": "yes_no",
            "options": [],
            "allow_freeform": false,
            "context_display": null,
            "timeout_seconds": null,
            "review_target": {
                "label": "Quarry review exercise",
                "url": "https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef",
                "kind": "document"
            },
        }),
        Some("review"),
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "cache-checkpoint",
        "2026-04-19T12:00:01Z",
        "checkpoint.completed",
        json!({
            "status": "running",
            "current_node": "review",
            "completed_nodes": [],
            "node_retries": {},
            "context_values": {},
            "node_outcomes": {},
            "next_node_id": "review",
            "git_commit_sha": "cache-sha",
            "loop_failure_signatures": {},
            "restart_failure_signatures": {},
            "node_visits": { "review": 1 },
        }),
        Some("review"),
    )
    .await;

    let status = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let status = response_json!(status, StatusCode::OK).await;
    assert_eq!(run_json_status(&status)["kind"].as_str(), Some("running"));

    let state_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/state")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let state_body = response_json!(state_response, StatusCode::OK).await;
    assert_eq!(
        state_body["pending_interviews"]["q-cache"]["question"]["text"].as_str(),
        Some("Approve cached deploy?")
    );
    assert_eq!(
        state_body["pending_interviews"]["q-cache"]["question"]["review_target"]["url"].as_str(),
        Some("https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef")
    );

    let stages = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/stages")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let stages = response_json!(stages, StatusCode::OK).await;
    assert_eq!(stages["data"][0]["id"].as_str(), Some("review@1"));

    let questions = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/questions")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let questions = response_json!(questions, StatusCode::OK).await;
    assert_eq!(
        questions["data"][0]["text"].as_str(),
        Some("Approve cached deploy?")
    );
    assert_eq!(
        questions["data"][0]["review_target"]["label"].as_str(),
        Some("Quarry review exercise")
    );

    let settings = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/settings")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(settings, StatusCode::OK).await;

    let checkpoint = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/checkpoint")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let checkpoint = response_json!(checkpoint, StatusCode::OK).await;
    assert_eq!(checkpoint["git_commit_sha"].as_str(), Some("cache-sha"));

    let billing = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/billing")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let billing = response_json!(billing, StatusCode::OK).await;
    assert_eq!(billing["stages"][0]["stage"]["id"].as_str(), Some("review"));
}

#[tokio::test]
async fn get_run_state_includes_provenance_from_user_agent() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .header("user-agent", "fabro-cli/1.2.3")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/state")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        body["spec"]["provenance"]["server"]["version"],
        FABRO_VERSION
    );
    assert_eq!(
        body["spec"]["provenance"]["client"]["user_agent"],
        "fabro-cli/1.2.3"
    );
    assert_eq!(body["spec"]["provenance"]["client"]["name"], "fabro-cli");
    assert_eq!(body["spec"]["provenance"]["client"]["version"], "1.2.3");
    assert_eq!(body["spec"]["provenance"]["subject"]["kind"], "user");
    assert_eq!(
        body["spec"]["provenance"]["subject"]["auth_method"],
        "dev_token"
    );
    assert_eq!(body["spec"]["provenance"]["subject"]["login"], "dev");
    assert_eq!(
        body["spec"]["provenance"]["subject"]["identity"]["issuer"],
        "fabro:dev"
    );
}

#[tokio::test]
async fn dev_token_web_login_authorizes_cookie_backed_api_requests() {
    const DEV_TOKEN: &str =
        "fabro_dev_abababababababababababababababababababababababababababababababab";

    let state = test_app_state_with_session_key(
        default_test_server_settings(),
        RunLayer::default(),
        Some("server-test-session-key-0123456789"),
    );
    let app = build_router(
        Arc::clone(&state),
        AuthMode::Enabled(ConfiguredAuth {
            methods:    vec![ServerAuthMethod::DevToken],
            dev_token:  Some(DEV_TOKEN.to_string()),
            jwt_key:    Some(
                auth::derive_jwt_key(b"server-test-session-key-0123456789")
                    .expect("test JWT key should derive"),
            ),
            jwt_issuer: Some("https://fabro.example".to_string()),
        }),
    );

    let login_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/auth/login/dev-token")
                .header(header::CONTENT_TYPE, "application/json")
                .body(Body::from(json!({ "token": DEV_TOKEN }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let login_response = checked_response!(login_response, StatusCode::OK).await;
    let session_cookie = login_response
        .headers()
        .get(header::SET_COOKIE)
        .and_then(|value| value.to_str().ok())
        .and_then(|value| value.split(';').next())
        .expect("session cookie should be set")
        .to_string();

    let create_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header(header::CONTENT_TYPE, "application/json")
                .header(header::COOKIE, &session_cookie)
                .body(manifest_body(MINIMAL_DOT))
                .unwrap(),
        )
        .await
        .unwrap();
    let create_body = response_json!(create_response, StatusCode::CREATED).await;
    let run_id = create_body["id"].as_str().unwrap();

    let state_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/state")))
                .header(header::COOKIE, &session_cookie)
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let state_body = response_json!(state_response, StatusCode::OK).await;
    assert_eq!(
        state_body["spec"]["provenance"]["subject"]["auth_method"],
        "dev_token"
    );
    assert_eq!(state_body["spec"]["provenance"]["subject"]["login"], "dev");
}

#[tokio::test]
async fn create_run_persists_manifest_and_definition_blobs_without_bundle_file() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let raw_manifest = serde_json::to_string_pretty(&minimal_manifest_json(MINIMAL_DOT)).unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(Body::from(raw_manifest.clone()))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let events = run_store.list_events().await.unwrap();
    let created = events[0].event.to_value().unwrap();
    let submitted = events[1].event.to_value().unwrap();
    let manifest_blob = created["properties"]["manifest_blob"]
        .as_str()
        .expect("run.created should carry manifest_blob")
        .parse::<BlobHash>()
        .unwrap();
    let definition_blob = submitted["properties"]["definition_blob"]
        .as_str()
        .expect("run.submitted should carry definition_blob")
        .parse::<BlobHash>()
        .unwrap();

    let submitted_manifest_bytes = run_store
        .read_blob(&manifest_blob)
        .await
        .unwrap()
        .expect("submitted manifest blob should exist");
    assert_eq!(submitted_manifest_bytes.as_ref(), raw_manifest.as_bytes());

    let accepted_definition_bytes = run_store
        .read_blob(&definition_blob)
        .await
        .unwrap()
        .expect("accepted definition blob should exist");
    let accepted_definition: serde_json::Value =
        serde_json::from_slice(&accepted_definition_bytes).unwrap();
    assert!(
        accepted_definition.get("version").is_none(),
        "accepted run definition should not carry compatibility versioning"
    );
    assert_eq!(accepted_definition["workflow_path"], "workflow.fabro");
    assert!(accepted_definition["workflows"]["workflow.fabro"].is_object());
}

#[tokio::test]
async fn list_run_events_returns_paginated_json() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/events?since_seq=1&limit=5")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert!(body["data"].is_array());
    assert!(body["meta"]["has_more"].is_boolean());
}

#[tokio::test]
async fn list_run_events_descends_from_latest_with_exclusive_cursor() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunRunnable {
            source: fabro_types::RunRunnableSource::StartRequested,
            actor:  None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/events?order=desc&limit=2")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let seqs = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .map(|event| event["seq"].as_u64().unwrap())
        .collect::<Vec<_>>();
    assert_eq!(seqs, vec![4, 3]);
    assert_eq!(body["meta"]["has_more"], true);

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!(
                    "/runs/{run_id}/events?order=desc&before_seq=3&limit=2"
                )))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let seqs = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .map(|event| event["seq"].as_u64().unwrap())
        .collect::<Vec<_>>();
    assert_eq!(seqs, vec![2, 1]);
    assert_eq!(body["meta"]["has_more"], false);
}

#[tokio::test]
async fn list_run_events_rejects_cursor_for_opposite_order() {
    let app = crate::test_support::build_test_router(test_app_state());
    let run_id = RunId::new();
    let cases = [
        (
            format!("/runs/{run_id}/events?order=desc&since_seq=2"),
            "since_seq cannot be combined with order=desc; use before_seq instead.",
        ),
        (
            format!("/runs/{run_id}/events?before_seq=2"),
            "before_seq requires order=desc.",
        ),
    ];

    for (path, expected_detail) in cases {
        let response = app
            .clone()
            .oneshot(
                Request::builder()
                    .method("GET")
                    .uri(api(&path))
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap();
        let body = response_json!(response, StatusCode::BAD_REQUEST).await;
        assert_eq!(body["errors"][0]["detail"], expected_detail);
    }
}

#[tokio::test]
async fn append_run_event_rejects_run_id_mismatch() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/events")))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::json!({
                "id": "evt-test",
                "ts": "2026-03-27T12:00:00Z",
                "run_id": fixtures::RUN_64.to_string(),
                "event": "run.submitted",
                "properties": {}
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn append_run_event_rejects_reserved_archive_event() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT).await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/events")))
        .header("content-type", "application/json")
        .body(Body::from(
            json!({
                "id": "evt-run-archived",
                "ts": "2026-04-19T12:00:00Z",
                "run_id": run_id,
                "event": "run.archived",
                "properties": {
                    "actor": null
                }
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;
    assert!(
        body["errors"][0]["detail"]
            .as_str()
            .is_some_and(|message| message.contains("run.archived is a lifecycle event")),
        "expected lifecycle rejection, got: {body}"
    );
}

#[tokio::test]
async fn get_checkpoint_returns_null_initially() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Start a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    // Get checkpoint immediately (before run completes, may be null)
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/checkpoint")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    checked_response!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn write_and_read_run_blob_accepts_uppercase_hash() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/blobs")))
        .header("content-type", "application/octet-stream")
        .body(Body::from("hello blob"))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let blob_hash = body["hash"].as_str().unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/blobs/{}",
            blob_hash.to_uppercase()
        )))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let bytes = response_bytes!(response, StatusCode::OK).await;
    assert_eq!(&bytes[..], b"hello blob");
}

#[tokio::test]
async fn stage_artifacts_round_trip() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT).await;
    let stage_id = "code@2";

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts?filename=src/lib.rs&retry=1"
        )))
        .header("content-type", "application/octet-stream")
        .body(Body::from("fn main() {}"))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/stages/{stage_id}/artifacts")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["data"][0]["filename"], "src/lib.rs");
    assert_eq!(body["data"][0]["retry"], 1);
    assert_eq!(body["data"][0]["size"], 12);

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts/download?filename=src/lib.rs"
        )))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts/download?filename=src/lib.rs&retry=1"
        )))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let bytes = response_bytes!(response, StatusCode::OK).await;
    assert_eq!(&bytes[..], b"fn main() {}");
}

#[tokio::test]
async fn stage_artifacts_keep_same_filename_per_retry() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT).await;
    let stage_id = "code@2";

    for (retry, body) in [(1, "first"), (2, "second")] {
        let req = Request::builder()
            .method("POST")
            .uri(api(&format!(
                "/runs/{run_id}/stages/{stage_id}/artifacts?filename=logs/output.txt&retry={retry}"
            )))
            .header("content-type", "application/octet-stream")
            .body(Body::from(body))
            .unwrap();
        let response = app.clone().oneshot(req).await.unwrap();
        assert_status!(response, StatusCode::NO_CONTENT).await;
    }

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/stages/{stage_id}/artifacts")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["data"][0]["filename"], "logs/output.txt");
    assert_eq!(body["data"][0]["retry"], 1);
    assert_eq!(body["data"][1]["filename"], "logs/output.txt");
    assert_eq!(body["data"][1]["retry"], 2);

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts/download?filename=logs/output.txt&retry=2"
        )))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let bytes = response_bytes!(response, StatusCode::OK).await;
    assert_eq!(&bytes[..], b"second");
}

#[tokio::test]
async fn run_artifacts_download_streams_latest_files_as_zip() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    for event in [
        workflow_event::Event::RunRunnable {
            source: fabro_types::RunRunnableSource::StartRequested,
            actor:  None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ] {
        workflow_event::append_event(&run_store, &run_id, &event)
            .await
            .unwrap();
    }
    append_scoped_stage_event(
        &state,
        run_id,
        "build",
        1,
        &stage_started_event("build", "command"),
    )
    .await;
    append_scoped_stage_event(
        &state,
        run_id,
        "verify",
        1,
        &stage_started_event("verify", "command"),
    )
    .await;

    for (stage_id, retry, path, contents) in [
        (
            StageId::new("unknown", 1),
            99,
            "reports/result.txt",
            &b"unknown stage"[..],
        ),
        (
            StageId::new("build", 1),
            1,
            "reports/result.txt",
            &b"build result"[..],
        ),
        (
            StageId::new("verify", 1),
            1,
            "reports/result.txt",
            &b"first verify"[..],
        ),
        (
            StageId::new("verify", 1),
            2,
            "reports/result.txt",
            &b"latest verify"[..],
        ),
        (
            StageId::new("build", 1),
            1,
            "logs/run.txt",
            &b"build log"[..],
        ),
        (
            StageId::new("start", 1),
            1,
            "control-start.txt",
            &b"excluded"[..],
        ),
        (
            StageId::new("exit", 1),
            1,
            "control-exit.txt",
            &b"excluded"[..],
        ),
        // Neither stage reached the projection, so both rank equally on stage
        // order and retry. The serialized stage ID breaks the tie the same way
        // the artifacts page does: "unknown@2" sorts above "unknown@10".
        (
            StageId::new("unknown", 10),
            1,
            "orphan.txt",
            &b"visit ten"[..],
        ),
        (
            StageId::new("unknown", 2),
            1,
            "orphan.txt",
            &b"visit two"[..],
        ),
    ] {
        state
            .artifact_store
            .put(&run_id, &ArtifactKey::new(stage_id, retry, path), contents)
            .await
            .unwrap();
    }

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}/artifacts/download")))
                .header(header::ACCEPT_ENCODING, "gzip")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(
        response
            .headers()
            .get(header::CONTENT_TYPE)
            .and_then(|value| value.to_str().ok()),
        Some("application/zip")
    );
    assert_eq!(
        response
            .headers()
            .get(header::CONTENT_DISPOSITION)
            .and_then(|value| value.to_str().ok()),
        Some(format!("attachment; filename=\"fabro-artifacts-{run_id}.zip\"").as_str())
    );
    assert_eq!(
        response
            .headers()
            .get(header::CACHE_CONTROL)
            .and_then(|value| value.to_str().ok()),
        Some("private, no-store")
    );
    assert!(response.headers().get(header::CONTENT_ENCODING).is_none());

    let bytes = response_bytes!(response, StatusCode::OK).await;
    let archive = ZipFileReader::new(bytes).await.unwrap();
    let names = archive
        .file()
        .entries()
        .iter()
        .map(|entry| entry.filename().as_str().unwrap().to_string())
        .collect::<Vec<_>>();
    assert_eq!(names, vec![
        "logs/run.txt",
        "orphan.txt",
        "reports/result.txt"
    ]);

    let mut contents_by_name = HashMap::new();
    for (index, name) in names.into_iter().enumerate() {
        let mut entry = archive.reader_with_entry(index).await.unwrap();
        let mut contents = Vec::new();
        entry.read_to_end_checked(&mut contents).await.unwrap();
        contents_by_name.insert(name, contents);
    }
    assert_eq!(contents_by_name["logs/run.txt"], b"build log");
    assert_eq!(contents_by_name["reports/result.txt"], b"latest verify");
    assert_eq!(contents_by_name["orphan.txt"], b"visit two");
}

#[tokio::test]
async fn run_artifacts_download_returns_not_found_for_unknown_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{}/artifacts/download", RunId::new())))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn create_run_persists_run_spec() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let run_state = state
        .stores
        .runs
        .open_run_reader(&run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();

    assert_eq!(run_state.spec.graph.name, "Test");
}

#[tokio::test]
async fn create_run_keeps_missing_project_and_workflow_names_absent() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let manifest = serde_json::json!({
        "version": 1,
        "cwd": "/tmp/project",
        "target": {
            "path": "workflow.fabro",
        },
        "configs": [
            {
                "path": "/tmp/project/.fabro/project.toml",
                "source": "_version = 1\n",
                "type": "project",
            }
        ],
        "workflows": {
            "workflow.fabro": {
                "source": "digraph Demo { start [shape=Mdiamond] exit [shape=Msquare] start -> exit }",
                "config": {
                    "path": "workflow.toml",
                    "source": "_version = 1\n",
                },
                "files": {},
            }
        },
    });

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api("/runs"))
                .header("content-type", "application/json")
                .body(Body::from(manifest.to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let run_state = state
        .stores
        .runs
        .open_run_reader(&run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();

    assert_eq!(run_state.spec.settings.project.name.as_deref(), None);
    assert_eq!(run_state.spec.settings.workflow.name.as_deref(), None);
    assert_eq!(run_state.spec.graph_name(), Some("Demo"));
}

#[tokio::test]
async fn stage_artifact_upload_rejects_invalid_filename() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT).await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!(
            "/runs/{run_id}/stages/code@2/artifacts?filename=../escape.txt&retry=1"
        )))
        .header("content-type", "application/octet-stream")
        .body(Body::from("nope"))
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn worker_token_accepts_run_scoped_routes_and_falls_back_to_user_jwt() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let other_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let other_worker_token = issue_test_worker_token(&other_run_id);
    let blob_hash = state
        .stores
        .runs
        .open_run(&run_id)
        .await
        .unwrap()
        .write_blob(b"preloaded blob")
        .await
        .unwrap();

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/state"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    for path in [
        format!("/runs/{run_id}"),
        format!("/runs/{run_id}/questions"),
    ] {
        let response = app
            .clone()
            .oneshot(bearer_request(
                Method::GET,
                &path,
                &worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert_status!(response, StatusCode::OK).await;
    }

    let append_body = serde_json::to_vec(&serde_json::json!({
        "id": "evt-run-notice",
        "ts": "2026-04-23T12:00:00Z",
        "event": "run.notice",
        "run_id": run_id.to_string(),
        "properties": {
            "level": "info",
            "code": "worker",
            "message": "hello"
        }
    }))
    .unwrap();
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method(Method::POST)
                .uri(api(&format!("/runs/{run_id}/events")))
                .header(header::AUTHORIZATION, format!("Bearer {worker_token}"))
                .header(header::CONTENT_TYPE, "application/json")
                .body(Body::from(append_body))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/events"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::POST,
            &format!("/runs/{run_id}/blobs"),
            &worker_token,
            Body::from("worker blob"),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/blobs/{blob_hash}"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/state"),
            &user_jwt,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/state"),
            &other_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::FORBIDDEN).await;
}

#[tokio::test]
async fn run_tool_worker_token_can_use_client_backend_routes_across_runs() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let parent_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let target_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let run_tool_worker_token = issue_test_run_tools_worker_token(&parent_run_id);

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            "/runs",
            &run_tool_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/resolve?selector={target_run_id}"),
            &run_tool_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    for path in [
        format!("/runs/{target_run_id}"),
        format!("/runs/{target_run_id}/state"),
        format!("/runs/{target_run_id}/events"),
        format!("/runs/{target_run_id}/questions"),
    ] {
        let response = app
            .clone()
            .oneshot(bearer_request(
                Method::GET,
                &path,
                &run_tool_worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert_status!(response, StatusCode::OK).await;
    }

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{target_run_id}/start"),
            &run_tool_worker_token,
            &json!({ "resume": false }),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::POST,
            &format!("/runs/{target_run_id}/cancel"),
            &run_tool_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    for path in [
        format!("/runs/{target_run_id}/archive"),
        format!("/runs/{target_run_id}/unarchive"),
        format!("/runs/{target_run_id}/interrupt"),
    ] {
        let response = app
            .clone()
            .oneshot(bearer_request(
                Method::POST,
                &path,
                &run_tool_worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert_ne!(response.status(), StatusCode::UNAUTHORIZED, "{path}");
        assert_ne!(response.status(), StatusCode::FORBIDDEN, "{path}");
    }

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{target_run_id}/steer"),
            &run_tool_worker_token,
            &json!({ "text": "continue", "interrupt": false }),
        ))
        .await
        .unwrap();
    assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
    assert_ne!(response.status(), StatusCode::FORBIDDEN);

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{target_run_id}/questions/q-1/answer"),
            &run_tool_worker_token,
            &json!({ "kind": "yes" }),
        ))
        .await
        .unwrap();
    assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
    assert_ne!(response.status(), StatusCode::FORBIDDEN);

    let created_child = create_run_with_bearer(&app, &run_tool_worker_token).await;
    let cached = state
        .stores
        .runs
        .get_cached_run(&created_child)
        .await
        .unwrap()
        .expect("created run should be cached");
    assert_eq!(
        cached.projection.spec.provenance.subject,
        Principal::Worker {
            run_id: parent_run_id,
        },
    );

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::PUT,
            &format!("/runs/{created_child}/parent"),
            &run_tool_worker_token,
            &json!({ "parent_id": target_run_id.to_string() }),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::DELETE,
            &format!("/runs/{created_child}/parent"),
            &run_tool_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn run_tools_worker_can_read_pair_status_and_transcript_across_runs() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let origin_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let target_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_run_tools_worker_token(&origin_run_id);
    let pair_id = append_pair_transcript_fixture(&state, target_run_id).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{target_run_id}/pair"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    let status_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(status_body["run_id"], target_run_id.to_string());

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{target_run_id}/pair/{pair_id}/transcript"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    let transcript_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(transcript_body["data"].as_array().unwrap().len(), 1);
}

#[tokio::test]
async fn run_tools_worker_start_pair_reaches_worker_control_domain_across_runs() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let origin_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let target_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_run_tools_worker_token(&origin_run_id);
    let target = pair_test_target();
    let _temp_dir = insert_running_control_run(&state, target_run_id, None);
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.get_mut(&target_run_id)
            .unwrap()
            .active_api_targets
            .insert(target.stage_id.clone(), target.clone());
    }

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{target_run_id}/pair"),
            &worker_token,
            &json!({ "stage_id": target.stage_id.to_string() }),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await;
    assert_eq!(body["errors"][0]["code"], "worker_control_unavailable");
}

#[tokio::test]
async fn cross_run_base_worker_remains_forbidden_from_pair_routes() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let origin_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let target_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&origin_run_id);

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{target_run_id}/pair"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::FORBIDDEN).await;
}

#[tokio::test]
async fn run_tools_worker_cannot_call_user_only_non_mcp_routes() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let origin_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let target_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_run_tools_worker_token(&origin_run_id);

    for (method, path) in [
        (Method::POST, format!("/runs/{target_run_id}/approve")),
        (Method::POST, format!("/runs/{target_run_id}/deny")),
        (Method::GET, format!("/runs/{target_run_id}/timeline")),
    ] {
        let response = app
            .clone()
            .oneshot(bearer_request(
                method.clone(),
                &path,
                &worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert!(
            matches!(
                response.status(),
                StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
            ),
            "{method} {path} unexpectedly accepted run-tools worker token with status {}",
            response.status()
        );
    }
}

#[tokio::test]
async fn base_worker_token_is_rejected_by_run_tool_only_routes() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);

    for (method, path) in [
        (Method::GET, "/runs".to_string()),
        (Method::POST, "/runs".to_string()),
        (Method::GET, "/runs/resolve?selector=latest".to_string()),
    ] {
        let response = app
            .clone()
            .oneshot(bearer_request(
                method.clone(),
                &path,
                &worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert!(
            matches!(
                response.status(),
                StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
            ),
            "{method} {path} unexpectedly accepted base worker token with status {}",
            response.status()
        );
    }
}

#[tokio::test]
async fn worker_token_controls_stage_artifact_route() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let other_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let mismatched_worker_token = issue_test_worker_token(&other_run_id);

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method(Method::POST)
                .uri(api(&format!(
                    "/runs/{run_id}/stages/code@2/artifacts?filename=artifact.txt&retry=1"
                )))
                .header(header::AUTHORIZATION, format!("Bearer {worker_token}"))
                .header(header::CONTENT_TYPE, "application/octet-stream")
                .body(Body::from("artifact"))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method(Method::POST)
                .uri(api(&format!(
                    "/runs/{run_id}/stages/code@2/artifacts?filename=artifact.txt&retry=1"
                )))
                .header(header::AUTHORIZATION, format!("Bearer {user_jwt}"))
                .header(header::CONTENT_TYPE, "application/octet-stream")
                .body(Body::from("artifact"))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method(Method::POST)
                .uri(api(&format!(
                    "/runs/{run_id}/stages/code@2/artifacts?filename=artifact.txt&retry=1"
                )))
                .header(
                    header::AUTHORIZATION,
                    format!("Bearer {mismatched_worker_token}"),
                )
                .header(header::CONTENT_TYPE, "application/octet-stream")
                .body(Body::from("artifact"))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::FORBIDDEN).await;

    let response = app
        .oneshot(
            Request::builder()
                .method(Method::POST)
                .uri(api(&format!(
                    "/runs/{run_id}/stages/code@2/artifacts?filename=artifact.txt&retry=1"
                )))
                .header(header::CONTENT_TYPE, "application/octet-stream")
                .body(Body::from("artifact"))
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::UNAUTHORIZED).await;
}

#[tokio::test]
async fn worker_token_controls_command_log_route() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let other_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let mismatched_worker_token = issue_test_worker_token(&other_run_id);
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::CommandStarted {
            node_id:    "code".to_string(),
            script:     "echo hello".to_string(),
            command:    "echo hello".to_string(),
            language:   "shell".to_string(),
            timeout_ms: None,
        },
    )
    .await
    .unwrap();

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/stages/code@1/logs/output"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/stages/code@1/logs/output"),
            &user_jwt,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/stages/code@1/logs/output"),
            &mismatched_worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::FORBIDDEN).await;

    let response = app
        .oneshot(
            Request::builder()
                .method(Method::GET)
                .uri(api(&format!("/runs/{run_id}/stages/code@1/logs/output")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::UNAUTHORIZED).await;
}

#[tokio::test]
async fn worker_token_is_rejected_on_user_only_routes() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let blob_hash = BlobHash::new(b"blob");
    let user_only_routes = vec![
        (Method::GET, "/runs".to_string()),
        (Method::POST, "/runs".to_string()),
        (Method::GET, "/runs/resolve".to_string()),
        (Method::POST, "/preflight".to_string()),
        (Method::POST, "/validate".to_string()),
        (Method::POST, "/graph/render".to_string()),
        (Method::GET, "/attach".to_string()),
        (Method::DELETE, format!("/runs/{run_id}")),
        (Method::GET, format!("/runs/{run_id}/attach")),
        (Method::GET, format!("/runs/{run_id}/checkpoint")),
        (Method::POST, format!("/runs/{run_id}/pause")),
        (Method::POST, format!("/runs/{run_id}/unpause")),
        (Method::GET, format!("/runs/{run_id}/graph")),
        (Method::GET, format!("/runs/{run_id}/graph/source")),
        (Method::GET, format!("/runs/{run_id}/stages")),
        (Method::GET, format!("/runs/{run_id}/artifacts")),
        (Method::GET, format!("/runs/{run_id}/artifacts/download")),
        (Method::GET, format!("/runs/{run_id}/files")),
        (
            Method::GET,
            format!("/runs/{run_id}/stages/code@2/artifacts"),
        ),
        (
            Method::GET,
            format!("/runs/{run_id}/stages/code@2/artifacts/download"),
        ),
        (Method::GET, format!("/runs/{run_id}/billing")),
        (Method::GET, format!("/runs/{run_id}/settings")),
        (Method::POST, format!("/runs/{run_id}/preview")),
        (Method::POST, format!("/runs/{run_id}/ssh")),
        (Method::GET, format!("/runs/{run_id}/sandbox/files")),
        (Method::GET, format!("/runs/{run_id}/sandbox/services")),
        (Method::GET, format!("/runs/{run_id}/sandbox/file")),
        (Method::PUT, format!("/runs/{run_id}/sandbox/file")),
    ];

    for (method, path) in user_only_routes {
        let response = app
            .clone()
            .oneshot(bearer_request(
                method.clone(),
                &path,
                &worker_token,
                Body::empty(),
            ))
            .await
            .unwrap();
        assert!(
            matches!(
                response.status(),
                StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
            ),
            "{method} {path} unexpectedly accepted worker token with status {}",
            response.status()
        );
    }

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            &format!("/runs/{run_id}/blobs/{blob_hash}"),
            &worker_token,
            Body::empty(),
        ))
        .await
        .unwrap();
    assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
}

#[tokio::test]
async fn stage_artifacts_multipart_round_trip() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT).await;
    let stage_id = "code@2";
    let source_bytes = b"fn main() {}\n";
    let log_bytes = b"build ok\n";
    let manifest = serde_json::json!({
        "entries": [
            {
                "part": "file1",
                "path": "src/lib.rs",
                "sha256": hex::encode(Sha256::digest(source_bytes)),
                "expected_bytes": source_bytes.len(),
                "content_type": "text/plain"
            },
            {
                "part": "file2",
                "path": "logs/output.txt",
                "sha256": hex::encode(Sha256::digest(log_bytes)),
                "expected_bytes": log_bytes.len(),
                "content_type": "text/plain"
            }
        ]
    });
    let boundary = "fabro-test-boundary";

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts?retry=1"
        )))
        .header(
            "content-type",
            format!("multipart/form-data; boundary={boundary}"),
        )
        .body(multipart_body(boundary, &manifest, &[
            ("file1", "src/lib.rs", source_bytes),
            ("file2", "logs/output.txt", log_bytes),
        ]))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/stages/{stage_id}/artifacts")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["data"][0]["filename"], "logs/output.txt");
    assert_eq!(body["data"][0]["retry"], 1);
    assert_eq!(body["data"][0]["size"], log_bytes.len());
    assert_eq!(body["data"][1]["filename"], "src/lib.rs");
    assert_eq!(body["data"][1]["retry"], 1);
    assert_eq!(body["data"][1]["size"], source_bytes.len());

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!(
            "/runs/{run_id}/stages/{stage_id}/artifacts/download?filename=logs/output.txt&retry=1"
        )))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let bytes = response_bytes!(response, StatusCode::OK).await;
    assert_eq!(&bytes[..], log_bytes);
}

#[tokio::test]
async fn stage_artifacts_multipart_requires_manifest_first() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_run(&app, MINIMAL_DOT).await;
    let boundary = "fabro-test-boundary";
    let body = format!(
        "--{boundary}\r\nContent-Disposition: form-data; name=\"file1\"; filename=\"src/lib.rs\"\r\n\r\nfn main() {{}}\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"manifest\"\r\nContent-Type: application/json\r\n\r\n{{\"entries\":[{{\"part\":\"file1\",\"path\":\"src/lib.rs\"}}]}}\r\n--{boundary}--\r\n"
    );

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!(
            "/runs/{run_id}/stages/code@2/artifacts?retry=1"
        )))
        .header(
            "content-type",
            format!("multipart/form-data; boundary={boundary}"),
        )
        .body(Body::from(body))
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[tokio::test]
async fn create_run_returns_submitted() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    assert_eq!(run_json_status(&body)["kind"], "submitted");
    assert_eq!(body["title"], "Test");
}

#[tokio::test]
async fn create_run_accepts_explicit_title() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let mut manifest = minimal_manifest_json(MINIMAL_DOT);
    manifest["title"] = json!("  Explicit server title  ");

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(Body::from(serde_json::to_string(&manifest).unwrap()))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    assert_eq!(body["title"], "Explicit server title");

    let run_id = body["id"].as_str().unwrap();
    let detail_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let detail = response_json!(detail_response, StatusCode::OK).await;
    assert_eq!(detail["title"], "Explicit server title");
}

#[tokio::test]
async fn create_run_rejects_invalid_titles() {
    let app = test_app_with();
    for title in [
        "   ".to_string(),
        "First\nSecond".to_string(),
        "x".repeat(101),
    ] {
        let mut manifest = minimal_manifest_json(MINIMAL_DOT);
        manifest["title"] = json!(title);
        let req = Request::builder()
            .method("POST")
            .uri(api("/runs"))
            .header("content-type", "application/json")
            .body(Body::from(serde_json::to_string(&manifest).unwrap()))
            .unwrap();

        let response = app.clone().oneshot(req).await.unwrap();
        assert_status!(response, StatusCode::BAD_REQUEST).await;
    }
}

#[tokio::test]
async fn start_run_transitions_to_runnable() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Create a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    // Start it
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/start")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "runnable");
    assert_eq!(body["title"], "Test");

    let status = state
        .stores
        .runs
        .open_run_reader(&run_id.parse::<RunId>().unwrap())
        .await
        .unwrap()
        .state()
        .await
        .unwrap()
        .status;
    assert_eq!(status, RunStatus::Runnable);
}

#[tokio::test]
async fn worker_started_child_run_requires_approval_before_becoming_runnable() {
    let (state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let parent_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_run_tools_worker_token(&parent_run_id);
    let mut child_manifest = minimal_manifest_json(MINIMAL_DOT);
    child_manifest["parent_id"] = json!(parent_run_id.to_string());

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            "/runs",
            &worker_token,
            &child_manifest,
        ))
        .await
        .unwrap();
    let child_body = response_json!(response, StatusCode::CREATED).await;
    let child_run_id = child_body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{child_run_id}/start"),
            &worker_token,
            &json!({ "resume": false }),
        ))
        .await
        .unwrap();
    let pending_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        run_json_status(&pending_body),
        &json!({
            "kind": "pending",
            "reason": "approval_required"
        })
    );
    assert_eq!(
        pending_body["lifecycle"]["approval"]["state"].as_str(),
        Some("pending")
    );

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::GET,
            "/system/info",
            &user_jwt,
            Body::empty(),
        ))
        .await
        .unwrap();
    let info_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(info_body["runs"]["active"], 1);
    assert_eq!(info_body["runs"]["scheduler_slots_used"], 0);

    {
        let runs = state.runs.lock().expect("runs lock poisoned");
        assert_eq!(
            runs.get(&child_run_id).map(|run| run.status),
            Some(RunStatus::Pending {
                reason: fabro_types::PendingReason::ApprovalRequired,
            })
        );
    }

    let response = app
        .clone()
        .oneshot(bearer_request(
            Method::POST,
            &format!("/runs/{child_run_id}/approve"),
            &user_jwt,
            Body::empty(),
        ))
        .await
        .unwrap();
    let approved_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        run_json_status(&approved_body),
        &json!({ "kind": "runnable" })
    );
    assert_eq!(
        approved_body["lifecycle"]["approval"]["state"].as_str(),
        Some("approved")
    );
    assert!(
        approved_body["lifecycle"]["approval"]["decided_at"]
            .as_str()
            .is_some()
    );

    let runs = state.runs.lock().expect("runs lock poisoned");
    assert_eq!(
        runs.get(&child_run_id).map(|run| run.status),
        Some(RunStatus::Runnable)
    );
}

#[tokio::test]
async fn denying_pending_child_run_fails_with_approval_denied() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let parent_run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_run_tools_worker_token(&parent_run_id);
    let mut child_manifest = minimal_manifest_json(MINIMAL_DOT);
    child_manifest["parent_id"] = json!(parent_run_id.to_string());

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            "/runs",
            &worker_token,
            &child_manifest,
        ))
        .await
        .unwrap();
    let child_body = response_json!(response, StatusCode::CREATED).await;
    let child_run_id = child_body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{child_run_id}/start"),
            &worker_token,
            &json!({ "resume": false }),
        ))
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(json_bearer_request(
            Method::POST,
            &format!("/runs/{child_run_id}/deny"),
            &user_jwt,
            &json!({ "reason": "  " }),
        ))
        .await
        .unwrap();
    let denied_body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        run_json_status(&denied_body),
        &json!({
            "kind": "failed",
            "reason": "approval_denied"
        })
    );
    assert_eq!(
        denied_body["lifecycle"]["approval"]["state"].as_str(),
        Some("denied")
    );
    assert!(denied_body["lifecycle"]["approval"]["denial_reason"].is_null());
}

#[tokio::test]
async fn patch_run_title_updates_active_and_archived_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    let patch_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("PATCH")
                .uri(api(&format!("/runs/{run_id}")))
                .header("content-type", "application/json")
                .body(Body::from(
                    json!({ "title": "  Active title  " }).to_string(),
                ))
                .unwrap(),
        )
        .await
        .unwrap();
    let patch_body = response_json!(patch_response, StatusCode::OK).await;
    assert_eq!(patch_body["title"], "Active title");

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let event_count = run_store.list_events().await.unwrap().len();
    let same_title_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("PATCH")
                .uri(api(&format!("/runs/{run_id}")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "title": "Active title" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let same_title_body = response_json!(same_title_response, StatusCode::OK).await;
    assert_eq!(same_title_body["title"], "Active title");
    assert_eq!(
        state
            .stores
            .runs
            .open_run_reader(&run_id)
            .await
            .unwrap()
            .list_events()
            .await
            .unwrap()
            .len(),
        event_count,
        "same-title PATCH should not append an event"
    );

    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    for event in [
        workflow_event::Event::RunRunnable {
            source: fabro_types::RunRunnableSource::StartRequested,
            actor:  None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ] {
        workflow_event::append_event(&run_store, &run_id, &event)
            .await
            .unwrap();
    }
    workflow_event::append_event(
        &run_store,
        &run_id,
        &workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    )
    .await
    .unwrap();
    response_json!(
        app.clone()
            .oneshot(
                Request::builder()
                    .method("POST")
                    .uri(api(&format!("/runs/{run_id}/archive")))
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap(),
        StatusCode::OK
    )
    .await;

    let archived_patch_response = app
        .oneshot(
            Request::builder()
                .method("PATCH")
                .uri(api(&format!("/runs/{run_id}")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "title": "Archived title" }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let archived_patch_body = response_json!(archived_patch_response, StatusCode::OK).await;
    assert_eq!(archived_patch_body["title"], "Archived title");
    assert!(run_json_archived(&archived_patch_body));
}

#[tokio::test]
async fn patch_run_title_rejects_invalid_titles() {
    let app = test_app_with();
    let run_id = create_run(&app, MINIMAL_DOT).await;

    for title in [String::new(), "Bad\rTitle".to_string(), "x".repeat(101)] {
        let response = app
            .clone()
            .oneshot(
                Request::builder()
                    .method("PATCH")
                    .uri(api(&format!("/runs/{run_id}")))
                    .header("content-type", "application/json")
                    .body(Body::from(json!({ "title": title }).to_string()))
                    .unwrap(),
            )
            .await
            .unwrap();
        assert_status!(response, StatusCode::BAD_REQUEST).await;
    }
}

#[tokio::test]
async fn start_run_conflict_when_not_submitted() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Create a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    // Start it (transitions to runnable)
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/start")))
        .body(Body::empty())
        .unwrap();
    app.clone().oneshot(req).await.unwrap();

    // Start it again — should 409
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/start")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::CONFLICT).await;
}

#[tokio::test]
async fn resume_cancelled_run_with_checkpoint_transitions_to_runnable() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let checkpoint = Checkpoint::from_context(
        &fabro_workflow::context::Context::new(),
        "start",
        vec!["start".to_string()],
        std::collections::HashMap::new(),
        std::collections::HashMap::new(),
        Some("exit".to_string()),
        std::collections::HashMap::new(),
        std::collections::HashMap::new(),
        std::collections::HashMap::new(),
    );
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunRunnable {
            source: fabro_types::RunRunnableSource::StartRequested,
            actor:  None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::CheckpointCompleted {
            graph_visit: None,
            resumed_from_stage_id: None,
            node_id: checkpoint.current_node.clone(),
            status: "succeeded".to_string(),
            current_node: checkpoint.current_node.clone(),
            completed_nodes: checkpoint.completed_nodes.clone(),
            node_retries: checkpoint.node_retries.clone().into_iter().collect(),
            context_values: checkpoint.context_values.clone().into_iter().collect(),
            node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(),
            next_node_id: checkpoint.next_node_id.clone(),
            git_commit_sha: checkpoint.git_commit_sha.clone(),
            loop_failure_signatures: std::collections::BTreeMap::new(),
            restart_failure_signatures: std::collections::BTreeMap::new(),
            node_visits: std::collections::BTreeMap::new(),
            diff: None,
            diff_summary: None,
        },
        workflow_event::Event::workflow_run_failed_from_error(
            &WorkflowError::Cancelled,
            fabro_types::RunTiming::wall_only(10),
            FailureReason::Cancelled,
            None,
            None,
            None,
            None,
        ),
    ])
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/start")))
                .header("content-type", "application/json")
                .body(Body::from(json!({ "resume": true }).to_string()))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "runnable");
    assert_eq!(body["timestamps"]["completed_at"], serde_json::Value::Null);
    assert_eq!(body["timing"], serde_json::Value::Null);

    let state = state
        .stores
        .runs
        .open_run_reader(&run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    assert_eq!(state.status, RunStatus::Runnable);
    assert!(state.conclusion.is_none());
}

#[tokio::test]
async fn retry_failed_run_creates_and_queues_new_run() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let source_run_id = RunId::new();
    create_durable_run_with_events(&state, source_run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::workflow_run_failed_from_error(
            &WorkflowError::engine("boom"),
            fabro_types::RunTiming::wall_only(10),
            FailureReason::WorkflowError,
            None,
            None,
            None,
            None,
        ),
    ])
    .await;
    let source_events_before = state
        .stores
        .runs
        .open_run(&source_run_id)
        .await
        .unwrap()
        .list_events()
        .await
        .unwrap()
        .len();

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{source_run_id}/retry")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let new_run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    assert_ne!(new_run_id, source_run_id);
    assert_eq!(body["retried_from"], source_run_id.to_string());
    assert_eq!(body["created_by"]["kind"], "user");
    assert_eq!(body["created_by"]["login"], "dev");
    assert_eq!(run_json_status(&body)["kind"], "runnable");

    let source_store = state.stores.runs.open_run(&source_run_id).await.unwrap();
    assert_eq!(
        source_store.list_events().await.unwrap().len(),
        source_events_before
    );
    assert_eq!(
        source_store.state().await.unwrap().status,
        RunStatus::Failed {
            reason: FailureReason::WorkflowError,
        }
    );

    let new_state = state
        .stores
        .runs
        .open_run(&new_run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    assert_eq!(new_state.retried_from, Some(source_run_id));
    assert_eq!(new_state.status, RunStatus::Runnable);
    assert!(new_state.checkpoints.is_empty());
}

#[tokio::test]
async fn retry_missing_run_returns_not_found() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{}/retry", fixtures::RUN_64)))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn retry_succeeded_run_creates_and_queues_new_run() {
    let state = test_app_state_with_isolated_storage();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let source_run_id = RunId::new();
    create_durable_run_with_events(&state, source_run_id, &[
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(10),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;
    let source_events_before = state
        .stores
        .runs
        .open_run(&source_run_id)
        .await
        .unwrap()
        .list_events()
        .await
        .unwrap()
        .len();

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{source_run_id}/retry")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let new_run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    assert_ne!(new_run_id, source_run_id);
    assert_eq!(body["retried_from"], source_run_id.to_string());
    assert_eq!(run_json_status(&body)["kind"], "runnable");

    let source_store = state.stores.runs.open_run(&source_run_id).await.unwrap();
    assert_eq!(
        source_store.list_events().await.unwrap().len(),
        source_events_before
    );
    assert_eq!(
        source_store.state().await.unwrap().status,
        RunStatus::Succeeded {
            reason: SuccessReason::Completed,
        }
    );

    let new_state = state
        .stores
        .runs
        .open_run(&new_run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    assert_eq!(new_state.retried_from, Some(source_run_id));
    assert_eq!(new_state.status, RunStatus::Runnable);
}

#[tokio::test]
async fn retry_active_run_returns_conflict() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let source_run_id = RunId::new();
    create_durable_run_with_events(&state, source_run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
    ])
    .await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{source_run_id}/retry")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::CONFLICT).await;
}

#[tokio::test]
async fn cancel_run_succeeds() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    // Cancel it
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    // Could be OK (cancelled) or CONFLICT (already completed)
    let status = response.status();
    assert!(
        status == StatusCode::OK || status == StatusCode::CONFLICT,
        "unexpected status: {status}"
    );
}

#[tokio::test]
async fn cancel_nonexistent_run_returns_not_found() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{missing_run_id}/cancel")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn cancel_terminal_durable_run_returns_conflict() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CONFLICT).await;
    assert_eq!(
        body["errors"][0]["detail"],
        "Run is already terminal and cannot be cancelled."
    );
}

#[tokio::test]
async fn steer_nonexistent_run_returns_not_found() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{missing_run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again"}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn steer_terminal_durable_run_returns_run_not_steerable() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again"}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CONFLICT).await;
    assert_eq!(body["errors"][0]["code"], "run_not_steerable");
    assert_eq!(body["errors"][0]["detail"], "Run is no longer steerable.");
}

#[tokio::test]
async fn steer_empty_text_returns_bad_request() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"   "}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    // 400 (whitespace-only text) or 409 (run not yet `running` when the
    // handler checks status) are both acceptable; the only outcome we
    // want to rule out is a successful enqueue.
    let status = response.status();
    assert!(
        matches!(status, StatusCode::BAD_REQUEST | StatusCode::CONFLICT),
        "expected 400 or 409, got {status}"
    );
}

fn insert_running_control_run(
    state: &Arc<AppState>,
    run_id: RunId,
    answer_transport: Option<RunAnswerTransport>,
) -> tempfile::TempDir {
    let temp_dir = tempfile::tempdir().unwrap();
    let mut run = managed_run(
        String::new(),
        RunStatus::Running,
        chrono::Utc::now(),
        temp_dir.path().join(run_id.to_string()),
        RunExecutionMode::Start,
    );
    run.answer_transport = answer_transport;
    state
        .runs
        .lock()
        .expect("runs lock poisoned")
        .insert(run_id, run);
    temp_dir
}

#[tokio::test]
async fn steer_without_active_steerable_session_forwards_plain_steer_for_buffering() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again"}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;
    let envelope = recv_worker_control_envelope(&mut control_rx).await;
    assert!(matches!(
        envelope.message,
        WorkerControlMessage::Steer { ref text, .. } if text == "try again"
    ));
}

#[tokio::test]
async fn steer_with_active_non_steerable_session_returns_conflict() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let stage_id = StageId::new("agent", 1);
    let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.get_mut(&run_id)
            .unwrap()
            .active_non_steerable_stages
            .insert(stage_id, "session-a".to_string());
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again"}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_eq!(response.status(), StatusCode::CONFLICT);
    let body = body_json(response.into_body()).await;
    assert_eq!(body["errors"][0]["code"], "agent_not_steerable");
}

#[tokio::test]
async fn steer_interrupt_without_active_steerable_session_returns_conflict() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again","interrupt":true}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_eq!(response.status(), StatusCode::CONFLICT);
    let body = body_json(response.into_body()).await;
    assert_eq!(body["errors"][0]["code"], "no_active_steerable_session");
}

#[tokio::test]
async fn interrupt_with_active_steerable_session_forwards_interrupt() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let stage_id = StageId::new("agent", 1);
    let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.get_mut(&run_id)
            .unwrap()
            .active_steerable_stages
            .insert(stage_id, "session-a".to_string());
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/interrupt")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;
    let envelope = recv_worker_control_envelope(&mut control_rx).await;
    assert!(matches!(
        envelope.message,
        WorkerControlMessage::Interrupt {
            actor: Principal::User(_),
        }
    ));
}

#[tokio::test]
async fn steer_interrupt_with_active_steerable_session_forwards_combined_control_message() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let stage_id = StageId::new("agent", 1);
    let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.get_mut(&run_id)
            .unwrap()
            .active_steerable_stages
            .insert(stage_id, "session-a".to_string());
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again","interrupt":true}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;
    let envelope = recv_worker_control_envelope(&mut control_rx).await;
    assert!(matches!(
        envelope.message,
        WorkerControlMessage::InterruptThenSteer { ref text, .. } if text == "try again"
    ));
}

#[tokio::test]
async fn interrupt_terminal_run_returns_run_not_interruptible() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let temp_dir = tempfile::tempdir().unwrap();
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.insert(
            run_id,
            managed_run(
                String::new(),
                RunStatus::Succeeded {
                    reason: SuccessReason::Completed,
                },
                chrono::Utc::now(),
                temp_dir.path().join(run_id.to_string()),
                RunExecutionMode::Start,
            ),
        );
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/interrupt")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_eq!(response.status(), StatusCode::CONFLICT);
    let body = body_json(response.into_body()).await;
    assert_eq!(body["errors"][0]["code"], "run_not_interruptible");
}

#[test]
fn injected_runnable_event_does_not_make_submitted_run_schedulable() {
    let state = test_app_state();
    let run_id = fixtures::RUN_1;
    let temp_dir = tempfile::tempdir().unwrap();
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.insert(
            run_id,
            managed_run(
                String::new(),
                RunStatus::Submitted,
                chrono::Utc::now(),
                temp_dir.path().join(run_id.to_string()),
                RunExecutionMode::Start,
            ),
        );
    }

    let runnable = workflow_event::to_run_event(&run_id, &workflow_event::Event::RunRunnable {
        source: fabro_types::RunRunnableSource::StartRequested,
        actor:  None,
    });
    update_live_run_from_event(&state, run_id, &runnable);

    {
        let runs = state.runs.lock().expect("runs lock poisoned");
        assert_eq!(runs.get(&run_id).unwrap().status, RunStatus::Submitted);
    }

    let starting = workflow_event::to_run_event(&run_id, &workflow_event::Event::RunStarting);
    update_live_run_from_event(&state, run_id, &starting);

    let runs = state.runs.lock().expect("runs lock poisoned");
    assert_eq!(runs.get(&run_id).unwrap().status, RunStatus::Starting);
}

#[test]
fn active_steerable_stage_projection_ignores_stale_deactivation() {
    let state = test_app_state();
    let run_id = fixtures::RUN_1;
    let temp_dir = tempfile::tempdir().unwrap();
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        runs.insert(
            run_id,
            managed_run(
                String::new(),
                RunStatus::Running,
                chrono::Utc::now(),
                temp_dir.path().join(run_id.to_string()),
                RunExecutionMode::Start,
            ),
        );
    }

    let stage_id = StageId::new("agent", 1);
    let activated_a =
        workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated {
            node_id:          "agent".to_string(),
            visit:            1,
            session_id:       "session-a".to_string(),
            thread_id:        None,
            provider:         Some("openai".to_string()),
            model:            Some("gpt-5.4".to_string()),
            reasoning_effort: None,
            speed:            None,
            permission_level: None,
            capabilities:     vec![SessionCapability::Steer],
        });
    update_live_run_from_event(&state, run_id, &activated_a);

    let deactivated_a =
        workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionDeactivated {
            node_id:    "agent".to_string(),
            visit:      1,
            session_id: "session-a".to_string(),
        });
    update_live_run_from_event(&state, run_id, &deactivated_a);

    let activated_b =
        workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated {
            node_id:          "agent".to_string(),
            visit:            1,
            session_id:       "session-b".to_string(),
            thread_id:        None,
            provider:         Some("openai".to_string()),
            model:            Some("gpt-5.4".to_string()),
            reasoning_effort: None,
            speed:            None,
            permission_level: None,
            capabilities:     vec![SessionCapability::Steer],
        });
    update_live_run_from_event(&state, run_id, &activated_b);
    update_live_run_from_event(&state, run_id, &deactivated_a);

    let runs = state.runs.lock().expect("runs lock poisoned");
    let run = runs.get(&run_id).unwrap();
    assert_eq!(
        run.active_steerable_stages
            .get(&stage_id)
            .map(String::as_str),
        Some("session-b")
    );
}

fn acp_event_for_stage(run_id: &RunId, event: &workflow_event::Event) -> fabro_types::RunEvent {
    workflow_event::to_run_event_at(
        run_id,
        event,
        Utc::now(),
        Some(&workflow_event::StageScope {
            node_id:            "agent".to_string(),
            visit:              1,
            parallel_group_id:  None,
            parallel_branch_id: None,
        }),
    )
}

#[tokio::test]
async fn steer_with_active_acp_session_forwards_to_worker() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await;
    let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));

    let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted {
        node_id:     "agent".to_string(),
        visit:       1,
        command:     "python fake_agent.py".to_string(),
        config_name: None,
    });
    update_live_run_from_event(&state, run_id, &started);
    let activated =
        workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated {
            node_id:          "agent".to_string(),
            visit:            1,
            session_id:       "acp-session".to_string(),
            thread_id:        None,
            provider:         Some(AgentBackend::Acp.to_string()),
            model:            None,
            reasoning_effort: None,
            speed:            None,
            permission_level: None,
            capabilities:     vec![SessionCapability::Steer],
        });
    update_live_run_from_event(&state, run_id, &activated);

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/steer")))
        .header("content-type", "application/json")
        .body(Body::from(r#"{"text":"try again"}"#))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;
    let envelope = recv_worker_control_envelope(&mut control_rx).await;
    assert!(matches!(
        envelope.message,
        WorkerControlMessage::Steer { ref text, .. } if text == "try again"
    ));
}

#[tokio::test]
async fn active_acp_steerable_marker_clears_on_terminal_paths() {
    let terminal_events: Vec<workflow_event::Event> = vec![
        workflow_event::Event::AgentAcpCompleted {
            node_id:     "agent".to_string(),
            stdout:      "done".to_string(),
            stderr:      String::new(),
            stop_reason: "end_turn".to_string(),
            duration_ms: 42,
        },
        workflow_event::Event::AgentAcpCancelled {
            node_id:     "agent".to_string(),
            stdout:      "partial".to_string(),
            stderr:      "cancelled".to_string(),
            duration_ms: 7,
        },
        workflow_event::Event::AgentAcpTimedOut {
            node_id:     "agent".to_string(),
            stdout:      "partial".to_string(),
            stderr:      "timeout".to_string(),
            duration_ms: 99,
        },
        workflow_event::Event::StageCompleted {
            node_id: "agent".to_string(),
            name: "agent".to_string(),
            index: 0,
            timing: fabro_types::StageTiming::wall_only(1),
            status: "success".to_string(),
            preferred_label: None,
            suggested_next_ids: Vec::new(),
            billing: None,
            failure: None,
            notes: None,
            files_touched: Vec::new(),
            context_updates: None,
            jump_to_node: None,
            context_values: None,
            node_visits: None,
            loop_failure_signatures: None,
            restart_failure_signatures: None,
            response: None,
            attempt: 1,
            max_attempts: 1,
        },
        workflow_event::Event::StageFailed {
            node_id:    "agent".to_string(),
            name:       "agent".to_string(),
            index:      0,
            failure:    FailureDetail::new("failed", FailureCategory::Deterministic),
            will_retry: false,
            timing:     fabro_types::StageTiming::wall_only(1),
            billing:    None,
            actor:      None,
        },
    ];

    for terminal_event in terminal_events {
        let state = test_app_state();
        let app = crate::test_support::build_test_router(Arc::clone(&state));
        let run_id = fixtures::RUN_1;
        let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;
        let _temp_dir = insert_running_control_run(&state, run_id, Some(transport));
        let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted {
            node_id:     "agent".to_string(),
            visit:       1,
            command:     "python fake_agent.py".to_string(),
            config_name: None,
        });
        update_live_run_from_event(&state, run_id, &started);
        let activated =
            workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated {
                node_id:          "agent".to_string(),
                visit:            1,
                session_id:       "acp-session".to_string(),
                thread_id:        None,
                provider:         Some(AgentBackend::Acp.to_string()),
                model:            None,
                reasoning_effort: None,
                speed:            None,
                permission_level: None,
                capabilities:     vec![SessionCapability::Steer],
            });
        update_live_run_from_event(&state, run_id, &activated);
        let terminal = acp_event_for_stage(&run_id, &terminal_event);
        update_live_run_from_event(&state, run_id, &terminal);

        let req = Request::builder()
            .method("POST")
            .uri(api(&format!("/runs/{run_id}/interrupt")))
            .body(Body::empty())
            .unwrap();

        let response = app.oneshot(req).await.unwrap();
        assert_eq!(response.status(), StatusCode::CONFLICT);
        let body = body_json(response.into_body()).await;
        assert_eq!(body["errors"][0]["code"], "no_active_steerable_session");
    }
}

#[tokio::test]
async fn get_graph_returns_svg() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Start a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "version": 1,
                "cwd": "/tmp",
                "target": {
                    "path": "workflow.fabro",
                },
                "workflows": {
                    "workflow.fabro": {
                        "source": MINIMAL_DOT,
                        "files": {},
                    },
                },
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    // Request graph SVG
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/graph")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();

    let response = checked_response!(response, StatusCode::OK).await;

    let content_type = response
        .headers()
        .get("content-type")
        .expect("content-type header should be present")
        .to_str()
        .unwrap();
    assert_eq!(content_type, "image/svg+xml");

    let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
    let svg = String::from_utf8_lossy(&bytes);
    assert!(
        svg.contains("<?xml") || svg.contains("<svg"),
        "expected SVG content, got: {}",
        &svg[..svg.len().min(200)]
    );
}

#[tokio::test]
async fn get_graph_source_returns_dot() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "version": 1,
                "cwd": "/tmp",
                "target": {
                    "path": "workflow.fabro",
                },
                "workflows": {
                    "workflow.fabro": {
                        "source": MINIMAL_DOT,
                        "files": {},
                    },
                },
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/graph/source")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let response = checked_response!(response, StatusCode::OK).await;

    let content_type = response
        .headers()
        .get("content-type")
        .expect("content-type header should be present")
        .to_str()
        .unwrap();
    assert_eq!(content_type, "text/vnd.graphviz");

    let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
    let dot = String::from_utf8(bytes.to_vec()).unwrap();
    assert_eq!(dot, MINIMAL_DOT);
}

#[tokio::test]
async fn render_graph_from_manifest_returns_svg() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/graph/render"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "manifest": {
                    "version": 1,
                    "cwd": "/tmp",
                    "target": {
                        "path": "workflow.fabro",
                    },
                    "workflows": {
                        "workflow.fabro": {
                            "source": MINIMAL_DOT,
                            "files": {},
                        },
                    },
                },
                "format": "svg",
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();

    let response = checked_response!(response, StatusCode::OK).await;
    assert_eq!(
        response
            .headers()
            .get("content-type")
            .expect("content-type header should be present")
            .to_str()
            .unwrap(),
        "image/svg+xml"
    );

    let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
    let svg = String::from_utf8_lossy(&bytes);
    assert!(
        svg.contains("<?xml") || svg.contains("<svg"),
        "expected SVG content, got: {}",
        &svg[..svg.len().min(200)]
    );
}

#[tokio::test]
async fn render_graph_from_manifest_accepts_fabro_dotted_attributes() {
    let app = test_app_with();
    let dot_source = r#"digraph X {
  start [shape=Mdiamond]
  exit [shape=Msquare]
  a [label="A", acp.command="codex"]
  start -> a -> exit
}"#;

    let req = Request::builder()
        .method("POST")
        .uri(api("/graph/render"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({
                "manifest": {
                    "version": 1,
                    "cwd": "/tmp",
                    "target": {
                        "path": "workflow.fabro",
                    },
                    "workflows": {
                        "workflow.fabro": {
                            "source": dot_source,
                            "files": {},
                        },
                    },
                },
                "format": "svg",
            }))
            .unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();

    let response = checked_response!(response, StatusCode::OK).await;
    assert_eq!(
        response
            .headers()
            .get("content-type")
            .expect("content-type header should be present")
            .to_str()
            .unwrap(),
        "image/svg+xml"
    );
}

#[cfg(unix)]
#[tokio::test]
async fn render_graph_bytes_returns_bad_request_for_render_error_protocol() {
    let (_dir, script_path) = write_test_executable(
        "#!/bin/sh\ncat >/dev/null\nprintf 'RENDER_ERROR:failed to parse DOT source'\nexit 0\n",
    );

    let response =
        render_graph_bytes_with_exe_override("not valid dot {{{", Some(&script_path)).await;

    assert_status!(response, StatusCode::BAD_REQUEST).await;
}

#[cfg(unix)]
fn write_test_executable(script: &str) -> (tempfile::TempDir, PathBuf) {
    let dir = tempfile::tempdir().expect("temp dir should exist");
    let path = dir.path().join("fake-fabro");
    std::fs::write(&path, script).expect("script should be written");
    std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))
        .expect("script should be executable");
    (dir, path)
}

#[cfg(unix)]
async fn render_graph_with_override(dot_source: &str, exe_path: &Path) -> Response {
    render_graph_bytes_with_exe_override(dot_source, Some(exe_path)).await
}

#[cfg(unix)]
#[tokio::test]
async fn render_dot_subprocess_returns_child_crashed_for_nonzero_exit() {
    let (_dir, script_path) = write_test_executable("#!/bin/sh\nexit 1\n");

    let result = render_dot_subprocess("digraph { a -> b }", Some(&script_path)).await;

    assert!(matches!(
        result,
        Err(RenderSubprocessError::ChildCrashed(_))
    ));
}

#[cfg(unix)]
#[tokio::test]
async fn render_graph_bytes_returns_internal_server_error_for_child_crash() {
    let (_dir, script_path) = write_test_executable("#!/bin/sh\nexit 1\n");

    let response = render_graph_with_override("digraph { a -> b }", &script_path).await;

    assert_status!(response, StatusCode::INTERNAL_SERVER_ERROR).await;
}

#[cfg(unix)]
#[tokio::test]
async fn render_dot_subprocess_returns_protocol_violation_for_garbage_stdout() {
    let (_dir, script_path) =
        write_test_executable("#!/bin/sh\ncat >/dev/null\nprintf 'garbage'\nexit 0\n");

    let result = render_dot_subprocess("digraph { a -> b }", Some(&script_path)).await;

    assert!(matches!(
        result,
        Err(RenderSubprocessError::ProtocolViolation(_))
    ));
}

#[tokio::test]
async fn get_graph_not_found() {
    let app = test_app_with();
    let missing_run_id = fixtures::RUN_64;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{missing_run_id}/graph")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn list_runs_returns_started_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // List should be empty initially
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["data"].as_array().unwrap().len(), 0);
    assert_eq!(body["meta"]["has_more"].as_bool(), Some(false));

    // Start a run
    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    // List should now contain one run
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let items = body["data"].as_array().unwrap();
    assert_eq!(items.len(), 1);
    assert_eq!(run_json_id(&items[0]).unwrap(), run_id.to_string());
    assert!(items[0]["goal"].is_string());
    assert!(items[0]["title"].is_string());
    assert!(items[0]["repository"]["name"].is_string());
    assert!(items[0]["timestamps"]["created_at"].is_string());
    assert!(run_json_status(&items[0]).is_object());
    assert!(items[0]["labels"].is_object());
    assert!(run_json_pending_control(&items[0]).is_null());
    assert!(items[0]["billing"].is_null());
}

#[tokio::test]
async fn archive_and_unarchive_updates_listing_visibility() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let archive_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/archive")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let archive_body = response_json!(archive_response, StatusCode::OK).await;
    assert!(run_json_archived(&archive_body));
    assert_eq!(run_json_status(&archive_body)["kind"], "succeeded");
    assert_eq!(run_json_status(&archive_body)["reason"], "completed");

    let hidden_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let hidden_body = response_json!(hidden_response, StatusCode::OK).await;
    assert!(
        !hidden_body["data"]
            .as_array()
            .unwrap()
            .iter()
            .any(|item| run_json_id(item) == Some(&run_id.to_string())),
        "archived run should be hidden from default listing"
    );

    let visible_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs?include_archived=true"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let visible_body = response_json!(visible_response, StatusCode::OK).await;
    let archived_item = visible_body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|item| run_json_id(item) == Some(&run_id.to_string()))
        .expect("archived run should appear when include_archived=true");
    assert!(run_json_archived(archived_item));
    assert_eq!(run_json_status(archived_item)["kind"], "succeeded");
    assert_eq!(run_json_status(archived_item)["reason"], "completed");

    let unarchive_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/unarchive")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let unarchive_body = response_json!(unarchive_response, StatusCode::OK).await;
    assert!(!run_json_archived(&unarchive_body));
    assert_eq!(run_json_status(&unarchive_body)["kind"], "succeeded");
    assert_eq!(run_json_status(&unarchive_body)["reason"], "completed");

    let restored_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let restored_body = response_json!(restored_response, StatusCode::OK).await;
    let restored_item = restored_body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|item| run_json_id(item) == Some(&run_id.to_string()))
        .expect("unarchived run should reappear in default listing");
    assert_eq!(run_json_status(restored_item)["kind"], "succeeded");
    assert_eq!(run_json_status(restored_item)["reason"], "completed");
}

fn run_submitted_event() -> workflow_event::Event {
    workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }
}

fn workflow_completed_event() -> workflow_event::Event {
    workflow_event::Event::WorkflowRunCompleted {
        timing:               fabro_types::RunTiming::wall_only(1000),
        artifact_count:       0,
        status:               "succeeded".to_string(),
        reason:               SuccessReason::Completed,
        total_usd_micros:     None,
        final_git_commit_sha: None,
        final_patch:          None,
        diff_summary:         None,
        billing:              None,
    }
}

async fn create_succeeded_run(state: &Arc<AppState>, run_id: RunId) {
    create_durable_run_with_events(state, run_id, &[
        run_submitted_event(),
        workflow_completed_event(),
    ])
    .await;
}

async fn create_running_run(state: &Arc<AppState>, run_id: RunId) {
    create_durable_run_with_events(state, run_id, &[
        run_submitted_event(),
        workflow_event::Event::RunRunning,
    ])
    .await;
}

async fn create_preserved_local_sandbox_run(state: &Arc<AppState>, run_id: RunId) {
    let mut settings = fabro_types::WorkflowSettings::default();
    settings.run.environment.lifecycle.preserve = true;
    let graph = Graph::new("test");

    create_durable_run_with_events(state, run_id, &[
        workflow_event::Event::RunCreated {
            run_id,
            title: None,
            settings: serde_json::to_value(settings).unwrap(),
            graph: serde_json::to_value(graph).unwrap(),
            workflow_source: None,
            labels: std::collections::BTreeMap::default(),
            source_directory: Some("/tmp/fabro-run".to_string()),
            workflow_slug: Some("test".to_string()),
            workflow_version_id: None,
            automation: None,
            provenance: test_support::test_run_provenance(),
            manifest_blob: None,
            spec_blob: None,
            git: None,
            fork_source_ref: None,
            retried_from: None,
            parent_id: None,
            web_url: None,
        },
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::SandboxInitialized {
            provider:          SandboxProviderKind::Local,
            id:                "sandbox-preserve-1".to_string(),
            working_directory: "/tmp/fabro-preserved-sandbox".to_string(),
            image:             None,
            snapshot:          None,
            repo_cloned:       None,
            clone_origin_url:  None,
            clone_branch:      None,
            workspace_root:    None,
            repos_root:        None,
            primary_repo_path: None,
            primary_repo_link: None,
        },
    ])
    .await;
}

fn batch_lifecycle_body(run_ids: &[RunId]) -> serde_json::Value {
    json!({
        "run_ids": run_ids.iter().map(ToString::to_string).collect::<Vec<_>>(),
    })
}

fn batch_delete_body(run_ids: &[RunId], force: bool) -> serde_json::Value {
    json!({
        "run_ids": run_ids.iter().map(ToString::to_string).collect::<Vec<_>>(),
        "force": force,
    })
}

fn assert_batch_result(result: &serde_json::Value, run_id: RunId, ok: bool, outcome: &str) {
    assert_eq!(result["run_id"], run_id.to_string());
    assert_eq!(result["ok"], ok);
    assert_eq!(result["outcome"], outcome);
    if ok {
        assert!(
            result["run"].is_object(),
            "successful result should include run: {result}"
        );
        assert!(
            result["error"].is_null(),
            "successful result should omit error: {result}"
        );
    } else {
        assert!(
            result["error"].is_object(),
            "failed result should include error: {result}"
        );
        assert!(
            result["run"].is_null(),
            "failed result should omit run: {result}"
        );
    }
}

fn assert_batch_delete_result(result: &serde_json::Value, run_id: RunId, ok: bool, outcome: &str) {
    assert_eq!(result["run_id"], run_id.to_string());
    assert_eq!(result["ok"], ok);
    assert_eq!(result["outcome"], outcome);
    if ok {
        assert!(
            result["error"].is_null(),
            "successful delete result should omit error: {result}"
        );
    } else {
        assert!(
            result["error"].is_object(),
            "failed delete result should include error: {result}"
        );
    }
}

#[tokio::test]
async fn batch_archive_and_unarchive_updates_listing_visibility() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let first_id = RunId::new();
    let second_id = RunId::new();
    create_succeeded_run(&state, first_id).await;
    create_succeeded_run(&state, second_id).await;

    let archive_response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/archive",
            &batch_lifecycle_body(&[first_id, second_id]),
        ))
        .await
        .unwrap();
    let archive_body = response_json!(archive_response, StatusCode::OK).await;
    assert_eq!(archive_body["summary"]["requested"], 2);
    assert_eq!(archive_body["summary"]["succeeded"], 2);
    assert_eq!(archive_body["summary"]["failed"], 0);
    let archive_results = archive_body["results"].as_array().unwrap();
    assert_batch_result(&archive_results[0], first_id, true, "archived");
    assert!(run_json_archived(&archive_results[0]["run"]));
    assert_batch_result(&archive_results[1], second_id, true, "archived");
    assert!(run_json_archived(&archive_results[1]["run"]));

    let hidden_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let hidden_body = response_json!(hidden_response, StatusCode::OK).await;
    assert!(
        hidden_body["data"].as_array().unwrap().iter().all(|item| {
            let item_id = run_json_id(item);
            item_id != Some(&first_id.to_string()) && item_id != Some(&second_id.to_string())
        }),
        "archived runs should be hidden from default listing"
    );

    let unarchive_response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/unarchive",
            &batch_lifecycle_body(&[first_id, second_id]),
        ))
        .await
        .unwrap();
    let unarchive_body = response_json!(unarchive_response, StatusCode::OK).await;
    assert_eq!(unarchive_body["summary"]["requested"], 2);
    assert_eq!(unarchive_body["summary"]["succeeded"], 2);
    assert_eq!(unarchive_body["summary"]["failed"], 0);
    let unarchive_results = unarchive_body["results"].as_array().unwrap();
    assert_batch_result(&unarchive_results[0], first_id, true, "unarchived");
    assert!(!run_json_archived(&unarchive_results[0]["run"]));
    assert_batch_result(&unarchive_results[1], second_id, true, "unarchived");
    assert!(!run_json_archived(&unarchive_results[1]["run"]));

    let restored_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/runs"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let restored_body = response_json!(restored_response, StatusCode::OK).await;
    for run_id in [first_id, second_id] {
        let restored_item = restored_body["data"]
            .as_array()
            .unwrap()
            .iter()
            .find(|item| run_json_id(item) == Some(&run_id.to_string()))
            .expect("unarchived run should reappear in default listing");
        assert_eq!(run_json_status(restored_item)["kind"], "succeeded");
    }
}

#[tokio::test]
async fn batch_archive_reports_ordered_mixed_results_without_rollback() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let already_archived_id = RunId::new();
    let terminal_id = RunId::new();
    let running_id = RunId::new();
    let missing_id = RunId::new();
    create_succeeded_run(&state, already_archived_id).await;
    create_succeeded_run(&state, terminal_id).await;
    create_running_run(&state, running_id).await;

    let already_archived_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{already_archived_id}/archive")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(already_archived_response, StatusCode::OK).await;

    let response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/archive",
            &batch_lifecycle_body(&[already_archived_id, terminal_id, running_id, missing_id]),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 4);
    assert_eq!(body["summary"]["succeeded"], 2);
    assert_eq!(body["summary"]["failed"], 2);
    let results = body["results"].as_array().unwrap();
    assert_batch_result(&results[0], already_archived_id, true, "already_archived");
    assert_batch_result(&results[1], terminal_id, true, "archived");
    assert_batch_result(&results[2], running_id, false, "conflict");
    assert_eq!(results[2]["error"]["status"], "409");
    assert_batch_result(&results[3], missing_id, false, "not_found");
    assert_eq!(results[3]["error"]["status"], "404");

    let terminal_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{terminal_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let terminal_body = response_json!(terminal_response, StatusCode::OK).await;
    assert!(run_json_archived(&terminal_body));
}

#[tokio::test]
async fn batch_unarchive_treats_terminal_not_archived_as_success() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let archived_id = RunId::new();
    let not_archived_id = RunId::new();
    create_succeeded_run(&state, archived_id).await;
    create_succeeded_run(&state, not_archived_id).await;

    let archive_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{archived_id}/archive")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(archive_response, StatusCode::OK).await;

    let response = app
        .oneshot(json_request(
            Method::POST,
            "/runs/unarchive",
            &batch_lifecycle_body(&[archived_id, not_archived_id]),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 2);
    assert_eq!(body["summary"]["succeeded"], 2);
    assert_eq!(body["summary"]["failed"], 0);
    let results = body["results"].as_array().unwrap();
    assert_batch_result(&results[0], archived_id, true, "unarchived");
    assert_batch_result(&results[1], not_archived_id, true, "not_archived");
}

#[tokio::test]
async fn batch_lifecycle_rejects_invalid_requests_before_mutating_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_succeeded_run(&state, run_id).await;
    let too_many_ids = (0..251)
        .map(|_| RunId::new().to_string())
        .collect::<Vec<_>>();
    let invalid_requests = [
        json!({ "run_ids": [] }),
        json!({ "run_ids": [run_id.to_string(), run_id.to_string()] }),
        json!({ "run_ids": ["not-a-run-id"] }),
        json!({ "run_ids": too_many_ids }),
    ];

    for body in invalid_requests {
        let response = app
            .clone()
            .oneshot(json_request(Method::POST, "/runs/archive", &body))
            .await
            .unwrap();
        assert_status!(response, StatusCode::BAD_REQUEST).await;
    }

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert!(!run_json_archived(&body));
}

#[tokio::test]
async fn batch_lifecycle_requires_user_authentication() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let body = batch_lifecycle_body(&[run_id]);

    let unauthenticated = app
        .clone()
        .oneshot(json_request(Method::POST, "/runs/archive", &body))
        .await
        .unwrap();
    assert_status!(unauthenticated, StatusCode::UNAUTHORIZED).await;

    for path in ["/runs/archive", "/runs/unarchive"] {
        let worker_response = app
            .clone()
            .oneshot(json_bearer_request(
                Method::POST,
                path,
                &worker_token,
                &body,
            ))
            .await
            .unwrap();
        assert!(
            matches!(
                worker_response.status(),
                StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
            ),
            "{path} unexpectedly accepted worker token with status {}",
            worker_response.status()
        );
    }
}

#[tokio::test]
async fn batch_delete_removes_runs_and_reports_ordered_results() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let first_id = RunId::new();
    let second_id = RunId::new();
    create_succeeded_run(&state, first_id).await;
    create_succeeded_run(&state, second_id).await;

    let response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/delete",
            &batch_delete_body(&[first_id, second_id], false),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 2);
    assert_eq!(body["summary"]["succeeded"], 2);
    assert_eq!(body["summary"]["failed"], 0);
    let results = body["results"].as_array().unwrap();
    assert_batch_delete_result(&results[0], first_id, true, "deleted");
    assert_batch_delete_result(&results[1], second_id, true, "deleted");

    for run_id in [first_id, second_id] {
        let response = app
            .clone()
            .oneshot(
                Request::builder()
                    .method("GET")
                    .uri(api(&format!("/runs/{run_id}")))
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap();
        assert_status!(response, StatusCode::NOT_FOUND).await;
    }
}

#[tokio::test]
async fn batch_delete_reports_mixed_results_without_rollback() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let terminal_id = RunId::new();
    let running_id = RunId::new();
    let missing_id = RunId::new();
    create_succeeded_run(&state, terminal_id).await;
    create_running_run(&state, running_id).await;

    let response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/delete",
            &batch_delete_body(&[terminal_id, running_id, missing_id], false),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 3);
    assert_eq!(body["summary"]["succeeded"], 2);
    assert_eq!(body["summary"]["failed"], 1);
    let results = body["results"].as_array().unwrap();
    assert_batch_delete_result(&results[0], terminal_id, true, "deleted");
    assert_batch_delete_result(&results[1], running_id, false, "conflict");
    assert_eq!(results[1]["error"]["status"], "409");
    assert_batch_delete_result(&results[2], missing_id, true, "already_absent");

    let deleted_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{terminal_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(deleted_response, StatusCode::NOT_FOUND).await;

    let running_response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{running_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(running_response, StatusCode::OK).await;
}

#[tokio::test]
async fn batch_delete_force_removes_active_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_running_run(&state, run_id).await;

    let response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/delete",
            &batch_delete_body(&[run_id], true),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 1);
    assert_eq!(body["summary"]["succeeded"], 1);
    assert_eq!(body["summary"]["failed"], 0);
    let results = body["results"].as_array().unwrap();
    assert_batch_delete_result(&results[0], run_id, true, "deleted");

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn batch_delete_with_preserved_sandbox_returns_handoff() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_preserved_local_sandbox_run(&state, run_id).await;

    let response = app
        .clone()
        .oneshot(json_request(
            Method::POST,
            "/runs/delete",
            &batch_delete_body(&[run_id], true),
        ))
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["summary"]["requested"], 1);
    assert_eq!(body["summary"]["succeeded"], 1);
    assert_eq!(body["summary"]["failed"], 0);
    let results = body["results"].as_array().unwrap();
    assert_batch_delete_result(&results[0], run_id, true, "sandbox_preserved");
    assert_eq!(results[0]["sandbox"]["provider"], "local");
    assert_eq!(results[0]["sandbox"]["id"], "sandbox-preserve-1");

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn batch_delete_rejects_invalid_requests_before_mutating_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_succeeded_run(&state, run_id).await;
    let too_many_ids = (0..251)
        .map(|_| RunId::new().to_string())
        .collect::<Vec<_>>();
    let invalid_requests = [
        json!({ "run_ids": [], "force": false }),
        json!({ "run_ids": [run_id.to_string(), run_id.to_string()], "force": false }),
        json!({ "run_ids": ["not-a-run-id"], "force": false }),
        json!({ "run_ids": too_many_ids, "force": false }),
    ];

    for body in invalid_requests {
        let response = app
            .clone()
            .oneshot(json_request(Method::POST, "/runs/delete", &body))
            .await
            .unwrap();
        assert_status!(response, StatusCode::BAD_REQUEST).await;
    }

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api(&format!("/runs/{run_id}")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn batch_delete_requires_user_authentication() {
    let (_state, app) = jwt_auth_app();
    let user_jwt = issue_test_user_jwt();
    let run_id = create_run_with_bearer(&app, &user_jwt).await;
    let worker_token = issue_test_worker_token(&run_id);
    let body = batch_delete_body(&[run_id], false);

    let unauthenticated = app
        .clone()
        .oneshot(json_request(Method::POST, "/runs/delete", &body))
        .await
        .unwrap();
    assert_status!(unauthenticated, StatusCode::UNAUTHORIZED).await;

    let worker_response = app
        .oneshot(json_bearer_request(
            Method::POST,
            "/runs/delete",
            &worker_token,
            &body,
        ))
        .await
        .unwrap();
    assert!(
        matches!(
            worker_response.status(),
            StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
        ),
        "/runs/delete unexpectedly accepted worker token with status {}",
        worker_response.status()
    );
}

#[tokio::test]
async fn archive_unknown_run_returns_not_found() {
    let app = test_app_with();
    let run_id = fixtures::RUN_64;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri(api(&format!("/runs/{run_id}/archive")))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn delete_run_removes_durable_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}?force=true")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn delete_run_force_removes_unreadable_durable_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_unreadable_durable_run(&state, run_id).await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/system/repair/runs"))
        .body(Body::empty())
        .unwrap();
    let body = response_json!(app.clone().oneshot(req).await.unwrap(), StatusCode::OK).await;
    assert_eq!(body["total_count"], 1);
    assert_eq!(body["runs"][0]["run_id"], run_id.to_string());

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}?force=true")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/system/repair/runs"))
        .body(Body::empty())
        .unwrap();
    let body = response_json!(app.oneshot(req).await.unwrap(), StatusCode::OK).await;
    assert_eq!(body["total_count"], 0);
    assert!(body["runs"].as_array().unwrap().is_empty());
}

#[tokio::test]
async fn delete_run_without_force_keeps_active_durable_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_unreadable_durable_run(&state, run_id).await;

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    response_json!(response, StatusCode::CONFLICT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/system/repair/runs"))
        .body(Body::empty())
        .unwrap();
    let body = response_json!(app.oneshot(req).await.unwrap(), StatusCode::OK).await;
    assert_eq!(body["total_count"], 1);
    assert_eq!(body["runs"][0]["run_id"], run_id.to_string());
}

#[tokio::test]
async fn delete_run_with_preserved_sandbox_returns_handoff() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    create_preserved_local_sandbox_run(&state, run_id).await;

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}?force=true")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["deleted"].as_bool(), Some(true));
    assert_eq!(body["sandbox_preserved"].as_bool(), Some(true));
    assert_eq!(body["sandbox"]["provider"].as_str(), Some("local"));
    assert_eq!(body["sandbox"]["id"].as_str(), Some("sandbox-preserve-1"));
    assert!(body["sandbox"].get("identifier").is_none());

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn delete_run_retry_after_missing_provider_resource_removes_metadata() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = RunId::new();
    let graph = Graph::new("test");

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunCreated {
            run_id,
            title: None,
            settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(),
            graph: serde_json::to_value(graph).unwrap(),
            workflow_source: None,
            labels: std::collections::BTreeMap::default(),
            source_directory: Some("/tmp/fabro-run".to_string()),
            workflow_slug: Some("test".to_string()),
            workflow_version_id: None,
            automation: None,
            provenance: test_support::test_run_provenance(),
            manifest_blob: None,
            spec_blob: None,
            git: None,
            fork_source_ref: None,
            retried_from: None,
            parent_id: None,
            web_url: None,
        },
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::SandboxInitialized {
            provider:          SandboxProviderKind::Docker,
            id:                "missing-sandbox".to_string(),
            working_directory: "/tmp/fabro-missing-sandbox".to_string(),
            image:             None,
            snapshot:          None,
            repo_cloned:       Some(false),
            clone_origin_url:  None,
            clone_branch:      None,
            workspace_root:    None,
            repos_root:        None,
            primary_repo_path: None,
            primary_repo_link: None,
        },
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    response_json!(response, StatusCode::CONFLICT).await;

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn delete_active_run_requires_force() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CONFLICT).await;
    let short_run_id = &run_id[..12.min(run_id.len())];
    let expected = format!(
        "cannot remove active run {short_run_id} (status: submitted, use force=true or --force to force)"
    );
    assert_eq!(
        body["errors"][0]["detail"].as_str(),
        Some(expected.as_str())
    );

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::OK).await;
}

#[tokio::test]
async fn delete_active_run_force_succeeds() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap();

    let req = Request::builder()
        .method("DELETE")
        .uri(api(&format!("/runs/{run_id}?force=true")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NO_CONTENT).await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn get_aggregate_billing_returns_zeros_initially() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("GET")
        .uri(api("/billing"))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["totals"]["runs"].as_i64().unwrap(), 0);
    assert_eq!(body["totals"]["input_tokens"].as_i64().unwrap(), 0);
    assert_eq!(body["totals"]["output_tokens"].as_i64().unwrap(), 0);
    assert_eq!(
        body["totals"]["timing"]["wall_time_ms"].as_u64().unwrap(),
        0
    );
    assert!(body["totals"]["total_usd_micros"].is_null());
    assert!(body["by_model"].as_array().unwrap().is_empty());
}

#[tokio::test]
async fn get_aggregate_billing_returns_provider_model_speed_identity() {
    let state = test_app_state();
    {
        let mut agg = state
            .aggregate_billing
            .lock()
            .expect("aggregate billing lock");
        agg.total_runs = 1;
        agg.by_model.insert(
            ModelRef {
                provider: ProviderId::anthropic(),
                model_id: "claude-opus-4-6".into(),
                speed:    None,
            },
            ModelBillingTotals {
                stages:  1,
                billing: BilledTokenCounts {
                    input_tokens:       10,
                    output_tokens:      1,
                    total_tokens:       11,
                    reasoning_tokens:   0,
                    cache_read_tokens:  0,
                    cache_write_tokens: 0,
                    total_usd_micros:   Some(11),
                },
            },
        );
        agg.by_model.insert(
            ModelRef {
                provider: ProviderId::anthropic(),
                model_id: "claude-opus-4-6".into(),
                speed:    Some(Speed::Fast),
            },
            ModelBillingTotals {
                stages:  1,
                billing: BilledTokenCounts {
                    input_tokens:       20,
                    output_tokens:      2,
                    total_tokens:       22,
                    reasoning_tokens:   0,
                    cache_read_tokens:  0,
                    cache_write_tokens: 0,
                    total_usd_micros:   Some(22),
                },
            },
        );
    }
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/billing"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let by_model = body["by_model"].as_array().unwrap();

    assert_eq!(by_model.len(), 2);
    let standard = by_model
        .iter()
        .find(|entry| entry["model"]["speed"].is_null())
        .unwrap();
    let fast = by_model
        .iter()
        .find(|entry| entry["model"]["speed"] == "fast")
        .unwrap();
    assert_eq!(standard["model"]["provider"], "anthropic");
    assert_eq!(standard["model"]["model_id"], "claude-opus-4-6");
    assert_eq!(standard["billing"]["input_tokens"], 10);
    assert_eq!(fast["model"]["provider"], "anthropic");
    assert_eq!(fast["model"]["model_id"], "claude-opus-4-6");
    assert_eq!(fast["billing"]["input_tokens"], 20);
}

#[tokio::test]
async fn get_aggregate_billing_saturates_total_cost_across_models() {
    let state = test_app_state();
    {
        let mut agg = state
            .aggregate_billing
            .lock()
            .expect("aggregate billing lock");
        for (model_id, total_usd_micros) in [("maximum", i64::MAX), ("one", 1)] {
            agg.by_model.insert(
                ModelRef {
                    provider: ProviderId::openai(),
                    model_id: model_id.into(),
                    speed:    None,
                },
                ModelBillingTotals {
                    stages:  1,
                    billing: BilledTokenCounts {
                        total_usd_micros: Some(total_usd_micros),
                        ..BilledTokenCounts::default()
                    },
                },
            );
        }
    }
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(api("/billing"))
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json!(response, StatusCode::OK).await;

    assert_eq!(body["totals"]["total_usd_micros"].as_i64(), Some(i64::MAX));
}

#[test]
fn aggregate_billing_counts_projection_rollup_usage_visits() {
    let mut accumulator = BillingAccumulator::default();
    let rollup = fabro_workflow::ProjectionBillingRollup {
        stages:             Vec::new(),
        totals:             BilledTokenCounts {
            input_tokens:       300,
            output_tokens:      30,
            total_tokens:       330,
            reasoning_tokens:   0,
            cache_read_tokens:  0,
            cache_write_tokens: 0,
            total_usd_micros:   Some(330),
        },
        by_model:           vec![
            fabro_workflow::ProjectionBillingByModel {
                model:   ModelRef {
                    provider: ProviderId::openai(),
                    model_id: "gpt-5.4".into(),
                    speed:    None,
                },
                stages:  1,
                billing: BilledTokenCounts {
                    input_tokens:       100,
                    output_tokens:      10,
                    total_tokens:       110,
                    reasoning_tokens:   0,
                    cache_read_tokens:  0,
                    cache_write_tokens: 0,
                    total_usd_micros:   Some(110),
                },
            },
            fabro_workflow::ProjectionBillingByModel {
                model:   ModelRef {
                    provider: ProviderId::openai(),
                    model_id: "gpt-5.4".into(),
                    speed:    Some(Speed::Fast),
                },
                stages:  1,
                billing: BilledTokenCounts {
                    input_tokens:       200,
                    output_tokens:      20,
                    total_tokens:       220,
                    reasoning_tokens:   0,
                    cache_read_tokens:  0,
                    cache_write_tokens: 0,
                    total_usd_micros:   Some(220),
                },
            },
        ],
        timing:             fabro_types::RunTiming::wall_only(2000),
        billed_visit_count: 2,
    };

    accumulate_billing_rollup(&mut accumulator, &rollup);

    assert_eq!(accumulator.total_runs, 1);
    assert_eq!(accumulator.total_timing.wall_time_ms, 2000);
    assert_eq!(accumulator.by_model.len(), 2);
    assert_eq!(
        accumulator.by_model[&ModelRef {
            provider: ProviderId::openai(),
            model_id: "gpt-5.4".into(),
            speed:    None,
        }]
            .stages,
        1
    );
    assert_eq!(
        accumulator.by_model[&ModelRef {
            provider: ProviderId::openai(),
            model_id: "gpt-5.4".into(),
            speed:    None,
        }]
            .billing
            .input_tokens,
        100
    );
    assert_eq!(
        accumulator.by_model[&ModelRef {
            provider: ProviderId::openai(),
            model_id: "gpt-5.4".into(),
            speed:    Some(Speed::Fast),
        }]
            .stages,
        1
    );
    assert_eq!(
        accumulator.by_model[&ModelRef {
            provider: ProviderId::openai(),
            model_id: "gpt-5.4".into(),
            speed:    Some(Speed::Fast),
        }]
            .billing
            .input_tokens,
        200
    );
}

#[tokio::test]
async fn post_runs_returns_submitted_status() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    // Check status is submitted (no start, no scheduler running)
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    assert_eq!(run_json_status(&body)["kind"], "submitted");
}

#[expect(
    clippy::disallowed_methods,
    reason = "test asserts the raw template source"
)]
#[tokio::test]
async fn start_run_persists_full_settings_snapshot() {
    let source = r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[run.execution]
mode = "dry_run"

[run.model]
provider = "anthropic"
name = "claude-sonnet-4-5"

[run.environment]
id = "local"

[[run.hooks]]
name = "snapshot-hook"
event = "run_start"
command = ["echo", "snapshot"]
blocking = false
timeout = "1s"
sandbox = false

[run.git.author]
name = "Snapshot Bot"
email = "snapshot@example.com"

[server.integrations.github]
app_id = "12345"

[server.web]
url = "http://example.test"

[server.api]
url = "http://api.example.test"

[server.logging]
level = "debug"
"#;
    let state = test_app_state_with_options(
        server_settings_from_toml(source),
        manifest_run_defaults_from_toml(source),
        5,
    );
    state
        .stores
        .vault
        .set(
            EnvVars::ANTHROPIC_API_KEY,
            "test-anthropic-api-key",
            SecretType::Token,
            None,
        )
        .await
        .unwrap();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::CREATED).await;
    let run_id = body["id"].as_str().unwrap().parse::<RunId>().unwrap();

    let _run_dir = {
        let runs = state.runs.lock().expect("runs lock poisoned");
        runs.get(&run_id)
            .and_then(|run| run.run_dir.clone())
            .expect("run_dir should be recorded")
    };
    let run_spec = state
        .stores
        .runs
        .open_run_reader(&run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap()
        .spec;
    let resolved_run = &run_spec.settings.run;

    // Verify a sampling of the persisted v2 settings, including inherited
    // run execution mode from server settings.
    assert_eq!(
        match &resolved_run.goal {
            Some(fabro_types::settings::run::RunGoal::Inline(value)) => Some(value.as_source()),
            _ => None,
        }
        .as_deref(),
        Some("Test"),
        "goal should be persisted from the manifest"
    );
    assert!(
        resolved_run.execution.mode == fabro_types::settings::run::RunMode::DryRun,
        "run execution mode should inherit from server settings"
    );
    assert_eq!(
        resolved_run.model.name.as_deref(),
        Some("claude-sonnet-4-5"),
    );

    // Server-operational fields (auth, integrations, etc.) deliberately
    // do not flow into the run's persisted settings — they live on the
    // server and are read via AppState::server_settings().
    let settings_json = serde_json::to_value(&run_spec.settings).unwrap();
    assert!(settings_json.pointer("/server").is_none());
}

#[tokio::test]
async fn cancel_runnable_run_succeeds() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    // Cancel it
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::OK).await;

    // Verify status is cancelled
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    assert_eq!(run_json_status(&body)["kind"], "failed");
    assert_eq!(run_json_status(&body)["reason"], "cancelled");

    // Cancelled runs appear in the runs list with a "failed" status
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id_str = run_id.to_string();
    let list_item = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|item| run_json_id(item) == Some(run_id_str.as_str()));
    assert!(
        list_item.is_some(),
        "cancelled run should appear in the list"
    );
    assert_eq!(
        run_json_status(list_item.unwrap())["kind"].as_str(),
        Some("failed"),
        "cancelled run should preserve the failed lifecycle status"
    );

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let status = run_store.state().await.unwrap().status;
    assert_eq!(status, RunStatus::Failed {
        reason: FailureReason::Cancelled,
    });
}

#[tokio::test]
async fn cancel_run_overwrites_pending_pause_request() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
    }
    append_control_request(state.as_ref(), run_id, RunControlAction::Pause, None)
        .await
        .unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::ACCEPTED).await;
    assert_eq!(run_json_pending_control(&body).as_str(), Some("cancel"));

    let summary = state
        .stores
        .runs
        .runs()
        .find(&run_id)
        .await
        .unwrap()
        .unwrap();
    assert_eq!(
        summary.lifecycle.pending_control,
        Some(RunControlAction::Cancel)
    );
}

async fn advance_past_worker_cancel_grace() {
    tokio::task::yield_now().await;
    tokio::time::advance(WORKER_CANCEL_GRACE).await;
    for _ in 0..10 {
        tokio::task::yield_now().await;
    }
}

#[tokio::test]
async fn cancel_run_requests_worker_runtime_stop_when_control_unavailable() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .worker_runtime(runtime.clone())
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let worker_ref = test_worker_ref(u32::MAX);
    tokio::time::pause();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.answer_transport = None;
        managed_run.worker_ref = Some(worker_ref.clone());
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    assert_eq!(runtime.requested_refs(), vec![worker_ref.clone()]);

    advance_past_worker_cancel_grace().await;
    runtime.wait_for_forced_ref(&worker_ref).await;

    assert_eq!(runtime.forced_refs(), vec![worker_ref]);
}

#[tokio::test]
async fn cancel_run_force_stops_worker_when_delivered_control_does_not_converge() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .worker_runtime(runtime.clone())
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let worker_ref = test_worker_ref(u32::MAX);
    let (answer_transport, _receiver) = worker_transport_with_receiver(run_id).await;
    tokio::time::pause();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.answer_transport = Some(answer_transport);
        managed_run.worker_ref = Some(worker_ref.clone());
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    assert!(runtime.requested_refs().is_empty());
    assert!(runtime.forced_refs().is_empty());

    advance_past_worker_cancel_grace().await;
    runtime.wait_for_forced_ref(&worker_ref).await;

    assert_eq!(runtime.forced_refs(), vec![worker_ref]);
}

#[tokio::test]
async fn cancel_run_watchdog_does_not_stop_replacement_worker() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .worker_runtime(runtime.clone())
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let cancelled_worker_ref = test_worker_ref(u32::MAX - 1);
    let replacement_worker_ref = test_worker_ref(u32::MAX);
    let (answer_transport, _receiver) = worker_transport_with_receiver(run_id).await;
    tokio::time::pause();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.answer_transport = Some(answer_transport);
        managed_run.worker_ref = Some(cancelled_worker_ref);
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    tokio::task::yield_now().await;
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.worker_ref = Some(replacement_worker_ref);
    }
    advance_past_worker_cancel_grace().await;

    assert!(runtime.forced_refs().is_empty());
}

#[tokio::test]
async fn cancel_run_watchdog_does_not_stop_worker_after_live_ref_clears() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .worker_runtime(runtime.clone())
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let worker_ref = test_worker_ref(u32::MAX);
    let (answer_transport, _receiver) = worker_transport_with_receiver(run_id).await;
    tokio::time::pause();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.answer_transport = Some(answer_transport);
        managed_run.worker_ref = Some(worker_ref);
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    tokio::task::yield_now().await;
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.worker_ref = None;
    }
    advance_past_worker_cancel_grace().await;

    assert!(runtime.forced_refs().is_empty());
}

#[tokio::test]
async fn repeated_cancel_request_arms_one_watchdog_and_persists_one_intent() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")])
        .worker_runtime(runtime.clone())
        .build();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let worker_ref = test_worker_ref(u32::MAX);
    let (answer_transport, _receiver) = worker_transport_with_receiver(run_id).await;
    tokio::time::pause();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.answer_transport = Some(answer_transport);
        managed_run.worker_ref = Some(worker_ref.clone());
    }

    let first_request = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let second_request = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let (first_response, second_response) = tokio::join!(
        app.clone().oneshot(first_request),
        app.clone().oneshot(second_request)
    );
    assert_status!(first_response.unwrap(), StatusCode::ACCEPTED).await;
    assert_status!(second_response.unwrap(), StatusCode::ACCEPTED).await;

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();
    let request_count = run_store
        .list_events()
        .await
        .unwrap()
        .iter()
        .filter(|event| event.event.event_name() == "run.cancel.requested")
        .count();
    assert_eq!(request_count, 1);

    advance_past_worker_cancel_grace().await;
    runtime.wait_for_forced_ref(&worker_ref).await;

    assert_eq!(runtime.forced_refs(), vec![worker_ref]);
}

#[tokio::test]
async fn cancel_durably_blocked_in_process_run_cancels_pending_interview_without_abort_signal() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunRunning,
        workflow_event::Event::RunBlocked {
            blocked_reason: BlockedReason::HumanInputRequired,
        },
    ])
    .await;

    let interviewer = Arc::new(ControlInterviewer::new());
    let mut question = Question::new("approve?", QuestionType::YesNo);
    question.id = "q-1".to_string();
    let ask_interviewer = Arc::clone(&interviewer);
    let ask = tokio::spawn(async move { ask_interviewer.ask(question).await });
    tokio::task::yield_now().await;

    let (cancel_tx, mut cancel_rx) = oneshot::channel();
    let cancel_token = CancellationToken::new();
    let temp_dir = tempfile::tempdir().unwrap();
    let mut run = managed_run(
        MINIMAL_DOT.to_string(),
        RunStatus::Running,
        Utc::now(),
        temp_dir.path().join(run_id.to_string()),
        RunExecutionMode::Start,
    );
    run.answer_transport = Some(RunAnswerTransport::InProcess {
        interviewer,
        steering_hub: Arc::new(fabro_workflow::SteeringHub::new(Arc::new(
            fabro_workflow::event::Emitter::new(run_id),
        ))),
    });
    run.cancel_token = Some(cancel_token);
    run.cancel_tx = Some(cancel_tx);
    state
        .runs
        .lock()
        .expect("runs lock poisoned")
        .insert(run_id, run);

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    let submission = tokio::time::timeout(std::time::Duration::from_millis(100), ask)
        .await
        .expect("cancel should resolve the pending in-process interview")
        .expect("interview task should not panic");
    assert_eq!(submission.answer.value, AnswerValue::Cancelled);
    assert!(
        matches!(
            cancel_rx.try_recv(),
            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
        ),
        "blocked in-process cancellation should let the workflow unwind instead of aborting it"
    );
}

#[tokio::test]
async fn pause_run_rejects_when_control_is_already_pending() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
    }
    append_control_request(state.as_ref(), run_id, RunControlAction::Cancel, None)
        .await
        .unwrap();

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/pause")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::CONFLICT).await;

    let summary = state
        .stores
        .runs
        .runs()
        .find(&run_id)
        .await
        .unwrap()
        .unwrap();
    assert_eq!(
        summary.lifecycle.pending_control,
        Some(RunControlAction::Cancel)
    );
}

#[tokio::test]
async fn pause_run_sets_pending_control_on_board_response() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Running;
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
        managed_run.answer_transport = Some(transport);
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/pause")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "runnable");
    assert_eq!(run_json_pending_control(&body).as_str(), Some("pause"));

    // Verify pending_control via /runs/{id} (board no longer includes this field)
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    assert_eq!(run_json_pending_control(&body).as_str(), Some("pause"));

    // Verify the run appears in the runs list with runnable status.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let item = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .find(|item| run_json_id(item) == Some(run_id_str.as_str()))
        .expect("board item should exist");
    assert!(run_json_status(item).is_object());
    assert_eq!(run_json_pending_control(item).as_str(), Some("pause"));
}

#[tokio::test]
async fn pause_run_immediately_pauses_blocked_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    append_raw_run_event(
        &state,
        run_id,
        "pause-starting",
        "2026-04-19T11:59:58Z",
        "run.starting",
        json!({}),
        None,
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "pause-running",
        "2026-04-19T11:59:59Z",
        "run.running",
        json!({}),
        None,
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "pause-blocked",
        "2026-04-19T12:00:00Z",
        "run.blocked",
        json!({ "blocked_reason": "human_input_required" }),
        None,
    )
    .await;

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Blocked {
            blocked_reason: BlockedReason::HumanInputRequired,
        };
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/pause")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "paused");
    assert_eq!(
        run_json_status(&body)["prior_block"],
        "human_input_required"
    );
    assert_eq!(run_json_pending_control(&body), &serde_json::Value::Null);

    let summary = state
        .stores
        .runs
        .runs()
        .find(&run_id)
        .await
        .unwrap()
        .unwrap();
    assert_eq!(summary.lifecycle.status, RunStatus::Paused {
        prior_block: Some(BlockedReason::HumanInputRequired),
    });
    assert_eq!(summary.lifecycle.pending_control, None);
}

#[tokio::test]
async fn unpause_run_sets_pending_control() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    let (transport, _control_rx) = worker_transport_with_receiver(run_id).await;
    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Paused { prior_block: None };
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
        managed_run.answer_transport = Some(transport);
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/unpause")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "runnable");
    assert_eq!(run_json_pending_control(&body).as_str(), Some("unpause"));

    let summary = state
        .stores
        .runs
        .runs()
        .find(&run_id)
        .await
        .unwrap()
        .unwrap();
    assert_eq!(
        summary.lifecycle.pending_control,
        Some(RunControlAction::Unpause)
    );
}

#[tokio::test]
async fn unpause_run_returns_blocked_when_human_gate_is_still_unresolved() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    append_raw_run_event(
        &state,
        run_id,
        "paused-blocked-starting",
        "2026-04-19T11:59:58Z",
        "run.starting",
        json!({}),
        None,
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "paused-blocked-running",
        "2026-04-19T11:59:59Z",
        "run.running",
        json!({}),
        None,
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "paused-blocked-paused",
        "2026-04-19T12:00:00Z",
        "run.paused",
        json!({}),
        None,
    )
    .await;
    append_raw_run_event(
        &state,
        run_id,
        "paused-blocked-status",
        "2026-04-19T12:00:01Z",
        "run.blocked",
        json!({ "blocked_reason": "human_input_required" }),
        None,
    )
    .await;

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&run_id).expect("run should exist");
        managed_run.status = RunStatus::Paused {
            prior_block: Some(BlockedReason::HumanInputRequired),
        };
        managed_run.worker_ref = Some(test_worker_ref(u32::MAX));
    }

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/unpause")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(run_json_status(&body)["kind"], "blocked");
    assert_eq!(
        run_json_status(&body)["blocked_reason"],
        "human_input_required"
    );
    assert_eq!(run_json_pending_control(&body), &serde_json::Value::Null);

    let summary = state
        .stores
        .runs
        .runs()
        .find(&run_id)
        .await
        .unwrap()
        .unwrap();
    assert_eq!(summary.lifecycle.status, RunStatus::Blocked {
        blocked_reason: BlockedReason::HumanInputRequired,
    });
    assert_eq!(summary.lifecycle.pending_control, None);
}

#[tokio::test]
async fn startup_reconciliation_marks_inflight_runs_terminal() {
    let state = test_app_state();

    create_durable_run_with_events(&state, fixtures::RUN_1, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
    ])
    .await;
    create_durable_run_with_events(&state, fixtures::RUN_2, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    create_durable_run_with_events(&state, fixtures::RUN_3, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::RunPaused,
        workflow_event::Event::RunCancelRequested { actor: None },
    ])
    .await;

    let reconciled = reconcile_incomplete_runs_on_startup(&state).await.unwrap();
    assert_eq!(reconciled, 2);

    let run_1 = state
        .stores
        .runs
        .open_run_reader(&fixtures::RUN_1)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    assert_eq!(run_1.status, RunStatus::Submitted);

    let run_2 = state
        .stores
        .runs
        .open_run_reader(&fixtures::RUN_2)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    let run_2_status = run_2.status;
    assert_eq!(run_2_status, RunStatus::Failed {
        reason: FailureReason::Terminated,
    });

    let run_3 = state
        .stores
        .runs
        .open_run_reader(&fixtures::RUN_3)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    let run_3_status = run_3.status;
    assert_eq!(run_3_status, RunStatus::Failed {
        reason: FailureReason::Cancelled,
    });
    assert_eq!(run_3.pending_control, None);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_active_workers_uses_worker_runtime_for_live_refs() {
    let runtime = StdArc::new(RecordingWorkerRuntime::default());
    runtime.set_alive(true);
    let state = TestAppStateBuilder::new()
        .worker_runtime(runtime.clone())
        .build();
    let worker_refs = [test_worker_ref(u32::MAX - 1), test_worker_ref(u32::MAX)];
    let run_ids = [RunId::new(), RunId::new()];
    let temp_dir = tempfile::tempdir().unwrap();

    for (run_id, worker_ref) in run_ids.iter().zip(worker_refs.iter()) {
        create_durable_run_with_events(&state, *run_id, &[
            workflow_event::Event::RunSubmitted {
                definition_blob: None,
            },
            workflow_event::Event::RunStarting,
            workflow_event::Event::RunRunning,
        ])
        .await;

        let mut run = managed_run(
            String::new(),
            RunStatus::Running,
            chrono::Utc::now(),
            temp_dir.path().join(run_id.to_string()),
            RunExecutionMode::Start,
        );
        run.worker_ref = Some(worker_ref.clone());
        state
            .runs
            .lock()
            .expect("runs lock poisoned")
            .insert(*run_id, run);
    }

    let terminated = shutdown_active_workers_with_grace(
        &state,
        Duration::from_millis(0),
        Duration::from_millis(1),
    )
    .await
    .unwrap();

    assert_eq!(terminated, 2);
    assert_eq!(runtime.requested_refs().len(), 2);
    assert_eq!(runtime.forced_refs().len(), 2);
}

#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_active_workers_terminates_process_groups() {
    let state = test_app_state();
    let run_id = fixtures::RUN_4;

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    let temp_dir = tempfile::tempdir().unwrap();
    let mut child = tokio::process::Command::new("sh");
    child
        .arg("-c")
        .arg("trap '' TERM; while :; do sleep 1; done")
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::null());
    fabro_proc::pre_exec_setpgid(child.as_std_mut());
    let mut child = child.spawn().unwrap();
    let worker_process_id = child.id().expect("worker pid should be available");

    {
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let mut run = managed_run(
            String::new(),
            RunStatus::Running,
            chrono::Utc::now(),
            temp_dir.path().join(run_id.to_string()),
            RunExecutionMode::Start,
        );
        run.worker_ref = Some(test_worker_ref(worker_process_id));
        runs.insert(run_id, run);
    }

    let terminated = shutdown_active_workers_with_grace(
        &state,
        Duration::from_millis(50),
        Duration::from_millis(10),
    )
    .await
    .unwrap();
    assert_eq!(terminated, 1);

    let exit_status = tokio::time::timeout(Duration::from_secs(2), child.wait())
        .await
        .expect("worker should exit after shutdown")
        .expect("wait should succeed");
    assert!(!exit_status.success());
    assert!(!fabro_proc::process_group_alive(worker_process_id));

    let run_state = state
        .stores
        .runs
        .open_run_reader(&run_id)
        .await
        .unwrap()
        .state()
        .await
        .unwrap();
    let run_status = run_state.status;
    assert_eq!(run_status, RunStatus::Failed {
        reason: FailureReason::Terminated,
    });
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancel_during_startup_persists_cancelled_reason() {
    let source = r#"
_version = 1

[server.auth]
methods = ["dev-token"]

[[run.prepare.steps]]
script = "sleep 5"

[run.prepare]
timeout = "30s"

[run.environment]
id = "local"
"#;
    let state = test_app_state_with_settings_and_registry_factory(
        server_settings_from_toml(source),
        manifest_run_defaults_from_toml(source),
        |interviewer| fabro_workflow::handler::default_registry(interviewer, || None),
    );
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    let runner = tokio::spawn(
        execute_run(Arc::clone(&state), run_id)
            .instrument(tracing::info_span!("run", id = %run_id)),
    );
    let mut live_status_before_cancel = None;
    for _ in 0..50 {
        live_status_before_cancel = {
            let runs = state.runs.lock().expect("runs lock poisoned");
            runs.get(&run_id).map(|run| run.status)
        };
        if matches!(
            live_status_before_cancel,
            Some(
                RunStatus::Starting
                    | RunStatus::Running
                    | RunStatus::Blocked { .. }
                    | RunStatus::Paused { .. }
            )
        ) {
            break;
        }
        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
    }
    assert!(
        matches!(
            live_status_before_cancel,
            Some(
                RunStatus::Starting
                    | RunStatus::Running
                    | RunStatus::Blocked { .. }
                    | RunStatus::Paused { .. }
            )
        ),
        "run should become cancellable before finishing, saw {live_status_before_cancel:?}"
    );

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let response_status = response.status();
    let response_body = body_json(response.into_body()).await;
    assert_eq!(
        response_status,
        StatusCode::ACCEPTED,
        "unexpected cancel response body: {response_body}; live status before cancel: {live_status_before_cancel:?}"
    );

    runner.await.unwrap();

    let runs = state.runs.lock().expect("runs lock poisoned");
    let managed_run = runs.get(&run_id).expect("run should exist");
    assert_eq!(managed_run.status, RunStatus::Failed {
        reason: FailureReason::Cancelled,
    });
    drop(runs);

    let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap();

    let mut status_record = None;
    for _ in 0..50 {
        let record = run_store.state().await.unwrap().status;
        if record
            == (RunStatus::Failed {
                reason: FailureReason::Cancelled,
            })
        {
            status_record = Some(record);
            break;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }

    let status_record = status_record.expect("status record should be persisted");
    assert_eq!(status_record, RunStatus::Failed {
        reason: FailureReason::Cancelled,
    });
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[expect(
    clippy::disallowed_methods,
    reason = "This test intentionally blocks inside a sync registry factory to simulate slow startup before cancellation."
)]
async fn cancel_before_run_transitions_to_running_returns_empty_attach_stream() {
    let state = test_app_state_with_registry_factory(|interviewer| {
        std::thread::sleep(std::time::Duration::from_millis(200));
        fabro_workflow::handler::default_registry(interviewer, || None)
    });
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let run_id_str = create_and_start_run(&app, MINIMAL_DOT).await;
    let run_id = run_id_str.parse::<RunId>().unwrap();

    let runner = tokio::spawn(
        execute_run(Arc::clone(&state), run_id)
            .instrument(tracing::info_span!("run", id = %run_id)),
    );
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;

    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/cancel")))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::ACCEPTED).await;

    runner.await.unwrap();

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/attach")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_bytes!(response, StatusCode::OK).await;
    assert!(
        body.is_empty(),
        "expected an empty attach stream, got {}",
        String::from_utf8_lossy(&body)
    );
}

/// Reasoning has to survive the whole durable path, not just the local
/// struct conversion: emitted event → run store → attach SSE JSON.
#[tokio::test]
async fn attach_stream_replays_agent_message_reasoning() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;

    create_durable_run_with_events(&state, run_id, &[
        stage_started_event("code", "agent"),
        workflow_event::Event::Agent {
            stage:             "code".to_string(),
            visit:             1,
            event:             fabro_agent::AgentEvent::AssistantMessage {
                text:            String::new(),
                model:           ModelRef {
                    provider: ProviderId::openai(),
                    model_id: "gpt-5.4".into(),
                    speed:    None,
                },
                usage:           TokenCounts::default(),
                cost_usd:        None,
                cost_source:     None,
                tool_call_count: 1,
                context_window:  None,
                reasoning:       Some(fabro_types::ReasoningOutput::new(
                    "inspect the sink first",
                    "read events.rs, then attach",
                )),
            },
            session_id:        Some("session-1".to_string()),
            parent_session_id: None,
            tool_call_id:      None,
        },
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}/attach?since_seq=1")))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_bytes!(response, StatusCode::OK).await;
    let text = String::from_utf8(body.clone()).unwrap();

    let message = text
        .lines()
        .filter_map(|line| line.strip_prefix("data: "))
        .filter_map(|data| serde_json::from_str::<serde_json::Value>(data).ok())
        .find(|value| value["event"] == "agent.message")
        .expect("attach stream should replay the agent message");
    assert_eq!(
        message["properties"]["reasoning"]["summary"],
        "inspect the sink first"
    );
    assert_eq!(
        message["properties"]["reasoning"]["trace"],
        "read events.rs, then attach"
    );
}

#[tokio::test]
async fn queue_position_reported_for_runnable_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Create and start two runs (no scheduler, both stay runnable)
    let first_run_id = create_and_start_run(&app, MINIMAL_DOT).await;
    let second_run_id = create_and_start_run(&app, MINIMAL_DOT).await;

    // Queue position is tracked in memory even when runnable runs are also
    // visible on the board.
    let runs = state.runs.lock().expect("runs lock poisoned");
    let positions = compute_queue_positions(&runs);
    let first_id = first_run_id.parse::<RunId>().unwrap();
    let second_id = second_run_id.parse::<RunId>().unwrap();
    assert_eq!(positions.get(&first_id).copied(), Some(1));
    assert_eq!(positions.get(&second_id).copied(), Some(2));
}

#[test]
fn scheduler_capacity_counts_only_runs_occupying_slots() {
    assert!(!counts_toward_scheduler_capacity(RunStatus::Submitted));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Pending {
        reason: PendingReason::ApprovalRequired,
    }));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Runnable));
    assert!(counts_toward_scheduler_capacity(RunStatus::Starting));
    assert!(counts_toward_scheduler_capacity(RunStatus::Running));
    assert!(counts_toward_scheduler_capacity(RunStatus::Blocked {
        blocked_reason: BlockedReason::HumanInputRequired,
    }));
    assert!(counts_toward_scheduler_capacity(RunStatus::Paused {
        prior_block: None,
    }));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Removing));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Succeeded {
        reason: SuccessReason::Completed,
    }));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Failed {
        reason: FailureReason::WorkflowError,
    }));
    assert!(!counts_toward_scheduler_capacity(RunStatus::Dead));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrency_limit_respected() {
    let state = test_app_state_with_options(default_test_server_settings(), RunLayer::default(), 1);
    let app = test_app_with_scheduler(Arc::clone(&state));

    // Create and start two runs with max_concurrent_runs=1
    create_and_start_run(&app, MINIMAL_DOT).await;
    create_and_start_run(&app, MINIMAL_DOT).await;

    // Give scheduler time to pick up the first run
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;

    // With max_concurrent_runs=1, at most one run should be live "running".
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let items = body["data"].as_array().unwrap();
    let active_count = items
        .iter()
        .filter(|item| run_json_status(item)["kind"].as_str() == Some("running"))
        .count();
    assert!(
        active_count <= 1,
        "expected at most 1 active run, got {active_count}"
    );
}

#[tokio::test]
async fn submit_answer_to_unstarted_run_returns_conflict() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/runs"))
        .header("content-type", "application/json")
        .body(manifest_body(MINIMAL_DOT))
        .unwrap();

    let response = app.clone().oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let run_id = body["id"].as_str().unwrap().to_string();

    // Try to submit an answer to a run with no active worker.
    let req = Request::builder()
        .method("POST")
        .uri(api(&format!("/runs/{run_id}/questions/q1/answer")))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::to_string(&serde_json::json!({"kind": "yes"})).unwrap(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::CONFLICT).await;
}

#[tokio::test]
async fn create_completion_missing_messages_returns_422() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from("{}"))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::UNPROCESSABLE_ENTITY).await;
}

#[tokio::test]
async fn create_completion_invalid_reasoning_effort_returns_422() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::json!({
                "messages": [],
                "reasoning_effort": "bogus"
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::UNPROCESSABLE_ENTITY).await;
}

#[tokio::test]
async fn create_completion_unknown_provider_returns_clear_error() {
    let app = test_app_with();

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::json!({
                "provider": "missing-provider",
                "model": "gpt-5.4",
                "stream": false,
                "messages": [
                    {
                        "role": "user",
                        "content": [{"kind": "text", "data": "hi"}]
                    }
                ]
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::BAD_REQUEST).await;
    assert_eq!(
        body["errors"][0]["detail"],
        "unknown model provider 'missing-provider'"
    );
}

#[tokio::test]
async fn create_completion_unsupported_reasoning_efforts_return_bad_request() {
    let upstream = MockServer::start();
    let completion = upstream.mock(|when, then| {
        when.method(POST);
        then.status(500);
    });
    let state = TestAppStateBuilder::new()
        .provider_base_url("moonshot", upstream.url("/v1"))
        .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")])
        .build();
    let app = crate::test_support::build_test_router(state);

    for stream in [false, true] {
        for effort in ["medium", "xhigh"] {
            let req = Request::builder()
                .method("POST")
                .uri(api("/completions"))
                .header("content-type", "application/json")
                .body(Body::from(
                    serde_json::json!({
                        "provider": "moonshot",
                        "model": "kimi-k3",
                        "reasoning_effort": effort,
                        "stream": stream,
                        "messages": [
                            {
                                "role": "user",
                                "content": [{"kind": "text", "data": "hi"}]
                            }
                        ]
                    })
                    .to_string(),
                ))
                .unwrap();

            let response = app.clone().oneshot(req).await.unwrap();
            let body = response_json!(response, StatusCode::BAD_REQUEST).await;
            assert_eq!(
                body["errors"][0]["detail"],
                format!(
                    "model 'kimi-k3' does not support reasoning_effort '{effort}'; allowed values: low, high, max"
                ),
                "stream={stream}"
            );
        }
    }

    completion.assert_calls(0);
}

#[tokio::test]
async fn create_completion_returns_disjoint_usage_buckets() {
    let upstream = MockServer::start();
    let completion = upstream.mock(|when, then| {
        when.method(POST).path("/chat/completions");
        then.status(200)
            .header("content-type", "application/json")
            .json_body(json!({
                "id": "chatcmpl-usage",
                "model": "kimi-k3",
                "choices": [{
                    "message": {"role": "assistant", "content": "OK"},
                    "finish_reason": "stop"
                }],
                "usage": {
                    "prompt_tokens": 200,
                    "completion_tokens": 30,
                    "total_tokens": 230,
                    "prompt_tokens_details": {
                        "cached_tokens": 50,
                        "cache_write_tokens": 100
                    },
                    "completion_tokens_details": {
                        "reasoning_tokens": 20
                    }
                }
            }));
    });
    let state = TestAppStateBuilder::new()
        .provider_base_url("moonshot", upstream.base_url())
        .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")])
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from(
            json!({
                "provider": "moonshot",
                "model": "kimi-k3",
                "stream": false,
                "messages": [{
                    "role": "user",
                    "content": [{"kind": "text", "data": "hi"}]
                }]
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(
        body["usage"],
        json!({
            "input_tokens": 50,
            "output_tokens": 10,
            "reasoning_tokens": 20,
            "cache_read_tokens": 50,
            "cache_write_tokens": 100
        })
    );
    completion.assert();
}

#[tokio::test]
async fn create_completion_default_model_uses_app_state_catalog() {
    let upstream = MockServer::start();
    let completion = upstream.mock(|when, then| {
        when.method(POST)
            .path("/chat/completions")
            .json_body_includes(r#"{"model":"acme-large"}"#);
        then.status(500)
            .header("content-type", "application/json")
            .json_body(json!({"error": {"message": "expected test failure"}}));
    });
    let llm_catalog_settings: LlmCatalogSettings = toml::from_str(&format!(
        r#"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
priority = 120

[providers.acme.auth]
credentials = ["vault:ACME_API_KEY"]

[providers.acme.models."acme-large"]
display_name = "Acme Large"
family = "acme"
default = true

[providers.acme.models."acme-large".limits]
context_window = 128000

[providers.acme.models."acme-large".features]
tools = true
vision = false
reasoning = false
"#,
        upstream.base_url()
    ))
    .expect("catalog fixture should parse");
    let state = TestAppStateBuilder::new()
        .llm_catalog_settings(llm_catalog_settings)
        .vault_entries([("ACME_API_KEY", "acme-test-key")])
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::json!({
                "stream": false,
                "messages": [
                    {
                        "role": "user",
                        "content": [{"kind": "text", "data": "hi"}]
                    }
                ]
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::BAD_GATEWAY).await;
    assert!(
        body["errors"][0]["detail"]
            .as_str()
            .unwrap()
            .contains("expected test failure"),
        "unexpected error body: {body:?}"
    );
    completion.assert();
}

#[tokio::test]
async fn create_completion_structured_output_forwards_reasoning_effort() {
    let upstream = MockServer::start();
    let completion = upstream.mock(|when, then| {
        when.method(POST)
            .path("/chat/completions")
            .json_body_includes(r#"{"model":"kimi-k3","reasoning_effort":"high"}"#);
        then.status(200)
            .header("content-type", "application/json")
            .json_body(json!({
                "id": "chatcmpl-kimi-structured",
                "model": "kimi-k3",
                "choices": [{
                    "message": {
                        "role": "assistant",
                        "content": "{\"answer\":42}"
                    },
                    "finish_reason": "stop"
                }],
                "usage": {
                    "prompt_tokens": 10,
                    "completion_tokens": 4,
                    "total_tokens": 14
                }
            }));
    });
    let state = TestAppStateBuilder::new()
        .provider_base_url("moonshot", upstream.base_url())
        .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")])
        .build();
    let app = crate::test_support::build_test_router(state);

    let req = Request::builder()
        .method("POST")
        .uri(api("/completions"))
        .header("content-type", "application/json")
        .body(Body::from(
            serde_json::json!({
                "provider": "moonshot",
                "model": "kimi-k3",
                "reasoning_effort": "high",
                "stream": false,
                "schema": {
                    "type": "object",
                    "properties": {
                        "answer": {"type": "integer"}
                    },
                    "required": ["answer"]
                },
                "messages": [
                    {
                        "role": "user",
                        "content": [{"kind": "text", "data": "Return the answer."}]
                    }
                ]
            })
            .to_string(),
        ))
        .unwrap();

    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["output"], json!({"answer": 42}));
    completion.assert();
}

#[tokio::test]
async fn demo_list_runs_returns_run_list_items() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    assert!(!data.is_empty(), "demo should return runs");
    let first = &data[0];
    assert!(first["id"].is_string());
    assert!(first["goal"].is_string());
    assert!(first["repository"].is_object());
    assert!(first["title"].is_string());
    assert!(run_json_status(first).is_object());
    assert!(first["workflow"]["slug"].is_string() || first["workflow"]["slug"].is_null());
    assert!(first["labels"].is_object());
    assert!(first["timestamps"]["created_at"].is_string());
}

#[tokio::test]
async fn demo_get_run_returns_run_summary_shape() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);
    let run_id = RunId::with_timestamp(
        "2026-03-06T14:30:00Z"
            .parse()
            .expect("demo timestamp should parse"),
        1,
    );
    let req = Request::builder()
        .method("GET")
        .uri(api(&format!("/runs/{run_id}")))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    // Should have Run fields, not RunStatusResponse fields
    assert!(body["id"].is_string(), "should have id field");
    assert!(body["goal"].is_string(), "should have goal field");
    assert!(
        body["workflow"]["slug"].is_string(),
        "should have workflow.slug field"
    );
    assert!(body["lifecycle"]["queue_position"].is_null());
}

#[tokio::test]
async fn demo_get_run_returns_404_for_unknown_run() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs/nonexistent-run-id"))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    assert_status!(response, StatusCode::NOT_FOUND).await;
}

#[tokio::test]
async fn demo_workflows_return_list_detail_and_runs() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(state);

    let list_req = Request::builder()
        .method("GET")
        .uri(api("/workflows"))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let list_response = app.clone().oneshot(list_req).await.unwrap();
    let list_body = response_json!(list_response, StatusCode::OK).await;
    let workflows = list_body["data"]
        .as_array()
        .expect("workflow list data should be an array");
    assert!(!workflows.is_empty(), "demo should return workflows");
    let first = &workflows[0];
    assert!(first["name"].is_string());
    assert!(first["slug"].is_string());
    assert!(first["filename"].is_string());
    assert!(first["last_run"].is_object() || first["last_run"].is_null());
    assert!(first["schedule"].is_object() || first["schedule"].is_null());

    let detail_req = Request::builder()
        .method("GET")
        .uri(api("/workflows/implement"))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let detail_response = app.clone().oneshot(detail_req).await.unwrap();
    let detail_body = response_json!(detail_response, StatusCode::OK).await;
    assert_eq!(detail_body["slug"], "implement");
    assert!(detail_body["settings"].is_object());
    assert!(
        detail_body["graph"]
            .as_str()
            .is_some_and(|graph| graph.contains("digraph"))
    );

    let runs_req = Request::builder()
        .method("GET")
        .uri(api("/workflows/implement/runs"))
        .header("X-Fabro-Demo", "1")
        .body(Body::empty())
        .unwrap();
    let runs_response = app.oneshot(runs_req).await.unwrap();
    let runs_body = response_json!(runs_response, StatusCode::OK).await;
    let runs = runs_body["data"]
        .as_array()
        .expect("workflow runs data should be an array");
    assert!(
        runs.iter()
            .all(|run| run["workflow"]["slug"].as_str() == Some("implement")),
        "workflow run list should be scoped to the requested workflow"
    );
}

#[tokio::test]
async fn list_runs_returns_run_list_items() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_and_start_run(&app, MINIMAL_DOT).await;

    {
        let id = run_id.parse::<RunId>().unwrap();
        let mut runs = state.runs.lock().expect("runs lock poisoned");
        let managed_run = runs.get_mut(&id).expect("run should exist");
        managed_run.status = RunStatus::Running;
    }

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    let item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&run_id))
        .expect("run should be in list");
    assert!(item["goal"].is_string());
    assert!(item["title"].is_string());
    assert!(item["repository"].is_object());
    assert!(item["workflow"]["slug"].is_string() || item["workflow"]["slug"].is_null());
    assert!(item["workflow"]["name"].is_string() || item["workflow"]["name"].is_null());
    assert!(item["workflow"]["graph_name"].is_string());
    assert!(item["labels"].is_object());
    assert!(run_json_status(item).is_object());
    assert!(item["timestamps"]["created_at"].is_string());
    assert!(run_json_pending_control(item).is_null());
    assert!(item["billing"].is_null());
}

#[tokio::test]
async fn list_runs_excludes_removing_status_by_default() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;

    // A run in Removing status should not appear by default
    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::RunRemoving,
    ])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    assert!(
        !data
            .iter()
            .any(|i| run_json_id(i) == Some(&run_id.to_string())),
        "removing run should not appear by default"
    );

    // ?status=removing opts the bucket in.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?status=removing"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    assert!(
        data.iter()
            .any(|i| run_json_id(i) == Some(&run_id.to_string())),
        "?status=removing should opt removing runs in"
    );
}

#[tokio::test]
async fn list_runs_excludes_archived_by_default() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = fixtures::RUN_1;

    create_durable_run_with_events(&state, run_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
        workflow_event::Event::RunArchived { actor: None },
    ])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    assert!(
        !data
            .iter()
            .any(|i| run_json_id(i) == Some(&run_id.to_string())),
        "archived run should be hidden when include_archived is unset",
    );
}

#[tokio::test]
async fn list_runs_includes_archived_when_flag_set() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let archived_id = fixtures::RUN_1;
    let succeeded_id = fixtures::RUN_2;

    create_durable_run_with_events(&state, archived_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
        workflow_event::Event::RunArchived { actor: None },
    ])
    .await;
    create_durable_run_with_events(&state, succeeded_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?include_archived=true"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");

    let archived_item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&archived_id.to_string()))
        .expect("archived run should appear when include_archived=true");
    assert!(run_json_archived(archived_item));
    assert_eq!(
        run_json_status(archived_item)["kind"].as_str().unwrap(),
        "succeeded"
    );

    let succeeded_item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&succeeded_id.to_string()))
        .expect("non-archived run should still appear");
    assert_eq!(
        run_json_status(succeeded_item)["kind"].as_str().unwrap(),
        "succeeded"
    );
}

#[tokio::test]
async fn get_run_exposes_canonical_operator_statuses() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let succeeded_id = fixtures::RUN_1;
    let removing_id = fixtures::RUN_2;
    let blocked_id = fixtures::RUN_3;

    create_durable_run_with_events(&state, succeeded_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    create_durable_run_with_events(&state, removing_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::RunRemoving,
    ])
    .await;
    create_durable_run_with_events(&state, blocked_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_raw_run_event(
        &state,
        blocked_id,
        "status-blocked",
        "2026-04-19T12:00:00Z",
        "run.blocked",
        json!({ "blocked_reason": "human_input_required" }),
        None,
    )
    .await;

    for (run_id, expected_status) in [
        (succeeded_id, "succeeded"),
        (removing_id, "removing"),
        (blocked_id, "blocked"),
    ] {
        let req = Request::builder()
            .method("GET")
            .uri(api(&format!("/runs/{run_id}")))
            .body(Body::empty())
            .unwrap();
        let response = app.clone().oneshot(req).await.unwrap();
        let body = response_json!(response, StatusCode::OK).await;
        assert_eq!(
            run_json_status(&body)["kind"].as_str(),
            Some(expected_status)
        );
    }
}

#[tokio::test]
async fn list_runs_preserves_underlying_run_status_payloads() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let paused_id = fixtures::RUN_1;
    let succeeded_id = fixtures::RUN_2;
    let blocked_id = fixtures::RUN_3;

    create_durable_run_with_events(&state, paused_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::RunPaused,
    ])
    .await;
    create_durable_run_with_events(&state, succeeded_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;
    create_durable_run_with_events(&state, blocked_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;
    append_raw_run_event(
        &state,
        blocked_id,
        "blocked-question-1",
        "2026-04-19T12:00:00Z",
        "interview.started",
        json!({
            "question_id": "q-older",
            "question": "Older unresolved question?",
            "stage": "gate",
            "question_type": "multiple_choice",
            "options": [],
            "allow_freeform": false,
            "context_display": null,
            "timeout_seconds": null,
        }),
        Some("gate"),
    )
    .await;
    append_raw_run_event(
        &state,
        blocked_id,
        "blocked-question-2",
        "2026-04-19T12:00:01Z",
        "interview.started",
        json!({
            "question_id": "q-newer",
            "question": "Newer unresolved question?",
            "stage": "gate",
            "question_type": "multiple_choice",
            "options": [],
            "allow_freeform": false,
            "context_display": null,
            "timeout_seconds": null,
        }),
        Some("gate"),
    )
    .await;
    append_raw_run_event(
        &state,
        blocked_id,
        "blocked-status",
        "2026-04-19T12:00:02Z",
        "run.blocked",
        json!({ "blocked_reason": "human_input_required" }),
        None,
    )
    .await;

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = body_json(response.into_body()).await;
    let data = body["data"].as_array().expect("data should be array");

    let paused_item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&paused_id.to_string()))
        .expect("paused run should be on board");
    assert_eq!(
        run_json_status(paused_item)["kind"].as_str().unwrap(),
        "paused"
    );
    assert!(run_json_status(paused_item)["prior_block"].is_null());

    let succeeded_item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&succeeded_id.to_string()))
        .expect("succeeded run should be on board");
    assert_eq!(
        run_json_status(succeeded_item)["kind"].as_str().unwrap(),
        "succeeded"
    );
    assert_eq!(
        run_json_status(succeeded_item)["reason"].as_str().unwrap(),
        "completed"
    );

    let blocked_item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&blocked_id.to_string()))
        .expect("blocked run should be on board");
    assert_eq!(
        run_json_status(blocked_item)["kind"].as_str().unwrap(),
        "blocked"
    );
    assert_eq!(
        run_json_status(blocked_item)["blocked_reason"]
            .as_str()
            .unwrap(),
        "human_input_required"
    );
    assert!(
        blocked_item["current_question"].is_object(),
        "blocked item should include the current question"
    );
}

#[tokio::test]
async fn list_runs_includes_live_metadata_from_run_state() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));
    let run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
    for event in [
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::SandboxInitialized {
            provider:          SandboxProviderKind::Local,
            id:                "sb-test".to_string(),
            working_directory: "/sandbox/workdir".to_string(),
            image:             None,
            snapshot:          None,
            repo_cloned:       None,
            clone_origin_url:  None,
            clone_branch:      None,
            workspace_root:    None,
            repos_root:        None,
            primary_repo_path: None,
            primary_repo_link: None,
        },
        workflow_event::Event::PullRequestCreated {
            pr_url:      "https://github.com/acme/repo/pull/42".to_string(),
            pr_number:   42,
            owner:       "acme".to_string(),
            repo:        "repo".to_string(),
            base_branch: "main".to_string(),
            head_branch: "fabro/run".to_string(),
            head_sha:    Some("final-sha".to_string()),
            title:       "Fix board metadata".to_string(),
            draft:       false,
        },
        workflow_event::Event::InterviewStarted {
            question_id:     "q-1".to_string(),
            question:        "Ship it?".to_string(),
            stage:           "review".to_string(),
            question_type:   "yes_no".to_string(),
            options:         vec![],
            allow_freeform:  false,
            timeout_seconds: None,
            context_display: None,
            review_target:   None,
        },
    ] {
        workflow_event::append_event(&run_store, &run_id, &event)
            .await
            .unwrap();
    }

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let data = body["data"].as_array().expect("data should be array");
    let item = data
        .iter()
        .find(|i| run_json_id(i) == Some(&run_id.to_string()))
        .expect("run should be in board");

    assert_eq!(item["pull_request"]["number"].as_u64(), Some(42));
    assert_eq!(item["sandbox"]["kind"].as_str(), Some("ready"));
    assert_eq!(
        item["sandbox"]["instance"]["runtime"]["id"].as_str(),
        Some("sb-test")
    );
    assert_eq!(
        item["sandbox"]["instance"]["runtime"]["working_directory"].as_str(),
        Some("/sandbox/workdir")
    );
    assert!(item["current_question"].is_object());
}

#[tokio::test]
async fn list_runs_page_limit_preserves_metadata_for_paged_items() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    let first_run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();
    let second_run_id = create_and_start_run(&app, MINIMAL_DOT)
        .await
        .parse::<RunId>()
        .unwrap();

    for (run_id, sandbox_id) in [(first_run_id, "sb-first"), (second_run_id, "sb-second")] {
        let run_store = state.stores.runs.open_run(&run_id).await.unwrap();
        for event in [
            workflow_event::Event::RunStarting,
            workflow_event::Event::RunRunning,
            workflow_event::Event::SandboxInitialized {
                provider:          SandboxProviderKind::Local,
                id:                sandbox_id.to_string(),
                working_directory: "/sandbox/workdir".to_string(),
                image:             None,
                snapshot:          None,
                repo_cloned:       None,
                clone_origin_url:  None,
                clone_branch:      None,
                workspace_root:    None,
                repos_root:        None,
                primary_repo_path: None,
                primary_repo_link: None,
            },
        ] {
            workflow_event::append_event(&run_store, &run_id, &event)
                .await
                .unwrap();
        }
    }

    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?page[limit]=1"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    assert_eq!(body["meta"]["has_more"].as_bool(), Some(true));

    let data = body["data"].as_array().expect("data should be array");
    assert_eq!(data.len(), 1);

    let item = &data[0];
    let sandbox_id = item["sandbox"]["instance"]["runtime"]["id"]
        .as_str()
        .expect("paged item should still include sandbox metadata");
    assert!(matches!(sandbox_id, "sb-first" | "sb-second"));
}

#[tokio::test]
async fn list_runs_status_filter_accepts_repeated_values() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // Running run (will map to BoardColumn::Running)
    let running_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, running_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    // Succeeded run (BoardColumn::Succeeded)
    let succeeded_id = fixtures::RUN_2;
    create_durable_run_with_events(&state, succeeded_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    // Pending run (BoardColumn::Pending via Submitted)
    let pending_id = fixtures::RUN_3;
    create_durable_run_with_events(&state, pending_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;

    // Single value: only running.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?status=running"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let ids: Vec<&str> = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .filter_map(run_json_id)
        .collect();
    assert!(ids.contains(&running_id.to_string().as_str()));
    assert!(!ids.contains(&succeeded_id.to_string().as_str()));
    assert!(!ids.contains(&pending_id.to_string().as_str()));

    // Repeated values: running + succeeded.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?status=running&status=succeeded"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let ids: Vec<&str> = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .filter_map(run_json_id)
        .collect();
    assert!(ids.contains(&running_id.to_string().as_str()));
    assert!(ids.contains(&succeeded_id.to_string().as_str()));
    assert!(!ids.contains(&pending_id.to_string().as_str()));
}

#[tokio::test]
async fn list_runs_sort_direction_reverses_order_with_stable_tiebreak() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // All fixtures share timestamp=0; the id-desc tiebreak controls order.
    let ids = [fixtures::RUN_1, fixtures::RUN_2, fixtures::RUN_3];
    for id in &ids {
        create_durable_run_with_events(&state, *id, &[
            workflow_event::Event::RunSubmitted {
                definition_blob: None,
            },
            workflow_event::Event::RunStarting,
            workflow_event::Event::RunRunning,
        ])
        .await;
    }

    // Default (sort=created_at desc): tiebreak puts higher ids first.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs"))
        .body(Body::empty())
        .unwrap();
    let response = app.clone().oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let observed: Vec<String> = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .filter_map(run_json_id)
        .map(str::to_string)
        .collect();
    let mut expected: Vec<String> = ids.iter().map(std::string::ToString::to_string).collect();
    expected.sort_by(|a, b| b.cmp(a)); // desc by id
    assert_eq!(observed, expected, "default desc order with id tiebreak");

    // Ascending: timestamps still tie, then id desc tiebreak still applies.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?sort=created_at&direction=asc"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let observed: Vec<String> = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .filter_map(run_json_id)
        .map(str::to_string)
        .collect();
    assert_eq!(
        observed, expected,
        "asc still uses id-desc tiebreak for tied keys"
    );
}

#[tokio::test]
async fn list_runs_sort_by_status_groups_by_bucket() {
    let state = test_app_state();
    let app = crate::test_support::build_test_router(Arc::clone(&state));

    // BoardColumn enum order: pending < runnable < initializing < running <
    // blocked < succeeded < failed < archived < removing. Use three distinct
    // buckets.
    let pending_id = fixtures::RUN_1;
    create_durable_run_with_events(&state, pending_id, &[workflow_event::Event::RunSubmitted {
        definition_blob: None,
    }])
    .await;

    let succeeded_id = fixtures::RUN_2;
    create_durable_run_with_events(&state, succeeded_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
        workflow_event::Event::WorkflowRunCompleted {
            timing:               fabro_types::RunTiming::wall_only(1000),
            artifact_count:       0,
            status:               "succeeded".to_string(),
            reason:               SuccessReason::Completed,
            total_usd_micros:     None,
            final_git_commit_sha: None,
            final_patch:          None,
            diff_summary:         None,
            billing:              None,
        },
    ])
    .await;

    let running_id = fixtures::RUN_3;
    create_durable_run_with_events(&state, running_id, &[
        workflow_event::Event::RunSubmitted {
            definition_blob: None,
        },
        workflow_event::Event::RunStarting,
        workflow_event::Event::RunRunning,
    ])
    .await;

    // sort=status asc: pending < running < succeeded.
    let req = Request::builder()
        .method("GET")
        .uri(api("/runs?sort=status&direction=asc"))
        .body(Body::empty())
        .unwrap();
    let response = app.oneshot(req).await.unwrap();
    let body = response_json!(response, StatusCode::OK).await;
    let observed: Vec<String> = body["data"]
        .as_array()
        .unwrap()
        .iter()
        .filter_map(run_json_id)
        .map(str::to_string)
        .collect();
    assert_eq!(observed, vec![
        pending_id.to_string(),
        running_id.to_string(),
        succeeded_id.to_string(),
    ]);
}

#[tokio::test]
async fn filtered_global_events_streams_only_matching_run_ids() {
    let run_one = fixtures::RUN_1;
    let run_two = fixtures::RUN_2;
    let (event_tx, _) = broadcast::channel(8);

    let stream = filtered_global_events(event_tx.subscribe(), Some(HashSet::from([run_one])));

    event_tx
        .send(test_event_envelope(
            1,
            run_two,
            EventBody::RunRunnable(fabro_types::run_event::RunRunnableProps {
                source: fabro_types::RunRunnableSource::StartRequested,
            }),
        ))
        .unwrap();
    event_tx
        .send(test_event_envelope(
            2,
            run_one,
            EventBody::RunRunnable(fabro_types::run_event::RunRunnableProps {
                source: fabro_types::RunRunnableSource::StartRequested,
            }),
        ))
        .unwrap();
    drop(event_tx);

    let events = stream.collect::<Vec<_>>().await;
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].seq, 2);
    assert_eq!(events[0].event.run_id, run_one);
}

#[test]
fn validate_github_slug_accepts_real_names() {
    assert!(super::validate_github_slug("owner", "anthropic", 39).is_ok());
    assert!(super::validate_github_slug("repo", "claude-code", 100).is_ok());
    assert!(super::validate_github_slug("repo", "repo.name_1", 100).is_ok());
}

#[test]
fn validate_github_slug_rejects_path_traversal_and_separators() {
    for bad in ["", "..", "foo/bar", "foo%2Fbar", "foo\\bar", "foo?x", "a b"] {
        assert!(
            super::validate_github_slug("owner", bad, 39).is_err(),
            "expected rejection for {bad:?}"
        );
    }
}

#[test]
fn validate_github_slug_rejects_overlong() {
    let long = "a".repeat(40);
    assert!(super::validate_github_slug("owner", &long, 39).is_err());
}