From 10555a292dce76bb8795136efcacea2bcfbbfb81 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Sat, 2 May 2026 13:32:49 -0400 Subject: [PATCH] refactor(server): split HTTP handlers into modules --- lib/crates/fabro-server/src/server.rs | 4817 +---------------- .../src/server/handler/artifacts.rs | 679 +++ .../src/server/handler/billing.rs | 236 + .../src/server/handler/completions.rs | 269 + .../fabro-server/src/server/handler/events.rs | 344 ++ .../fabro-server/src/server/handler/graph.rs | 298 + .../src/server/handler/lifecycle.rs | 797 +++ .../fabro-server/src/server/handler/mod.rs | 146 + .../fabro-server/src/server/handler/models.rs | 154 + .../src/server/handler/pull_requests.rs | 399 ++ .../fabro-server/src/server/handler/runs.rs | 780 +++ .../src/server/handler/sandbox.rs | 274 + .../src/server/handler/secrets.rs | 102 + .../fabro-server/src/server/handler/system.rs | 506 ++ 14 files changed, 5003 insertions(+), 4798 deletions(-) create mode 100644 lib/crates/fabro-server/src/server/handler/artifacts.rs create mode 100644 lib/crates/fabro-server/src/server/handler/billing.rs create mode 100644 lib/crates/fabro-server/src/server/handler/completions.rs create mode 100644 lib/crates/fabro-server/src/server/handler/events.rs create mode 100644 lib/crates/fabro-server/src/server/handler/graph.rs create mode 100644 lib/crates/fabro-server/src/server/handler/lifecycle.rs create mode 100644 lib/crates/fabro-server/src/server/handler/mod.rs create mode 100644 lib/crates/fabro-server/src/server/handler/models.rs create mode 100644 lib/crates/fabro-server/src/server/handler/pull_requests.rs create mode 100644 lib/crates/fabro-server/src/server/handler/runs.rs create mode 100644 lib/crates/fabro-server/src/server/handler/sandbox.rs create mode 100644 lib/crates/fabro-server/src/server/handler/secrets.rs create mode 100644 lib/crates/fabro-server/src/server/handler/system.rs diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 98b7b1153..c324aa61f 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -136,6 +136,18 @@ use crate::{ canonical_host, demo, diagnostics, run_manifest, security_headers, static_files, web_auth, }; +mod handler; + +#[cfg(test)] +pub(in crate::server) use handler::events::filtered_global_events; +pub(crate) use handler::graph::render_graph_bytes; +#[cfg(test)] +pub(in crate::server) use handler::graph::{ + RenderSubprocessError, render_dot_subprocess, render_graph_bytes_with_exe_override, +}; +#[cfg(test)] +pub(in crate::server) use handler::system::validate_github_slug; + pub(crate) type EnvLookup = Arc Option + Send + Sync>; pub fn default_page_limit() -> u32 { @@ -150,99 +162,12 @@ pub struct PaginationParams { pub offset: u32, } -#[derive(serde::Deserialize)] -struct ListRunsParams { - #[serde(rename = "page[limit]", default = "default_page_limit")] - limit: u32, - #[serde(rename = "page[offset]", default)] - offset: u32, - #[serde(default)] - include_archived: bool, -} - -impl ListRunsParams { - fn pagination(&self) -> PaginationParams { - PaginationParams { - limit: self.limit, - offset: self.offset, - } - } -} - -#[derive(serde::Deserialize)] -struct ModelListParams { - #[serde(rename = "page[limit]", default = "default_page_limit")] - limit: u32, - #[serde(rename = "page[offset]", default)] - offset: u32, - #[serde(default)] - provider: Option, - #[serde(default)] - query: Option, -} - -#[derive(serde::Deserialize)] -struct ModelTestParams { - #[serde(default)] - mode: Option, -} - -#[derive(serde::Deserialize)] -struct EventListParams { - #[serde(default)] - since_seq: Option, - #[serde(default)] - limit: Option, -} - -impl EventListParams { - fn since_seq(&self) -> u32 { - self.since_seq.unwrap_or(1).max(1) - } - - fn limit(&self) -> usize { - self.limit.unwrap_or(100).clamp(1, 1000) - } -} - -#[derive(serde::Deserialize)] -struct AttachParams { - #[serde(default)] - since_seq: Option, -} - #[derive(serde::Deserialize)] pub(crate) struct DfParams { #[serde(default)] pub(crate) verbose: bool, } -#[derive(serde::Deserialize)] -struct GlobalAttachParams { - #[serde(default)] - run_id: Option, -} - -#[derive(serde::Deserialize)] -struct ArtifactFilenameParams { - #[serde(default)] - filename: Option, - #[serde(default)] - retry: Option, -} - -#[derive(serde::Deserialize)] -struct SandboxFilesParams { - path: String, - #[serde(default)] - depth: Option, -} - -#[derive(serde::Deserialize)] -struct SandboxFileParams { - path: String, -} - /// Non-paginated list response wrapper with `has_more: false`. #[derive(serde::Serialize)] pub struct ListResponse { @@ -294,45 +219,6 @@ const WORKER_CANCEL_GRACE: Duration = Duration::from_secs(5); const TERMINAL_DELETE_WORKER_GRACE: Duration = Duration::from_millis(50); const WORKER_CONTROL_QUEUE_CAPACITY: usize = 8; const WORKER_CONTROL_ENQUEUE_TIMEOUT: Duration = Duration::from_secs(1); -const MAX_SINGLE_ARTIFACT_BYTES: u64 = 10 * 1024 * 1024; -const MAX_MULTIPART_ARTIFACTS: usize = 100; -const RENDER_ERROR_PREFIX: &[u8] = b"RENDER_ERROR:"; -const GRAPHVIZ_RENDER_CONCURRENCY_LIMIT: usize = 4; - -static GRAPHVIZ_RENDER_SEMAPHORE: LazyLock = - LazyLock::new(|| Semaphore::new(GRAPHVIZ_RENDER_CONCURRENCY_LIMIT)); - -#[derive(Debug, thiserror::Error)] -enum RenderSubprocessError { - #[error("failed to spawn render subprocess: {0}")] - SpawnFailed(String), - #[error("render subprocess crashed: {0}")] - ChildCrashed(String), - #[error("render subprocess returned invalid output: {0}")] - ProtocolViolation(String), - #[error("{0}")] - RenderFailed(String), -} -const MAX_MULTIPART_REQUEST_BYTES: u64 = 50 * 1024 * 1024; -const MAX_MULTIPART_MANIFEST_BYTES: usize = 256 * 1024; - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -struct ArtifactBatchUploadManifest { - entries: Vec, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -struct ArtifactBatchUploadEntry { - part: String, - path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - sha256: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - expected_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - content_type: Option, -} - /// Per-model billing totals. #[derive(Default)] struct ModelBillingTotals { @@ -978,10 +864,10 @@ pub fn build_router_with_options( let principal_layer = middleware::from_fn_with_state(Arc::clone(&state), principal_middleware); let api_common = if web_enabled { Router::new() - .route("/openapi.json", get(openapi_spec)) + .route("/openapi.json", get(handler::openapi_spec)) .merge(web_auth::api_routes()) } else { - Router::new().route("/openapi.json", get(openapi_spec)) + Router::new().route("/openapi.json", get(handler::openapi_spec)) }; let demo_router = Router::new() @@ -989,7 +875,7 @@ pub fn build_router_with_options( "/api/v1", api_common .clone() - .merge(demo_routes()) + .merge(handler::demo_routes()) .layer(principal_layer.clone()), ) .layer(axum::Extension(auth_mode.clone())) @@ -998,7 +884,9 @@ pub fn build_router_with_options( let mut real_router = Router::new().nest( "/api/v1", - api_common.merge(real_routes()).layer(principal_layer), + api_common + .merge(handler::real_routes()) + .layer(principal_layer), ); if web_enabled { real_router = real_router.nest("/auth", web_auth::routes().merge(auth::web_routes())); @@ -1023,7 +911,7 @@ pub fn build_router_with_options( }); let mut app_router = Router::new() - .route("/health", get(health)) + .route("/health", get(handler::health)) .fallback_service(service_fn(move |req: axum_extract::Request| { let dispatch = dispatch.clone(); let static_asset_root = static_asset_root.clone(); @@ -1186,205 +1074,6 @@ fn github_webhook_routes(secret: Arc<[u8]>, ip_allowlist_config: Arc Router> { - Router::new() - .route("/runs", get(demo::list_runs).post(demo::create_run_stub)) - .route("/runs/resolve", get(demo::resolve_run)) - .route("/boards/runs", get(demo::list_board_runs)) - .route("/preflight", post(run_preflight)) - .route("/validate", post(validate_run_manifest)) - .route("/graph/render", post(render_graph_from_manifest)) - .route("/attach", get(demo::attach_events_stub)) - .route("/runs/{id}", get(demo::get_run_status)) - .route("/runs/{id}/questions", get(demo::get_questions_stub)) - .route("/runs/{id}/questions/{qid}/answer", post(demo::answer_stub)) - .route("/runs/{id}/state", get(not_implemented)) - .route("/runs/{id}/logs", get(not_implemented)) - .route( - "/runs/{id}/events", - get(not_implemented).post(not_implemented), - ) - .route("/runs/{id}/attach", get(demo::run_events_stub)) - .route("/runs/{id}/blobs", post(not_implemented)) - .route("/runs/{id}/blobs/{blobId}", get(not_implemented)) - .route( - "/runs/{id}/stages/{stageId}/logs/{stream}", - get(not_implemented), - ) - .route("/runs/{id}/checkpoint", get(demo::checkpoint_stub)) - .route("/runs/{id}/cancel", post(demo::cancel_stub)) - .route("/runs/{id}/start", post(demo::start_run_stub)) - .route("/runs/{id}/pause", post(demo::pause_stub)) - .route("/runs/{id}/unpause", post(demo::unpause_stub)) - .route("/runs/{id}/graph", get(demo::get_run_graph)) - .route("/runs/{id}/graph/source", get(demo::get_run_graph_source)) - .route("/runs/{id}/stages", get(demo::get_run_stages)) - .route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub)) - .route("/runs/{id}/files", get(demo::list_run_files_stub)) - .route( - "/runs/{id}/stages/{stageId}/turns", - get(demo::get_stage_turns), - ) - .route( - "/runs/{id}/stages/{stageId}/artifacts", - get(not_implemented).post(not_implemented), - ) - .route( - "/runs/{id}/stages/{stageId}/artifacts/download", - get(not_implemented), - ) - .route("/runs/{id}/billing", get(demo::get_run_billing)) - .route("/runs/{id}/settings", get(demo::get_run_settings)) - .route("/runs/{id}/preview", post(demo::generate_preview_url_stub)) - .route("/runs/{id}/ssh", post(demo::create_ssh_access_stub)) - .route( - "/runs/{id}/sandbox/files", - get(demo::list_sandbox_files_stub), - ) - .route( - "/runs/{id}/sandbox/file", - get(demo::get_sandbox_file_stub).put(demo::put_sandbox_file_stub), - ) - .route( - "/insights/queries", - get(demo::list_saved_queries).post(demo::save_query_stub), - ) - .route( - "/insights/queries/{id}", - get(demo::get_saved_query) - .put(demo::update_query_stub) - .delete(demo::delete_query_stub), - ) - .route("/insights/execute", post(demo::execute_query_stub)) - .route("/insights/history", get(demo::list_query_history)) - .route("/models", get(list_models)) - .route("/models/{id}/test", post(test_model)) - .route( - "/secrets", - get(demo::list_secrets) - .post(demo::create_secret) - .delete(demo::delete_secret_by_name), - ) - .route("/repos/github/{owner}/{name}", get(demo::get_github_repo)) - .route("/health/diagnostics", post(demo::run_diagnostics)) - .route("/completions", post(create_completion)) - .route("/settings", get(demo::get_server_settings)) - .route("/system/info", get(demo::get_system_info)) - .route("/system/df", get(demo::get_system_disk_usage)) - .route("/system/prune/runs", post(demo::prune_runs)) - .route("/billing", get(demo::get_aggregate_billing)) -} - -fn real_routes() -> Router> { - Router::new() - .route("/runs", get(list_runs).post(create_run)) - .route("/runs/resolve", get(resolve_run)) - .route("/preflight", post(run_preflight)) - .route("/validate", post(validate_run_manifest)) - .route("/graph/render", post(render_graph_from_manifest)) - .route("/attach", get(attach_events)) - .route("/boards/runs", get(list_board_runs)) - .route("/runs/{id}", get(get_run_status).delete(delete_run)) - .route("/runs/{id}/questions", get(get_questions)) - .route("/runs/{id}/questions/{qid}/answer", post(submit_answer)) - .route("/runs/{id}/state", get(get_run_state)) - .route("/runs/{id}/logs", get(get_run_logs)) - .route( - "/runs/{id}/pull_request", - get(get_run_pull_request).post(create_run_pull_request), - ) - .route( - "/runs/{id}/pull_request/merge", - post(merge_run_pull_request), - ) - .route( - "/runs/{id}/pull_request/close", - post(close_run_pull_request), - ) - .route( - "/runs/{id}/events", - get(list_run_events).post(append_run_event), - ) - .route("/runs/{id}/attach", get(attach_run_events)) - .route("/runs/{id}/blobs", post(write_run_blob)) - .route("/runs/{id}/blobs/{blobId}", get(read_run_blob)) - .route( - "/runs/{id}/stages/{stageId}/logs/{stream}", - get(get_run_stage_command_log), - ) - .route("/runs/{id}/checkpoint", get(get_checkpoint)) - .route("/runs/{id}/cancel", post(cancel_run)) - .route("/runs/{id}/start", post(start_run)) - .route("/runs/{id}/pause", post(pause_run)) - .route("/runs/{id}/unpause", post(unpause_run)) - .route("/runs/{id}/archive", post(archive_run)) - .route("/runs/{id}/rewind", post(rewind_run)) - .route("/runs/{id}/fork", post(fork_run)) - .route("/runs/{id}/timeline", get(run_timeline)) - .route("/runs/{id}/unarchive", post(unarchive_run)) - .route("/runs/{id}/graph", get(get_graph)) - .route("/runs/{id}/graph/source", get(get_graph_source)) - .route("/runs/{id}/stages", get(list_run_stages)) - .route("/runs/{id}/artifacts", get(list_run_artifacts)) - .route("/runs/{id}/files", get(list_run_files)) - .route("/runs/{id}/stages/{stageId}/turns", get(not_implemented)) - .route( - "/runs/{id}/stages/{stageId}/artifacts", - get(list_stage_artifacts) - .post(put_stage_artifact) - .layer(DefaultBodyLimit::disable()), - ) - .route( - "/runs/{id}/stages/{stageId}/artifacts/download", - get(get_stage_artifact), - ) - .route("/runs/{id}/billing", get(get_run_billing)) - .route("/runs/{id}/settings", get(get_run_settings)) - .route("/runs/{id}/steer", post(not_implemented)) - .route("/runs/{id}/preview", post(generate_preview_url)) - .route("/runs/{id}/ssh", post(create_ssh_access)) - .route("/runs/{id}/sandbox/files", get(list_sandbox_files)) - .route( - "/runs/{id}/sandbox/file", - get(get_sandbox_file).put(put_sandbox_file), - ) - .route("/workflows", get(not_implemented)) - .route("/workflows/{name}", get(not_implemented)) - .route("/workflows/{name}/runs", get(not_implemented)) - .route( - "/insights/queries", - get(not_implemented).post(not_implemented), - ) - .route( - "/insights/queries/{id}", - get(not_implemented) - .put(not_implemented) - .delete(not_implemented), - ) - .route("/insights/execute", post(not_implemented)) - .route("/insights/history", get(not_implemented)) - .route("/models", get(list_models)) - .route("/models/{id}/test", post(test_model)) - .route( - "/secrets", - get(list_secrets) - .post(create_secret) - .delete(delete_secret_by_name), - ) - .route("/repos/github/{owner}/{name}", get(get_github_repo)) - .route("/health/diagnostics", post(run_diagnostics)) - .route("/completions", post(create_completion)) - .route("/settings", get(get_server_settings)) - .route("/system/info", get(get_system_info)) - .route("/system/df", get(get_system_df)) - .route("/system/prune/runs", post(prune_runs)) - .route("/billing", get(get_aggregate_billing)) -} - -async fn not_implemented() -> Response { - ApiError::new(StatusCode::NOT_IMPLEMENTED, "Not implemented.").into_response() -} - async fn github_webhook( State(secret): State>, RequestAuth(auth_slot): RequestAuth, @@ -1443,66 +1132,6 @@ async fn github_webhook( StatusCode::OK } -async fn health() -> Response { - Json(serde_json::json!({ - "status": "ok", - })) - .into_response() -} - -async fn get_server_settings(_auth: RequiredUser, State(state): State>) -> Response { - ( - StatusCode::OK, - Json(state.server_settings().as_ref().clone()), - ) - .into_response() -} - -async fn get_system_info(_auth: RequiredUser, State(state): State>) -> Response { - let manifest_run_settings = state.manifest_run_settings(); - let server_settings = state.server_settings(); - let (total_runs, active_runs) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - let active = runs - .values() - .filter(|run| { - matches!( - run.status, - RunStatus::Queued - | RunStatus::Starting - | RunStatus::Running - | RunStatus::Blocked { .. } - | RunStatus::Paused { .. } - ) - }) - .count(); - (runs.len(), active) - }; - - let response = SystemInfoResponse { - version: Some(FABRO_VERSION.to_string()), - server_url: Some(server_settings.server.web.url.as_source()), - git_sha: option_env!("FABRO_GIT_SHA").map(str::to_string), - build_date: option_env!("FABRO_BUILD_DATE").map(str::to_string), - profile: option_env!("FABRO_BUILD_PROFILE").map(str::to_string), - os: Some(std::env::consts::OS.to_string()), - arch: Some(std::env::consts::ARCH.to_string()), - storage_engine: Some("slatedb".to_string()), - storage_dir: Some(state.server_storage_dir().display().to_string()), - uptime_secs: Some(to_i64(state.started_at.elapsed().as_secs())), - runs: Some(SystemRunCounts { - total: Some(to_i64(total_runs)), - active: Some(to_i64(active_runs)), - }), - sandbox_provider: Some(system_sandbox_provider(&manifest_run_settings)), - features: Some(system_features( - server_settings.as_ref(), - &manifest_run_settings, - )), - }; - (StatusCode::OK, Json(response)).into_response() -} - fn system_features( server_settings: &ServerSettings, manifest_run_settings: &std::result::Result, @@ -1517,142 +1146,6 @@ fn system_features( } } -async fn get_system_df( - _auth: RequiredUser, - State(state): State>, - Query(params): Query, -) -> Response { - let storage_dir = state.server_storage_dir(); - let summaries = match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(summaries) => summaries, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let response = match spawn_blocking(move || { - build_disk_usage_response(&summaries, &storage_dir, params.verbose) - }) - .await - { - Ok(Ok(response)) => response, - Ok(Err(err)) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - (StatusCode::OK, Json(response)).into_response() -} - -async fn prune_runs( - _auth: RequiredUser, - State(state): State>, - Json(body): Json, -) -> Response { - let storage_dir = state.server_storage_dir(); - let summaries = match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(summaries) => summaries, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let dry_run = body.dry_run; - let body_for_plan = body.clone(); - let prune_plan = - match spawn_blocking(move || build_prune_plan(&body_for_plan, &summaries, &storage_dir)) - .await - { - Ok(Ok(plan)) => plan, - Ok(Err(err)) => { - return ApiError::new(StatusCode::BAD_REQUEST, err.to_string()).into_response(); - } - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - if dry_run { - return ( - StatusCode::OK, - Json(PruneRunsResponse { - dry_run: Some(true), - runs: Some(prune_plan.rows), - total_count: Some(to_i64(prune_plan.run_ids.len())), - total_size_bytes: Some(to_i64(prune_plan.total_size_bytes)), - deleted_count: Some(0), - freed_bytes: Some(0), - }), - ) - .into_response(); - } - - for run_id in &prune_plan.run_ids { - if let Err(response) = delete_run_internal(&state, *run_id, true).await { - return response; - } - } - - ( - StatusCode::OK, - Json(PruneRunsResponse { - dry_run: Some(false), - runs: None, - total_count: Some(to_i64(prune_plan.run_ids.len())), - total_size_bytes: Some(to_i64(prune_plan.total_size_bytes)), - deleted_count: Some(to_i64(prune_plan.run_ids.len())), - freed_bytes: Some(to_i64(prune_plan.total_size_bytes)), - }), - ) - .into_response() -} - -async fn attach_events( - _auth: RequiredUser, - State(state): State>, - Query(params): Query, -) -> Response { - let run_filter = match parse_global_run_filter(params.run_id.as_deref()) { - Ok(filter) => filter, - Err(err) => return ApiError::new(StatusCode::BAD_REQUEST, err).into_response(), - }; - - let stream = - filtered_global_events(state.global_event_tx.subscribe(), run_filter).filter_map(|event| { - sse_event_from_store(&event).map(Ok::) - }); - - Sse::new(stream) - .keep_alive(KeepAlive::default()) - .into_response() -} - -fn filtered_global_events( - event_rx: broadcast::Receiver, - run_filter: Option>, -) -> impl tokio_stream::Stream { - BroadcastStream::new(event_rx).filter_map(move |result| match result { - Ok(event) if event_matches_run_filter(&event, run_filter.as_ref()) => Some(event), - Ok(_) | Err(_) => None, - }) -} - struct PrunePlan { run_ids: Vec, rows: Vec, @@ -1839,54 +1332,6 @@ fn parse_system_duration(raw: &str) -> anyhow::Result { } } -fn parse_global_run_filter(raw: Option<&str>) -> Result>, String> { - let Some(raw) = raw else { - return Ok(None); - }; - - let mut run_ids = HashSet::new(); - for part in raw - .split(',') - .map(str::trim) - .filter(|part| !part.is_empty()) - { - let run_id = part - .parse::() - .map_err(|err| format!("invalid run_id '{part}': {err}"))?; - run_ids.insert(run_id); - } - - if run_ids.is_empty() { - Ok(None) - } else { - Ok(Some(run_ids)) - } -} - -fn event_matches_run_filter(event: &EventEnvelope, run_filter: Option<&HashSet>) -> bool { - let Some(run_filter) = run_filter else { - return true; - }; - run_filter.contains(&event.event.run_id) -} - -fn sse_event_from_store(event: &EventEnvelope) -> Option { - let data = serde_json::to_string(event).ok()?; - let data = redact_jsonl_line(&data); - Some(Event::default().data(data)) -} - -fn attach_event_is_terminal(event: &EventEnvelope) -> bool { - matches!( - &event.event.body, - EventBody::RunCompleted(_) | EventBody::RunFailed(_) - ) -} - -fn run_projection_is_active(state: &fabro_store::RunProjection) -> bool { - state.status.is_some_and(RunStatus::is_active) -} - fn dir_size(path: &std::path::Path) -> u64 { walkdir::WalkDir::new(path) .into_iter() @@ -1904,628 +1349,6 @@ where i64::try_from(value).unwrap_or(i64::MAX) } -async fn list_secrets(_auth: RequiredUser, State(state): State>) -> Response { - let data = state.vault.read().await.list(); - (StatusCode::OK, Json(serde_json::json!({ "data": data }))).into_response() -} - -async fn create_secret( - _auth: RequiredUser, - State(state): State>, - Json(body): Json, -) -> Response { - let secret_type = body.type_; - let name = body.name; - let value = body.value; - let description = body.description; - if secret_type == SecretType::Credential { - if let Err(err) = parse_credential_secret(&name, &value) { - return ApiError::bad_request(err).into_response(); - } - } - let state_for_write = Arc::clone(&state); - let result = spawn_blocking(move || { - let mut vault = state_for_write.vault.blocking_write(); - vault.set(&name, &value, secret_type, description.as_deref()) - }) - .await; - - match result { - Ok(Ok(meta)) => (StatusCode::OK, Json(meta)).into_response(), - Ok(Err(VaultError::InvalidName(_))) => { - ApiError::bad_request("invalid secret name").into_response() - } - Ok(Err(VaultError::Io(err))) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - Ok(Err(VaultError::Serde(err))) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - Ok(Err(VaultError::NotFound(_))) => ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "secret unexpectedly missing", - ) - .into_response(), - Err(err) => ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("secret write task failed: {err}"), - ) - .into_response(), - } -} - -async fn delete_secret_by_name( - _auth: RequiredUser, - State(state): State>, - Json(body): Json, -) -> Response { - let name = body.name; - let state_for_write = Arc::clone(&state); - let result = spawn_blocking(move || { - let mut vault = state_for_write.vault.blocking_write(); - vault.remove(&name) - }) - .await; - - match result { - Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(), - Ok(Err(VaultError::InvalidName(_))) => { - ApiError::bad_request("invalid secret name").into_response() - } - Ok(Err(VaultError::NotFound(name))) => { - ApiError::new(StatusCode::NOT_FOUND, format!("secret not found: {name}")) - .into_response() - } - Ok(Err(VaultError::Io(err))) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - Ok(Err(VaultError::Serde(err))) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - Err(err) => ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("secret delete task failed: {err}"), - ) - .into_response(), - } -} - -#[derive(serde::Deserialize)] -struct GitHubRepoResponse { - default_branch: String, - private: bool, - permissions: Option, -} - -/// Reject owner/repo path segments that could rewrite the GitHub API endpoint -/// via `..` traversal after URL normalization. Conservative compared to -/// GitHub's real rules, which is fine for server-side input validation. -#[allow( - clippy::result_large_err, - reason = "GitHub slug validation returns HTTP 400 responses directly." -)] -fn validate_github_slug(kind: &str, value: &str, max_len: usize) -> Result<(), Response> { - if value.is_empty() || value.len() > max_len || matches!(value, "." | "..") { - return Err(ApiError::bad_request(format!("invalid github {kind}")).into_response()); - } - if !value - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) - { - return Err(ApiError::bad_request(format!("invalid github {kind}")).into_response()); - } - Ok(()) -} - -async fn get_github_repo( - _auth: RequiredUser, - State(state): State>, - Path((owner, name)): Path<(String, String)>, -) -> Response { - if let Err(response) = validate_github_slug("owner", &owner, 39) { - return response; - } - if let Err(response) = validate_github_slug("repo", &name, 100) { - return response; - } - let settings = state.server_settings(); - let github_settings = &settings.server.integrations.github; - let base_url = fabro_github::github_api_base_url(); - let mut client: Option = None; - let token = match github_settings.strategy { - GithubIntegrationStrategy::App => { - let Some(app_id) = github_settings.app_id.as_ref() else { - return ApiError::new( - StatusCode::SERVICE_UNAVAILABLE, - "server.integrations.github.app_id is not configured", - ) - .into_response(); - }; - if let Err(err) = resolve_interp_string(app_id) { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) - .into_response(); - } - let creds = match state.github_credentials(github_settings) { - Ok(Some(fabro_github::GitHubCredentials::App(creds))) => creds, - Ok(Some(_)) => unreachable!("app strategy should not return token credentials"), - Ok(None) => { - return ApiError::new( - StatusCode::SERVICE_UNAVAILABLE, - "GITHUB_APP_PRIVATE_KEY is not configured", - ) - .into_response(); - } - Err(err) => { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err).into_response(); - } - }; - - let jwt = match fabro_github::sign_app_jwt(&creds.app_id, &creds.private_key_pem) { - Ok(jwt) => jwt, - Err(err) => { - tracing::error!(error = ?err, "failed to sign GitHub App JWT"); - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) - .into_response(); - } - }; - let install_url = match github_settings.slug.as_ref() { - Some(slug) => match resolve_interp_string(slug) { - Ok(slug) => format!("https://github.com/apps/{slug}/installations/new"), - Err(err) => { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) - .into_response(); - } - }, - None => format!("https://github.com/organizations/{owner}/settings/installations"), - }; - - if client.is_none() { - client = Some(match state.http_client() { - Ok(http) => http, - Err(err) => { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) - .into_response(); - } - }); - } - let client_ref = client.as_ref().expect("client initialized above"); - let installed = - match fabro_github::check_app_installed(client_ref, &jwt, &owner, &name, &base_url) - .await - { - Ok(installed) => installed, - Err(err) => { - tracing::error!(error = ?err, "failed to check GitHub App installation"); - return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()) - .into_response(); - } - }; - - if !installed { - return ( - StatusCode::OK, - Json(serde_json::json!({ - "owner": owner, - "name": name, - "accessible": false, - "default_branch": null, - "private": null, - "permissions": null, - "install_url": install_url, - })), - ) - .into_response(); - } - - match fabro_github::create_installation_access_token_with_permissions_and_install_url( - client_ref, - &jwt, - &owner, - &name, - &base_url, - serde_json::json!({ "contents": "write", "pull_requests": "write" }), - Some(&install_url), - ) - .await - { - Ok(token) => token, - Err(err) => { - tracing::error!( - error = ?err, - "failed to create GitHub App installation token" - ); - return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(); - } - } - } - GithubIntegrationStrategy::Token => match state.github_credentials(github_settings) { - Ok(Some(fabro_github::GitHubCredentials::Token(token))) => token, - Ok(Some(_)) => unreachable!("token strategy should not return app credentials"), - Ok(None) => { - return ApiError::new( - StatusCode::SERVICE_UNAVAILABLE, - "GITHUB_TOKEN is not configured", - ) - .into_response(); - } - Err(err) => { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err).into_response(); - } - }, - }; - - let client = match client { - Some(client) => client, - None => match state.http_client() { - Ok(http) => http, - Err(err) => { - return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) - .into_response(); - } - }, - }; - let repo_response = match client - .get(format!("{base_url}/repos/{owner}/{name}")) - .header("Authorization", format!("Bearer {token}")) - .header("Accept", "application/vnd.github+json") - .header("User-Agent", "fabro-server") - .send() - .await - { - Ok(response) if response.status().is_success() => response, - Ok(response) - if github_settings.strategy == GithubIntegrationStrategy::Token - && matches!( - response.status(), - fabro_http::StatusCode::FORBIDDEN | fabro_http::StatusCode::NOT_FOUND - ) => - { - return ( - StatusCode::OK, - Json(serde_json::json!({ - "owner": owner, - "name": name, - "accessible": false, - "default_branch": null, - "private": null, - "permissions": null, - "install_url": serde_json::Value::Null, - })), - ) - .into_response(); - } - Ok(response) - if github_settings.strategy == GithubIntegrationStrategy::Token - && response.status() == fabro_http::StatusCode::UNAUTHORIZED => - { - return ApiError::new( - StatusCode::SERVICE_UNAVAILABLE, - "Stored GitHub token is invalid — run fabro install or update GITHUB_TOKEN", - ) - .into_response(); - } - Ok(response) => { - return ApiError::new( - StatusCode::BAD_GATEWAY, - format!("GitHub repo lookup failed: {}", response.status()), - ) - .into_response(); - } - Err(err) => return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), - }; - - let repo = match repo_response.json::().await { - Ok(repo) => repo, - Err(err) => { - return ApiError::new( - StatusCode::BAD_GATEWAY, - format!("Failed to parse GitHub repo response: {err}"), - ) - .into_response(); - } - }; - - ( - StatusCode::OK, - Json(serde_json::json!({ - "owner": owner, - "name": name, - "accessible": true, - "default_branch": repo.default_branch, - "private": repo.private, - "permissions": repo.permissions, - "install_url": serde_json::Value::Null, - })), - ) - .into_response() -} - -async fn run_diagnostics(_auth: RequiredUser, State(state): State>) -> Response { - ( - StatusCode::OK, - Json(diagnostics::run_all(state.as_ref()).await), - ) - .into_response() -} - -async fn openapi_spec() -> Response { - let yaml = include_str!("../../../../docs/public/api-reference/fabro-api.yaml"); - let value: serde_json::Value = - serde_yaml::from_str(yaml).expect("embedded OpenAPI YAML is invalid"); - Json(value).into_response() -} - -fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState { - let latest = events.iter().rev().find(|envelope| { - envelope.event.node_id.as_deref() == Some(node_id) - && matches!( - envelope.event.event_name(), - "stage.retrying" | "stage.started" | "stage.completed" | "stage.failed" - ) - }); - - if latest.is_some_and(|e| e.event.event_name() == "stage.retrying") { - StageState::Retrying - } else { - StageState::Running - } -} - -async fn get_aggregate_billing( - _auth: RequiredUser, - State(state): State>, -) -> Response { - let agg = state - .aggregate_billing - .lock() - .expect("aggregate_billing lock poisoned"); - let by_model: Vec = agg - .by_model - .iter() - .map(|(model, totals)| BillingByModel { - billing: totals.billing.clone(), - model: ModelReference { id: model.clone() }, - stages: totals.stages, - }) - .collect(); - let total_billing = - agg.by_model - .values() - .fold(BilledTokenCounts::default(), |mut acc, totals| { - let billing = &totals.billing; - acc.input_tokens += billing.input_tokens; - acc.output_tokens += billing.output_tokens; - acc.reasoning_tokens += billing.reasoning_tokens; - acc.cache_read_tokens += billing.cache_read_tokens; - acc.cache_write_tokens += billing.cache_write_tokens; - acc.total_tokens += billing.total_tokens; - if let Some(value) = billing.total_usd_micros { - *acc.total_usd_micros.get_or_insert(0) += value; - } - acc - }); - let response = AggregateBilling { - totals: AggregateBillingTotals { - cache_read_tokens: total_billing.cache_read_tokens, - cache_write_tokens: total_billing.cache_write_tokens, - input_tokens: total_billing.input_tokens, - output_tokens: total_billing.output_tokens, - reasoning_tokens: total_billing.reasoning_tokens, - runs: agg.total_runs, - runtime_secs: agg.total_runtime_secs, - total_tokens: total_billing.total_tokens, - total_usd_micros: total_billing.total_usd_micros, - }, - by_model, - }; - (StatusCode::OK, Json(response)).into_response() -} - -async fn list_run_stages( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(_pagination): Query, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - - // Try live run first. - let (checkpoint, run_is_active) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => { - let active = !matches!( - managed_run.status, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ); - (managed_run.checkpoint.clone(), active) - } - None => (None, false), - } - }; - - // Fall back to stored run. - let (checkpoint, run_is_active) = if checkpoint.is_some() { - (checkpoint, run_is_active) - } else { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => { - let active = run_state.status.is_some_and(|status| !status.is_terminal()); - (run_state.checkpoint, active) - } - Err(_) => (None, false), - }, - Err(_) => return ApiError::not_found("Run not found.").into_response(), - } - }; - - let Some(checkpoint) = checkpoint else { - return ( - StatusCode::OK, - Json(ListResponse::new(Vec::::new())), - ) - .into_response(); - }; - - let events = match state.store.open_run_reader(&id).await { - Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => Vec::new(), - }; - let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events); - - let mut stages = Vec::new(); - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - let status = match checkpoint.node_outcomes.get(node_id) { - Some(outcome) => StageState::from(outcome.status), - None => StageState::Succeeded, - }; - stages.push(RunStage { - id: node_id.clone(), - name: node_id.clone(), - status, - duration_secs: Some(duration_ms as f64 / 1000.0), - dot_id: Some(node_id.clone()), - }); - } - - // Add next node as running if the run is still active. - // The checkpoint's current_node is the last *completed* stage; next_node_id - // is the stage that is currently executing. - if let Some(next_id) = &checkpoint.next_node_id { - if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) { - stages.push(RunStage { - id: next_id.clone(), - name: next_id.clone(), - status: active_stage_state_from_events(&events, next_id), - duration_secs: None, - dot_id: Some(next_id.clone()), - }); - } - } - - (StatusCode::OK, Json(ListResponse::new(stages))).into_response() -} - -async fn get_run_billing( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let run_store = match state.store.open_run_reader(&id).await { - Ok(run_store) => run_store, - Err(err) => { - return ApiError::new(StatusCode::NOT_FOUND, err.to_string()).into_response(); - } - }; - - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let Some(checkpoint) = checkpoint else { - let empty = RunBilling { - by_model: Vec::new(), - stages: Vec::new(), - totals: RunBillingTotals { - cache_read_tokens: 0, - cache_write_tokens: 0, - input_tokens: 0, - output_tokens: 0, - reasoning_tokens: 0, - runtime_secs: 0.0, - total_tokens: 0, - total_usd_micros: None, - }, - }; - return (StatusCode::OK, Json(empty)).into_response(); - }; - - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let mut by_model_totals = HashMap::::new(); - let mut billed_usages = Vec::new(); - let mut runtime_secs = 0.0_f64; - let mut stages = Vec::new(); - - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - runtime_secs += duration_ms as f64 / 1000.0; - - let Some(usage) = checkpoint - .node_outcomes - .get(node_id) - .and_then(|outcome| outcome.usage.as_ref()) - else { - continue; - }; - - billed_usages.push(usage.clone()); - let tokens = usage.tokens(); - let billing = BilledTokenCounts { - cache_read_tokens: tokens.cache_read_tokens, - cache_write_tokens: tokens.cache_write_tokens, - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - reasoning_tokens: tokens.reasoning_tokens, - total_tokens: tokens.total_tokens(), - total_usd_micros: usage.total_usd_micros, - }; - let model_id = usage.model_id().to_string(); - accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); - stages.push(RunBillingStage { - billing, - model: ModelReference { id: model_id }, - runtime_secs: duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: node_id.clone(), - name: node_id.clone(), - }, - }); - } - - let totals = BilledTokenCounts::from_billed_usage(&billed_usages); - let by_model = by_model_totals - .into_iter() - .map(|(model, totals)| BillingByModel { - billing: totals.billing, - model: ModelReference { id: model }, - stages: totals.stages, - }) - .collect::>(); - - let response = RunBilling { - by_model, - stages, - totals: RunBillingTotals { - cache_read_tokens: totals.cache_read_tokens, - cache_write_tokens: totals.cache_write_tokens, - input_tokens: totals.input_tokens, - output_tokens: totals.output_tokens, - reasoning_tokens: totals.reasoning_tokens, - runtime_secs, - total_tokens: totals.total_tokens, - total_usd_micros: totals.total_usd_micros, - }, - }; - - (StatusCode::OK, Json(response)).into_response() -} - fn worker_token_keys_from_server_secrets( server_secrets: &ServerSecrets, ) -> anyhow::Result { @@ -2616,265 +1439,8 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result Option<&'static str> { - match status { - RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"), - RunStatus::Running | RunStatus::Paused { .. } => Some("running"), - RunStatus::Blocked { .. } => Some("blocked"), - RunStatus::Succeeded { .. } => Some("succeeded"), - RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"), - RunStatus::Removing | RunStatus::Archived { .. } => None, - } -} - -pub(crate) fn board_columns() -> serde_json::Value { - serde_json::json!([ - {"id": "initializing", "name": "Initializing"}, - {"id": "running", "name": "Running"}, - {"id": "blocked", "name": "Blocked"}, - {"id": "succeeded", "name": "Succeeded"}, - {"id": "failed", "name": "Failed"}, - ]) -} - -async fn board_run_metadata( - state: &AppState, - run_id: RunId, -) -> serde_json::Map { - let mut metadata = serde_json::Map::new(); - let Ok(run_store) = state.store.open_run_reader(&run_id).await else { - return metadata; - }; - let Ok(run_state) = run_store.state().await else { - return metadata; - }; - - if let Some(pull_request) = run_state.pull_request { - metadata.insert( - "pull_request".to_string(), - serde_json::json!({ - "number": pull_request.number, - }), - ); - } - - if let Some(sandbox) = run_state.sandbox { - let mut sandbox_metadata = serde_json::Map::new(); - sandbox_metadata.insert( - "working_directory".to_string(), - serde_json::json!(sandbox.working_directory), - ); - if let Some(identifier) = sandbox.identifier { - sandbox_metadata.insert("id".to_string(), serde_json::json!(identifier)); - } - metadata.insert( - "sandbox".to_string(), - serde_json::Value::Object(sandbox_metadata), - ); - } - - if let Some((_, record)) = - run_state - .pending_interviews - .iter() - .min_by(|(left_id, left), (right_id, right)| { - left.started_at - .cmp(&right.started_at) - .then_with(|| left_id.cmp(right_id)) - }) - { - metadata.insert( - "question".to_string(), - serde_json::json!({ - "text": record.question.text, - }), - ); - } - - metadata -} - const MAX_PAGE_OFFSET: u32 = 1_000_000; -fn paginate_items(items: Vec, pagination: &PaginationParams) -> (Vec, bool) { - let limit = pagination.limit.clamp(1, 100) as usize; - let offset = pagination.offset.min(MAX_PAGE_OFFSET) as usize; - let mut data: Vec<_> = items.into_iter().skip(offset).take(limit + 1).collect(); - let has_more = data.len() > limit; - data.truncate(limit); - (data, has_more) -} - -async fn list_board_runs( - _auth: RequiredUser, - State(state): State>, - Query(pagination): Query, -) -> Response { - let summaries = match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(runs) => runs, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let board_summaries: Vec<_> = summaries - .into_iter() - .filter_map(|summary| { - let column = board_column(summary.status)?; - Some((summary, column)) - }) - .collect(); - let (page_summaries, has_more) = paginate_items(board_summaries, &pagination); - - let mut data = Vec::with_capacity(page_summaries.len()); - for (summary, column) in page_summaries { - let run_id = summary.run_id; - let mut item = - serde_json::to_value(&summary).expect("RunSummary serialization is infallible"); - item["column"] = serde_json::json!(column); - if let Some(object) = item.as_object_mut() { - object.extend(board_run_metadata(state.as_ref(), run_id).await); - } - data.push(item); - } - ( - StatusCode::OK, - Json(serde_json::json!({ - "columns": board_columns(), - "data": data, - "meta": { "has_more": has_more } - })), - ) - .into_response() -} - -async fn list_runs( - _auth: RequiredUser, - State(state): State>, - Query(params): Query, -) -> Response { - match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(runs) => { - let include_archived = params.include_archived; - let items = runs - .into_iter() - .filter(|summary| { - include_archived || !matches!(summary.status, RunStatus::Archived { .. }) - }) - .collect::>(); - let (data, has_more) = paginate_items(items, ¶ms.pagination()); - ( - StatusCode::OK, - Json(serde_json::json!({ - "data": data, - "meta": { "has_more": has_more } - })), - ) - .into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -#[derive(Debug, serde::Deserialize)] -struct ResolveRunQuery { - selector: String, -} - -#[derive(Debug, Default, serde::Deserialize)] -struct DeleteRunQuery { - #[serde(default)] - force: bool, -} - -fn default_command_log_limit() -> u64 { - 65_536 -} - -#[derive(Debug, serde::Deserialize)] -struct CommandLogQuery { - #[serde(default)] - offset: u64, - #[serde(default = "default_command_log_limit")] - limit: u64, -} - -#[derive(Debug, serde::Serialize)] -struct CommandLogResponseBody { - stream: CommandOutputStream, - offset: u64, - next_offset: u64, - total_bytes: u64, - bytes_base64: String, - eof: bool, - cas_ref: Option, - live_streaming: bool, -} - -async fn resolve_run( - _auth: RequiredUser, - State(state): State>, - Query(query): Query, -) -> Response { - let runs = match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(runs) => runs, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - match resolve_run_by_selector( - &runs, - &query.selector, - |run| run.run_id.to_string(), - |run| run.workflow_slug.clone(), - |run| run.workflow_name.clone(), - |run| run.run_id.created_at(), - |run| run.run_id.created_at().to_rfc3339(), - |run| run.repo_origin_url.clone(), - ) { - Ok(run) => (StatusCode::OK, Json(run.clone())).into_response(), - Err(err @ (ResolveRunError::InvalidSelector | ResolveRunError::AmbiguousPrefix { .. })) => { - ApiError::bad_request(err.to_string()).into_response() - } - Err(err @ ResolveRunError::NotFound { .. }) => { - ApiError::not_found(err.to_string()).into_response() - } - } -} - -async fn delete_run( - _auth: RequiredUser, - State(state): State>, - Query(query): Query, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - - match delete_run_internal(&state, id, query.force).await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(response) => response, - } -} - async fn delete_run_internal( state: &Arc, id: RunId, @@ -3957,331 +2523,6 @@ fn answer_from_request( } } -async fn create_run( - RequestAuth(auth_slot): RequestAuth, - State(state): State>, - headers: HeaderMap, - body: Bytes, -) -> Response { - let subject = match require_user(&auth_slot) { - Ok(subject) => subject, - Err(err) => return err.into_response(), - }; - let req = match serde_json::from_slice::(&body) { - Ok(req) => req, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let manifest_run_defaults = state.manifest_run_defaults(); - let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { - Ok(prepared) => prepared, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let run_id = prepared.run_id.unwrap_or_else(RunId::new); - info!(run_id = %run_id, "Run created"); - - let configured_providers = state.llm_source.configured_providers().await; - let mut create_input = run_manifest::create_run_input(prepared.clone(), configured_providers); - create_input.run_id = Some(run_id); - create_input.provenance = Some(run_provenance(&headers, &subject)); - create_input.submitted_manifest_bytes = Some(body.to_vec()); - - let storage_root = match resolve_interp_string(&state.server_settings().server.storage.root) { - Ok(path) => PathBuf::from(path), - Err(err) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to resolve server storage root: {err}"), - ) - .into_response(); - } - }; - let created = match Box::pin(operations::create( - state.store.as_ref(), - create_input, - storage_root, - )) - .await - { - Ok(created) => created, - Err(WorkflowError::ValidationFailed { .. } | WorkflowError::Parse(_)) => { - return ApiError::bad_request("Validation failed").into_response(); - } - Err(err) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to persist run state: {err}"), - ) - .into_response(); - } - }; - let created_at = created.run_id.created_at(); - - { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - runs.insert( - created.run_id, - managed_run( - created.persisted.source().to_string(), - RunStatus::Submitted, - created_at, - created.run_dir, - RunExecutionMode::Start, - ), - ); - } - - ( - StatusCode::CREATED, - Json(RunStatusResponse { - id: run_id.to_string(), - status: RunStatus::Submitted, - error: None, - queue_position: None, - pending_control: None, - created_at, - }), - ) - .into_response() -} - -fn run_provenance(headers: &HeaderMap, subject: &UserPrincipal) -> RunProvenance { - RunProvenance { - server: Some(RunServerProvenance { - version: FABRO_VERSION.to_string(), - }), - client: run_client_provenance(headers), - subject: Some(Principal::User(subject.clone())), - } -} - -fn run_client_provenance(headers: &HeaderMap) -> Option { - let user_agent = headers - .get(header::USER_AGENT) - .and_then(|value| value.to_str().ok()) - .map(str::to_string)?; - let (name, version) = parse_known_fabro_user_agent(&user_agent) - .map_or((None, None), |(name, version)| { - (Some(name.to_string()), Some(version.to_string())) - }); - Some(RunClientProvenance { - user_agent: Some(user_agent), - name, - version, - }) -} - -fn parse_known_fabro_user_agent(user_agent: &str) -> Option<(&str, &str)> { - let token = user_agent.split_whitespace().next()?; - let (name, version) = token.split_once('/')?; - if version.is_empty() { - return None; - } - match name { - "fabro-cli" | "fabro-web" => Some((name, version)), - _ => None, - } -} - -async fn run_preflight( - _auth: RequiredUser, - State(state): State>, - Json(req): Json, -) -> Response { - let manifest_run_defaults = state.manifest_run_defaults(); - let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { - Ok(prepared) => prepared, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let validated = match run_manifest::validate_prepared_manifest(&prepared) { - Ok(validated) => validated, - Err(WorkflowError::Parse(_)) => { - return ApiError::bad_request("Validation failed").into_response(); - } - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let response = match run_manifest::run_preflight(&state, &prepared, &validated).await { - Ok((response, _ok)) => response, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - (StatusCode::OK, Json(response)).into_response() -} - -async fn validate_run_manifest( - _auth: RequiredUser, - State(state): State>, - Json(req): Json, -) -> Response { - let manifest_run_defaults = state.manifest_run_defaults(); - let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { - Ok(prepared) => prepared, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let validated = match run_manifest::validate_prepared_manifest(&prepared) { - Ok(validated) => validated, - Err(WorkflowError::Parse(_)) => { - return ApiError::bad_request("Validation failed").into_response(); - } - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - ( - StatusCode::OK, - Json(run_manifest::validate_response(&prepared, &validated)), - ) - .into_response() -} - -async fn render_graph_from_manifest( - _auth: RequiredUser, - State(state): State>, - Json(req): Json, -) -> Response { - let manifest_run_defaults = state.manifest_run_defaults(); - let prepared = - match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req.manifest) { - Ok(prepared) => prepared, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - let validated = match run_manifest::validate_prepared_manifest(&prepared) { - Ok(validated) => validated, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - if validated.has_errors() { - return ApiError::bad_request("Validation failed").into_response(); - } - - let direction = req.direction.as_ref().map(|direction| match direction { - RenderWorkflowGraphDirection::Lr => "LR", - RenderWorkflowGraphDirection::Tb => "TB", - }); - let dot_source = run_manifest::graph_source(&prepared, direction); - render_graph_bytes(&dot_source).await -} - -async fn start_run( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - body: Option>, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let resume = body.is_some_and(|Json(req)| req.resume); - - { - let runs = state.runs.lock().expect("runs lock poisoned"); - if let Some(managed_run) = runs.get(&id) { - if matches!( - managed_run.status, - RunStatus::Queued - | RunStatus::Starting - | RunStatus::Running - | RunStatus::Blocked { .. } - | RunStatus::Paused { .. } - ) { - return ApiError::new( - StatusCode::CONFLICT, - if resume { - "an engine process is still running for this run — cannot resume" - } else { - "an engine process is still running for this run — cannot start" - }, - ) - .into_response(); - } - } - } - - let Ok(run_store) = state.store.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let run_state = match run_store.state().await { - Ok(state) => state, - Err(err) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to load run state: {err}"), - ) - .into_response(); - } - }; - - if resume { - if run_state.checkpoint.is_none() { - return ApiError::new(StatusCode::CONFLICT, "no checkpoint to resume from") - .into_response(); - } - } else if let Some(status) = run_state.status { - if !matches!( - status, - RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting - ) { - return ApiError::new( - StatusCode::CONFLICT, - format!("cannot start run: status is {status}, expected submitted"), - ) - .into_response(); - } - } - - if run_state.spec.is_none() { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "run spec missing from store", - ) - .into_response(); - } - let run_dir = Storage::new(state.server_storage_dir()) - .run_scratch(&id) - .root() - .to_path_buf(); - let dot_source = run_state.graph_source.unwrap_or_default(); - if let Err(err) = - workflow_event::append_event(&run_store, &id, &workflow_event::Event::RunQueued).await - { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - - { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - runs.insert( - id, - managed_run( - dot_source, - RunStatus::Queued, - id.created_at(), - run_dir, - if resume { - RunExecutionMode::Resume - } else { - RunExecutionMode::Start - }, - ), - ); - } - - state.scheduler_notify.notify_one(); - ( - StatusCode::OK, - Json(RunStatusResponse { - id: id.to_string(), - status: RunStatus::Queued, - error: None, - queue_position: None, - pending_control: None, - created_at: id.created_at(), - }), - ) - .into_response() -} - /// Execute a single run: transitions queued → starting → running → /// completed/failed/cancelled. async fn execute_run(state: Arc, run_id: RunId) { @@ -4935,1744 +3176,6 @@ pub fn spawn_scheduler(state: Arc) { }); } -async fn get_run_status( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - match state - .store - .list_runs(&fabro_store::ListRunsQuery::default()) - .await - { - Ok(runs) => match runs.into_iter().find(|run| run.run_id == id) { - Some(run) => (StatusCode::OK, Json(run)).into_response(), - None => ApiError::not_found("Run not found.").into_response(), - }, - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn get_run_settings( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let run_store = match state.store.open_run_reader(&id).await { - Ok(store) => store, - Err(fabro_store::Error::RunNotFound(_)) => { - return ApiError::not_found("Run not found.").into_response(); - } - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let run_state = match run_store.state().await { - Ok(state) => state, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let Some(run_spec) = run_state.spec else { - return ApiError::not_found("Run not found.").into_response(); - }; - (StatusCode::OK, Json(run_spec.settings)).into_response() -} - -async fn get_questions( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => { - let questions = run_state - .pending_interviews - .values() - .map(api_question_from_pending_interview) - .collect::>(); - (StatusCode::OK, Json(ListResponse::new(questions))).into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(fabro_store::Error::RunNotFound(_)) => { - ApiError::not_found("Run not found.").into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn submit_answer( - auth: RequiredUser, - State(state): State>, - Path((id, qid)): Path<(String, String)>, - Json(req): Json, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let pending = match load_pending_interview(state.as_ref(), id, &qid).await { - Ok(pending) => pending, - Err(response) => return response, - }; - let answer = match answer_from_request(req, &pending.question) { - Ok(answer) => answer, - Err(response) => return response, - }; - let submission = AnswerSubmission::new(answer, Principal::User(auth.0)); - match submit_pending_interview_answer(state.as_ref(), &pending, submission).await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(response) => response, - } -} - -async fn get_run_state( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, -) -> Response { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => Json(run_state).into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn get_run_logs( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, -) -> Response { - if state.store.open_run_reader(&id).await.is_err() { - return ApiError::not_found("Run not found.").into_response(); - } - - let path = Storage::new(state.server_storage_dir()) - .run_scratch(&id) - .runtime_dir() - .join("server.log"); - match fs::read(&path).await { - Ok(bytes) => ([(header::CONTENT_TYPE, "text/plain; charset=utf-8")], bytes).into_response(), - Err(err) if err.kind() == ErrorKind::NotFound => { - ApiError::not_found("Run log not available.").into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn get_run_stage_command_log( - RequireCommandLog(id, stage_id, stream): RequireCommandLog, - State(state): State>, - Query(query): Query, -) -> Response { - const MAX_COMMAND_LOG_LIMIT: u64 = 1_048_576; - - if query.limit == 0 { - return ApiError::bad_request("limit must be greater than 0").into_response(); - } - let limit = query.limit.min(MAX_COMMAND_LOG_LIMIT); - let Ok(run_store) = state.store.open_run_reader(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let run_state = match run_store.state().await { - Ok(run_state) => run_state, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let Some(node) = run_state.stage(&stage_id) else { - return ApiError::not_found("Stage not found.").into_response(); - }; - - let stream_value = match stream { - CommandOutputStream::Stdout => node.stdout.as_deref(), - CommandOutputStream::Stderr => node.stderr.as_deref(), - }; - let cas_ref = stream_value - .filter(|value| parse_blob_ref(value).is_some()) - .map(str::to_string); - let live_streaming = node - .live_streaming - .unwrap_or_else(|| cas_ref.is_none() && node.completion.is_none()); - let run_dir = Storage::new(state.server_storage_dir()) - .run_scratch(&id) - .root() - .to_path_buf(); - let scratch_path = command_log_path(&run_dir, &stage_id, stream); - - match read_log_slice(&scratch_path, query.offset, limit).await { - Ok((bytes, total_bytes)) => { - return build_command_log_response( - stream, - query.offset, - limit, - LogSource::Sliced { bytes, total_bytes }, - cas_ref.is_some(), - cas_ref, - live_streaming, - ); - } - Err(err) if err.kind() == ErrorKind::NotFound => {} - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - - if let Some(cas_ref) = cas_ref { - let text = match read_json_string_blob(&run_store.clone().into(), &cas_ref).await { - Ok(Some(text)) => text, - Ok(None) => String::new(), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - return build_command_log_response( - stream, - query.offset, - limit, - LogSource::Full(text.as_bytes()), - true, - Some(cas_ref), - live_streaming, - ); - } - - if let Some(inline_text) = stream_value { - return build_command_log_response( - stream, - query.offset, - limit, - LogSource::Full(inline_text.as_bytes()), - true, - None, - live_streaming, - ); - } - - build_command_log_response( - stream, - query.offset, - limit, - LogSource::Full(&[]), - node.completion.is_some(), - None, - live_streaming, - ) -} - -enum LogSource<'a> { - Sliced { - bytes: Vec, - total_bytes: u64, - }, - Full(&'a [u8]), -} - -fn build_command_log_response( - stream: CommandOutputStream, - requested_offset: u64, - limit: u64, - source: LogSource<'_>, - eof: bool, - cas_ref: Option, - live_streaming: bool, -) -> Response { - let (body_bytes, total_bytes, offset) = match source { - LogSource::Sliced { bytes, total_bytes } => { - let offset = requested_offset.min(total_bytes); - (bytes, total_bytes, offset) - } - LogSource::Full(bytes) => { - let total_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - let offset = requested_offset.min(total_bytes); - let start = usize::try_from(offset).unwrap_or(bytes.len()); - let end = start - .saturating_add(usize::try_from(limit).unwrap_or(usize::MAX)) - .min(bytes.len()); - (bytes[start..end].to_vec(), total_bytes, offset) - } - }; - Json(CommandLogResponseBody { - stream, - offset, - next_offset: offset + u64::try_from(body_bytes.len()).unwrap_or(u64::MAX), - total_bytes, - bytes_base64: BASE64_STANDARD.encode(body_bytes), - eof, - cas_ref, - live_streaming, - }) - .into_response() -} - -#[expect( - clippy::disallowed_types, - reason = "Pull-request API validates public github.com URLs; these raw URLs are not credential-bearing log output." -)] -fn parse_github_owner_repo_from_url(url: &str, kind: &str) -> Result<(String, String), ApiError> { - let parsed = fabro_http::Url::parse(url) - .map_err(|err| ApiError::bad_request(format!("Invalid {kind}: {err}")))?; - match parsed.host_str() { - Some("github.com") => {} - Some(host) => { - return Err(ApiError::with_code( - StatusCode::BAD_REQUEST, - format!("Pull request operations support github.com only (got {host})."), - "unsupported_host", - )); - } - None => { - return Err(ApiError::bad_request(format!( - "Invalid {kind}: missing host" - ))); - } - } - - fabro_github::parse_github_owner_repo(url).map_err(|err| ApiError::bad_request(err.to_string())) -} - -fn load_server_github_credentials( - state: &AppState, -) -> Result { - let settings = state.server_settings(); - match state.github_credentials(&settings.server.integrations.github) { - Ok(Some(creds)) => Ok(creds), - Ok(None) => { - warn!("GitHub integration unavailable on server: credentials not configured"); - Err(ApiError::with_code( - StatusCode::SERVICE_UNAVAILABLE, - "GitHub integration unavailable on server.", - "integration_unavailable", - )) - } - Err(err) => { - warn!(error = %err, "GitHub integration unavailable on server"); - Err(ApiError::with_code( - StatusCode::SERVICE_UNAVAILABLE, - "GitHub integration unavailable on server.", - "integration_unavailable", - )) - } - } -} - -fn server_github_context<'a>( - state: &'a AppState, - creds: &'a fabro_github::GitHubCredentials, -) -> Result, ApiError> { - let http_client = state.http_client().map_err(|err| { - ApiError::with_code( - StatusCode::SERVICE_UNAVAILABLE, - format!("GitHub integration unavailable on server: {err}"), - "integration_unavailable", - ) - })?; - Ok(fabro_github::GitHubContext::with_http_client( - creds, - state.github_api_base_url.as_str(), - http_client, - )) -} - -fn github_pull_request_not_found_error(record: &PullRequestRecord) -> ApiError { - ApiError::with_code( - StatusCode::BAD_GATEWAY, - format!("Pull request #{} was deleted on GitHub.", record.number), - "github_not_found", - ) -} - -struct PullRequestGithubContext { - record: PullRequestRecord, - creds: fabro_github::GitHubCredentials, -} - -async fn load_pull_request_github_context( - state: &Arc, - id: &RunId, -) -> Result { - let run_store = state - .store - .open_run_reader(id) - .await - .map_err(|_| ApiError::not_found("Run not found."))?; - let run_state = run_store - .state() - .await - .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; - let record = run_state.pull_request.ok_or_else(|| { - ApiError::with_code( - StatusCode::NOT_FOUND, - format!("No pull request found in store. Create one first with: fabro pr create {id}"), - "no_stored_record", - ) - })?; - parse_github_owner_repo_from_url(&record.html_url, "pull request URL")?; - let creds = load_server_github_credentials(state.as_ref())?; - Ok(PullRequestGithubContext { record, creds }) -} - -struct RunPrInputs<'a> { - goal: &'a str, - base_branch: &'a str, - run_branch: &'a str, - diff: &'a str, - conclusion: &'a fabro_types::Conclusion, - normalized_origin: String, -} - -impl<'a> RunPrInputs<'a> { - fn extract(run_state: &'a fabro_store::RunProjection, force: bool) -> Result { - if let Some(record) = run_state.pull_request.as_ref() { - return Err(ApiError::with_code( - StatusCode::CONFLICT, - format!("Pull request already exists at {}", record.html_url), - "pull_request_exists", - )); - } - let run_spec = run_state.spec.as_ref().ok_or_else(|| { - ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "Run spec missing from store.", - ) - })?; - let origin_url = run_spec.repo_origin_url().ok_or_else(|| { - ApiError::with_code( - StatusCode::BAD_REQUEST, - "Run has no repo origin URL — pull request creation requires git metadata.", - "missing_repo_origin", - ) - })?; - let base_branch = run_spec.base_branch().ok_or_else(|| { - ApiError::with_code( - StatusCode::BAD_REQUEST, - "Run has no base branch — pull request creation requires git metadata.", - "missing_base_branch", - ) - })?; - let run_branch = run_state - .start - .as_ref() - .and_then(|start| start.run_branch.as_deref()) - .ok_or_else(|| { - ApiError::with_code( - StatusCode::BAD_REQUEST, - "Run has no run_branch — was it run with git push enabled?", - "missing_run_branch", - ) - })?; - let diff = run_state - .final_patch - .as_deref() - .filter(|d| !d.trim().is_empty()) - .ok_or_else(|| { - ApiError::with_code( - StatusCode::BAD_REQUEST, - "Stored diff is empty — nothing to create a PR for", - "empty_diff", - ) - })?; - let conclusion = run_state.conclusion.as_ref().ok_or_else(|| { - ApiError::with_code( - StatusCode::BAD_REQUEST, - "Run is not finished yet.", - "run_not_finished", - ) - })?; - if !force && !conclusion.status.is_successful() { - return Err(ApiError::with_code( - StatusCode::BAD_REQUEST, - format!( - "Run status is '{}', expected succeeded or partially_succeeded", - conclusion.status - ), - "run_not_successful", - )); - } - let normalized_origin = fabro_github::normalize_repo_origin_url(origin_url); - parse_github_owner_repo_from_url(&normalized_origin, "repo origin URL")?; - Ok(Self { - goal: run_spec.graph.goal(), - base_branch, - run_branch, - diff, - conclusion, - normalized_origin, - }) - } -} - -async fn create_run_pull_request( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Json(body): Json, -) -> Response { - let _create_guard = lock_pull_request_create(&state.pull_request_create_locks, &id).await; - let Ok(run_store) = state.store.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let run_state = match run_store.state().await { - Ok(run_state) => run_state, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let inputs = match RunPrInputs::extract(&run_state, body.force) { - Ok(inputs) => inputs, - Err(err) => return err.into_response(), - }; - let creds = match load_server_github_credentials(state.as_ref()) { - Ok(creds) => creds, - Err(err) => return err.into_response(), - }; - let github = match server_github_context(state.as_ref(), &creds) { - Ok(ctx) => ctx, - Err(err) => return err.into_response(), - }; - let model = if let Some(model) = body.model { - model - } else { - let configured = state.llm_source.configured_providers().await; - Catalog::builtin() - .default_for_configured(&configured) - .id - .clone() - }; - - let run_store_handle = run_store.clone().into(); - let request = pull_request::OpenPullRequestRequest { - github, - origin_url: &inputs.normalized_origin, - base_branch: inputs.base_branch, - head_branch: inputs.run_branch, - goal: inputs.goal, - diff: inputs.diff, - model: &model, - draft: true, - auto_merge: None, - run_store: &run_store_handle, - llm_source: state.llm_source.as_ref(), - conclusion: Some(inputs.conclusion), - run_state: Some(&run_state), - }; - let pull_request = match pull_request::maybe_open_pull_request(request).await { - Ok(Some(record)) => record, - Ok(None) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "Pull request creation returned no record unexpectedly.", - ) - .into_response(); - } - Err(err) => return ApiError::new(StatusCode::BAD_GATEWAY, err).into_response(), - }; - - let event = workflow_event::Event::pull_request_created(&pull_request, true); - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - - Json(pull_request).into_response() -} - -async fn get_run_pull_request( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, -) -> Response { - let ctx = match load_pull_request_github_context(&state, &id).await { - Ok(ctx) => ctx, - Err(err) => return err.into_response(), - }; - let github = match server_github_context(state.as_ref(), &ctx.creds) { - Ok(github) => github, - Err(err) => return err.into_response(), - }; - - match fabro_github::get_pull_request( - &github, - &ctx.record.owner, - &ctx.record.repo, - ctx.record.number, - ) - .await - { - Ok(github) => Json(fabro_types::PullRequestDetail { - record: ctx.record, - github, - }) - .into_response(), - Err(fabro_github::PullRequestApiError::NotFound { .. }) => { - github_pull_request_not_found_error(&ctx.record).into_response() - } - Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), - } -} - -async fn merge_run_pull_request( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Json(body): Json, -) -> Response { - let ctx = match load_pull_request_github_context(&state, &id).await { - Ok(ctx) => ctx, - Err(err) => return err.into_response(), - }; - let github = match server_github_context(state.as_ref(), &ctx.creds) { - Ok(github) => github, - Err(err) => return err.into_response(), - }; - - match fabro_github::merge_pull_request( - &github, - &ctx.record.owner, - &ctx.record.repo, - ctx.record.number, - body.method, - ) - .await - { - Ok(()) => Json(MergeRunPullRequestResponse { - number: i64::try_from(ctx.record.number) - .expect("stored pull request number should fit in i64"), - html_url: ctx.record.html_url, - method: body.method, - }) - .into_response(), - Err(fabro_github::PullRequestApiError::NotFound { .. }) => { - github_pull_request_not_found_error(&ctx.record).into_response() - } - Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), - } -} - -async fn close_run_pull_request( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, -) -> Response { - let ctx = match load_pull_request_github_context(&state, &id).await { - Ok(ctx) => ctx, - Err(err) => return err.into_response(), - }; - let github = match server_github_context(state.as_ref(), &ctx.creds) { - Ok(github) => github, - Err(err) => return err.into_response(), - }; - - match fabro_github::close_pull_request( - &github, - &ctx.record.owner, - &ctx.record.repo, - ctx.record.number, - ) - .await - { - Ok(()) => Json(CloseRunPullRequestResponse { - number: i64::try_from(ctx.record.number) - .expect("stored pull request number should fit in i64"), - html_url: ctx.record.html_url, - }) - .into_response(), - Err(fabro_github::PullRequestApiError::NotFound { .. }) => { - github_pull_request_not_found_error(&ctx.record).into_response() - } - Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), - } -} - -async fn append_run_event( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Json(value): Json, -) -> Response { - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let event = match RunEvent::from_value(value.clone()) { - Ok(event) => event, - Err(err) => { - return ApiError::bad_request(format!("Invalid run event: {err}")).into_response(); - } - }; - if event.run_id != id { - return ApiError::bad_request("Event run_id does not match path run ID.").into_response(); - } - if let Some(denied) = denied_lifecycle_event_name(&event.body) { - return ApiError::bad_request(format!( - "{denied} is a lifecycle event; clients must call the corresponding operation endpoint instead of injecting it via append_run_event" - )) - .into_response(); - } - let payload = match EventPayload::new(value, &id) { - Ok(payload) => payload, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - - match state.store.open_run(&id).await { - Ok(run_store) => match run_store.append_event(&payload).await { - Ok(seq) => { - update_live_run_from_event(&state, id, &event); - Json(AppendEventResponse { - seq: i64::from(seq), - }) - .into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn list_run_events( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Query(params): Query, -) -> Response { - let since_seq = params.since_seq(); - let limit = params.limit(); - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store - .list_events_from_with_limit(since_seq, limit) - .await - { - Ok(mut events) => { - let has_more = events.len() > limit; - events.truncate(limit); - Json(PaginatedEventList { - data: events, - meta: PaginationMeta { has_more }, - }) - .into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn attach_run_events( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, -) -> Response { - const ATTACH_REPLAY_BATCH_LIMIT: usize = 256; - - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let Ok(run_store) = state.store.open_run_reader(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let start_seq = match params.since_seq { - Some(seq) if seq >= 1 => seq, - Some(_) => 1, - None => match run_store.list_events().await { - Ok(events) => events.last().map_or(1, |event| event.seq.saturating_add(1)), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }, - }; - let (sender, receiver) = mpsc::unbounded_channel(); - tokio::spawn(async move { - let mut next_seq = start_seq; - - loop { - let Ok(replay_batch) = run_store - .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) - .await - else { - return; - }; - let replay_has_more = replay_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; - - for event in replay_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { - next_seq = event.seq.saturating_add(1); - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - - if replay_has_more { - continue; - } - - let Ok(state) = run_store.state().await else { - return; - }; - - if run_projection_is_active(&state) { - break; - } - - let Ok(tail_batch) = run_store - .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) - .await - else { - return; - }; - let tail_has_more = tail_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; - - for event in tail_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { - next_seq = event.seq.saturating_add(1); - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - - if tail_has_more { - continue; - } - - return; - } - - let Ok(mut live_stream) = run_store.watch_events_from(next_seq) else { - return; - }; - - while let Some(result) = live_stream.next().await { - let Ok(event) = result else { - return; - }; - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - }); - - Sse::new(UnboundedReceiverStream::new(receiver)) - .keep_alive(KeepAlive::default()) - .into_response() -} - -async fn get_checkpoint( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let live_checkpoint = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => managed_run.checkpoint.clone(), - None => return ApiError::not_found("Run not found.").into_response(), - } - }; - if let Some(cp) = live_checkpoint { - return (StatusCode::OK, Json(cp)).into_response(); - } - - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => match run_state.checkpoint { - Some(cp) => (StatusCode::OK, Json(cp)).into_response(), - None => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), - }, - Err(err) => { - tracing::warn!(run_id = %id, error = %err, "Failed to load checkpoint state from store"); - (StatusCode::OK, Json(serde_json::json!(null))).into_response() - } - }, - Err(err) => { - tracing::warn!(run_id = %id, error = %err, "Failed to open run store reader"); - ApiError::not_found("Run not found.").into_response() - } - } -} - -async fn write_run_blob( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - body: Bytes, -) -> Response { - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - match state.store.open_run(&id).await { - Ok(run_store) => match run_store.write_blob(&body).await { - Ok(blob_id) => Json(WriteBlobResponse { - id: blob_id.to_string(), - }) - .into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn read_run_blob( - RequireRunBlob(id, blob_id): RequireRunBlob, - State(state): State>, -) -> Response { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.read_blob(&blob_id).await { - Ok(Some(bytes)) => octet_stream_response(bytes), - Ok(None) => ApiError::not_found("Blob not found.").into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn load_run_spec(state: &AppState, run_id: &RunId) -> Result { - let run_store = state - .store - .open_run_reader(run_id) - .await - .map_err(|_| ApiError::not_found("Run not found.").into_response())?; - let run_state = run_store.state().await.map_err(|err| { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - })?; - run_state.spec.ok_or_else(|| { - ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "run spec missing from store", - ) - .into_response() - }) -} - -async fn list_run_artifacts( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Err(response) = load_run_spec(state.as_ref(), &id).await { - return response; - } - - match state.artifact_store.list_for_run(&id).await { - Ok(entries) => Json(RunArtifactListResponse { - data: entries.into_iter().map(run_artifact_entry_from).collect(), - }) - .into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -fn run_artifact_entry_from(entry: NodeArtifact) -> RunArtifactEntry { - RunArtifactEntry { - stage_id: entry.node.to_string(), - node_slug: entry.node.node_id().to_string(), - retry: entry.retry.cast_signed(), - relative_path: entry.filename, - size: entry.size.cast_signed(), - } -} - -fn artifact_entry_from(entry: StageArtifactEntry) -> ArtifactEntry { - ArtifactEntry { - filename: entry.filename, - retry: entry.retry.cast_signed(), - size: entry.size.cast_signed(), - } -} - -async fn list_stage_artifacts( - _auth: RequiredUser, - State(state): State>, - Path((id, stage_id)): Path<(String, String)>, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let stage_id = match parse_stage_id_path(&stage_id) { - Ok(stage_id) => stage_id, - Err(response) => return response, - }; - if let Err(response) = load_run_spec(state.as_ref(), &id).await { - return response; - } - - match state.artifact_store.list_for_node(&id, &stage_id).await { - Ok(entries) => Json(ArtifactListResponse { - data: entries.into_iter().map(artifact_entry_from).collect(), - }) - .into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -enum ArtifactUploadContentType { - OctetStream, - Multipart { boundary: String }, -} - -struct ValidatedArtifactBatchEntry { - path: String, - sha256: Option, - expected_bytes: Option, -} - -#[allow( - clippy::result_large_err, - reason = "Upload content-type parsing returns HTTP client errors directly." -)] -fn artifact_upload_content_type( - headers: &HeaderMap, -) -> Result { - let value = headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .ok_or_else(|| { - ApiError::new( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "artifact uploads require a supported Content-Type", - ) - .into_response() - })?; - - let mime = value.split(';').next().unwrap_or(value).trim(); - match mime { - "application/octet-stream" => Ok(ArtifactUploadContentType::OctetStream), - "multipart/form-data" => multer::parse_boundary(value) - .map(|boundary| ArtifactUploadContentType::Multipart { boundary }) - .map_err(|err| bad_request_response(format!("invalid multipart boundary: {err}"))), - _ => Err(ApiError::new( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "artifact uploads only support application/octet-stream or multipart/form-data", - ) - .into_response()), - } -} - -#[allow( - clippy::result_large_err, - reason = "Content-Length parsing returns HTTP client errors directly." -)] -fn content_length_from_headers(headers: &HeaderMap) -> Result, Response> { - headers - .get(header::CONTENT_LENGTH) - .map(|value| { - value - .to_str() - .map_err(|err| { - bad_request_response(format!("invalid content-length header: {err}")) - }) - .and_then(|value| { - value.parse::().map_err(|err| { - bad_request_response(format!("invalid content-length header: {err}")) - }) - }) - }) - .transpose() -} - -#[allow( - clippy::result_large_err, - reason = "Multipart manifest parsing returns HTTP client errors directly." -)] -async fn read_multipart_manifest( - field: &mut multer::Field<'_>, -) -> Result { - let mut manifest_bytes = Vec::new(); - while let Some(chunk) = field - .chunk() - .await - .map_err(|err| bad_request_response(format!("invalid multipart body: {err}")))? - { - manifest_bytes.extend_from_slice(&chunk); - if manifest_bytes.len() > MAX_MULTIPART_MANIFEST_BYTES { - return Err(payload_too_large_response( - "multipart manifest exceeds the server limit", - )); - } - } - - serde_json::from_slice(&manifest_bytes) - .map_err(|err| bad_request_response(format!("invalid multipart manifest: {err}"))) -} - -#[allow( - clippy::result_large_err, - reason = "Artifact batch validation returns HTTP client errors directly." -)] -fn validate_artifact_batch_manifest( - manifest: ArtifactBatchUploadManifest, -) -> Result, Response> { - if manifest.entries.is_empty() { - return Err(bad_request_response( - "multipart manifest must include at least one artifact entry", - )); - } - if manifest.entries.len() > MAX_MULTIPART_ARTIFACTS { - return Err(payload_too_large_response(format!( - "multipart upload exceeds the {MAX_MULTIPART_ARTIFACTS} artifact limit" - ))); - } - - let mut entries = HashMap::with_capacity(manifest.entries.len()); - let mut seen_paths = HashSet::new(); - let mut expected_total_bytes = 0_u64; - - for entry in manifest.entries { - if entry.part.is_empty() { - return Err(bad_request_response( - "multipart manifest part names must not be empty", - )); - } - if entry.part == "manifest" { - return Err(bad_request_response( - "multipart manifest part name 'manifest' is reserved", - )); - } - let path = validate_relative_artifact_path("manifest path", &entry.path)?; - if !seen_paths.insert(path.clone()) { - return Err(bad_request_response(format!( - "duplicate artifact path in multipart manifest: {path}" - ))); - } - if let Some(sha256) = entry.sha256.as_ref() { - if sha256.len() != 64 || !sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(bad_request_response(format!( - "invalid sha256 for multipart part {}", - entry.part - ))); - } - } - if let Some(expected_bytes) = entry.expected_bytes { - if expected_bytes > MAX_SINGLE_ARTIFACT_BYTES { - return Err(payload_too_large_response(format!( - "artifact {path} exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" - ))); - } - expected_total_bytes = expected_total_bytes.saturating_add(expected_bytes); - if expected_total_bytes > MAX_MULTIPART_REQUEST_BYTES { - return Err(payload_too_large_response(format!( - "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" - ))); - } - } - if entries - .insert(entry.part.clone(), ValidatedArtifactBatchEntry { - path, - sha256: entry.sha256.map(|value| value.to_ascii_lowercase()), - expected_bytes: entry.expected_bytes, - }) - .is_some() - { - return Err(bad_request_response(format!( - "duplicate multipart part name in manifest: {}", - entry.part - ))); - } - } - - Ok(entries) -} - -async fn upload_stage_artifact_octet_stream( - state: &AppState, - run_id: &RunId, - stage_id: &StageId, - retry: u32, - filename: String, - body: Body, - content_length: Option, -) -> Response { - let relative_path = match validate_relative_artifact_path("filename", &filename) { - Ok(path) => path, - Err(response) => return response, - }; - - if content_length.is_some_and(|length| length > MAX_SINGLE_ARTIFACT_BYTES) { - return payload_too_large_response(format!( - "artifact exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" - )); - } - - let mut writer = match state.artifact_store.writer( - run_id, - &ArtifactKey::new(stage_id.clone(), retry, relative_path), - ) { - Ok(writer) => writer, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let mut bytes_written = 0_u64; - let mut data_stream = body.into_data_stream(); - while let Some(chunk) = data_stream.next().await { - let chunk = match chunk - .map_err(|err| bad_request_response(format!("invalid request body: {err}"))) - { - Ok(chunk) => chunk, - Err(response) => return response, - }; - bytes_written = - bytes_written.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX)); - if bytes_written > MAX_SINGLE_ARTIFACT_BYTES { - return payload_too_large_response(format!( - "artifact exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" - )); - } - if let Err(err) = writer.write_all(&chunk).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - - match writer.shutdown().await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn upload_stage_artifact_multipart( - state: &AppState, - run_id: &RunId, - stage_id: &StageId, - retry: u32, - boundary: String, - body: Body, -) -> Response { - let mut multipart = multer::Multipart::new(body.into_data_stream(), boundary); - let Some(mut manifest_field) = (match multipart - .next_field() - .await - .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) - { - Ok(field) => field, - Err(response) => return response, - }) else { - return bad_request_response("multipart upload must begin with a manifest part"); - }; - - if manifest_field.name() != Some("manifest") { - return bad_request_response("multipart upload must begin with a manifest part"); - } - - let manifest = match read_multipart_manifest(&mut manifest_field).await { - Ok(manifest) => manifest, - Err(response) => return response, - }; - drop(manifest_field); - let mut expected_parts = match validate_artifact_batch_manifest(manifest) { - Ok(entries) => entries, - Err(response) => return response, - }; - let mut total_bytes = 0_u64; - - while let Some(mut field) = match multipart - .next_field() - .await - .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) - { - Ok(field) => field, - Err(response) => return response, - } { - let Some(part_name) = field.name().map(ToOwned::to_owned) else { - return bad_request_response("multipart file parts must be named"); - }; - let Some(entry) = expected_parts.remove(&part_name) else { - return bad_request_response(format!("unexpected multipart part: {part_name}")); - }; - - let mut writer = match state.artifact_store.writer( - run_id, - &ArtifactKey::new(stage_id.clone(), retry, entry.path.clone()), - ) { - Ok(writer) => writer, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let mut bytes_written = 0_u64; - let mut sha256 = Sha256::new(); - - while let Some(chunk) = match field - .chunk() - .await - .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) - { - Ok(chunk) => chunk, - Err(response) => return response, - } { - let chunk_len = u64::try_from(chunk.len()).unwrap_or(u64::MAX); - bytes_written = bytes_written.saturating_add(chunk_len); - total_bytes = total_bytes.saturating_add(chunk_len); - - if bytes_written > MAX_SINGLE_ARTIFACT_BYTES { - return payload_too_large_response(format!( - "artifact {} exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit", - entry.path - )); - } - if total_bytes > MAX_MULTIPART_REQUEST_BYTES { - return payload_too_large_response(format!( - "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" - )); - } - - sha256.update(&chunk); - if let Err(err) = writer.write_all(&chunk).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - - if let Some(expected_bytes) = entry.expected_bytes { - if bytes_written != expected_bytes { - return bad_request_response(format!( - "multipart part {part_name} expected {expected_bytes} bytes but received {bytes_written}" - )); - } - } - if let Some(expected_sha256) = entry.sha256.as_ref() { - let actual_sha256 = hex::encode(sha256.finalize()); - if actual_sha256 != *expected_sha256 { - return bad_request_response(format!( - "multipart part {part_name} sha256 did not match manifest" - )); - } - } - - if let Err(err) = writer.shutdown().await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - - if !expected_parts.is_empty() { - let mut missing = expected_parts.into_keys().collect::>(); - missing.sort(); - return bad_request_response(format!( - "multipart upload is missing part(s): {}", - missing.join(", ") - )); - } - - StatusCode::NO_CONTENT.into_response() -} - -async fn put_stage_artifact( - State(state): State>, - RequireStageArtifact(id, stage_id): RequireStageArtifact, - Query(params): Query, - request: axum_extract::Request, -) -> Response { - let (parts, body) = request.into_parts(); - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - if let Err(response) = load_run_spec(state.as_ref(), &id).await.map(|_| ()) { - return response; - } - let retry = match required_query_param(params.retry.as_ref(), "retry") { - Ok(retry) => retry, - Err(response) => return response, - }; - - let content_length = match content_length_from_headers(&parts.headers) { - Ok(length) => length, - Err(response) => return response, - }; - match artifact_upload_content_type(&parts.headers) { - Ok(ArtifactUploadContentType::OctetStream) => { - let filename = match required_query_param(params.filename.as_ref(), "filename") { - Ok(filename) => filename, - Err(response) => return response, - }; - upload_stage_artifact_octet_stream( - state.as_ref(), - &id, - &stage_id, - retry, - filename, - body, - content_length, - ) - .await - } - Ok(ArtifactUploadContentType::Multipart { boundary }) => { - if content_length.is_some_and(|length| length > MAX_MULTIPART_REQUEST_BYTES) { - return payload_too_large_response(format!( - "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" - )); - } - upload_stage_artifact_multipart(state.as_ref(), &id, &stage_id, retry, boundary, body) - .await - } - Err(response) => response, - } -} - -async fn get_stage_artifact( - _auth: RequiredUser, - State(state): State>, - Path((id, stage_id)): Path<(String, String)>, - Query(params): Query, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let stage_id = match parse_stage_id_path(&stage_id) { - Ok(stage_id) => stage_id, - Err(response) => return response, - }; - let filename = match required_query_param(params.filename.as_ref(), "filename") { - Ok(filename) => filename, - Err(response) => return response, - }; - let retry = match required_query_param(params.retry.as_ref(), "retry") { - Ok(retry) => retry, - Err(response) => return response, - }; - let relative_path = match validate_relative_artifact_path("filename", &filename) { - Ok(path) => path, - Err(response) => return response, - }; - if let Err(response) = load_run_spec(state.as_ref(), &id).await { - return response; - } - - match state - .artifact_store - .get( - &id, - &ArtifactKey::new(stage_id.clone(), retry, relative_path), - ) - .await - { - Ok(Some(bytes)) => octet_stream_response(bytes), - Ok(None) => ApiError::not_found("Artifact not found.").into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn generate_preview_url( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Json(request): Json, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let Ok(port) = u16::try_from(request.port) else { - return ApiError::bad_request("Port must fit in a u16.").into_response(); - }; - let Ok(expires_in_secs) = i32::try_from(request.expires_in_secs.get()) else { - return ApiError::bad_request("Preview expiry exceeds supported range.").into_response(); - }; - - let sandbox = match reconnect_daytona_sandbox(&state, &id).await { - Ok(sandbox) => sandbox, - Err(response) => return response, - }; - - let response = if request.signed { - match sandbox - .get_signed_preview_url(port, Some(expires_in_secs)) - .await - { - Ok(preview) => PreviewUrlResponse { - token: None, - url: preview.url, - }, - Err(err) => { - return ApiError::new(StatusCode::CONFLICT, err.display_with_causes()) - .into_response(); - } - } - } else { - match sandbox.get_preview_link(port).await { - Ok(preview) => PreviewUrlResponse { - token: Some(preview.token), - url: preview.url, - }, - Err(err) => { - return ApiError::new(StatusCode::CONFLICT, err.display_with_causes()) - .into_response(); - } - } - }; - - (StatusCode::CREATED, Json(response)).into_response() -} - -async fn create_ssh_access( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Json(request): Json, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let sandbox = match reconnect_daytona_sandbox(&state, &id).await { - Ok(sandbox) => sandbox, - Err(response) => return response, - }; - match sandbox.create_ssh_access(Some(request.ttl_minutes)).await { - Ok(command) => (StatusCode::CREATED, Json(SshAccessResponse { command })).into_response(), - Err(err) => ApiError::new(StatusCode::CONFLICT, err.display_with_causes()).into_response(), - } -} - -async fn list_sandbox_files( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let sandbox = match reconnect_run_sandbox(&state, &id).await { - Ok(sandbox) => sandbox, - Err(response) => return response, - }; - match sandbox.list_directory(¶ms.path, params.depth).await { - Ok(entries) => Json(SandboxFileListResponse { - data: entries - .into_iter() - .map(|entry| SandboxFileEntry { - is_dir: entry.is_dir, - name: entry.name, - size: entry.size.map(u64::cast_signed), - }) - .collect(), - }) - .into_response(), - Err(err) => ApiError::new(StatusCode::NOT_FOUND, err.display_with_causes()).into_response(), - } -} - -async fn get_sandbox_file( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let sandbox = match reconnect_run_sandbox(&state, &id).await { - Ok(sandbox) => sandbox, - Err(response) => return response, - }; - let temp = match NamedTempFile::new() { - Ok(temp) => temp, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - if let Err(err) = sandbox - .download_file_to_local(¶ms.path, temp.path()) - .await - { - return ApiError::new(StatusCode::NOT_FOUND, err.display_with_causes()).into_response(); - } - match fs::read(temp.path()).await { - Ok(bytes) => octet_stream_response(bytes.into()), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -async fn put_sandbox_file( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, - body: Bytes, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let sandbox = match reconnect_run_sandbox(&state, &id).await { - Ok(sandbox) => sandbox, - Err(response) => return response, - }; - let temp = match NamedTempFile::new() { - Ok(temp) => temp, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - if let Err(err) = fs::write(temp.path(), &body).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - match sandbox - .upload_file_from_local(temp.path(), ¶ms.path) - .await - { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(err) => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.display_with_causes()) - .into_response(), - } -} - -async fn reconnect_run_sandbox( - state: &Arc, - run_id: &RunId, -) -> Result, Response> { - let record = load_run_sandbox_record(state, run_id).await?; - let daytona_api_key = state.vault_or_env(EnvVars::DAYTONA_API_KEY); - reconnect(&record, daytona_api_key).await.map_err(|err| { - let detail = render_with_causes(&err.to_string(), &collect_causes(err.as_ref())); - ApiError::new(StatusCode::CONFLICT, detail).into_response() - }) -} - -async fn reconnect_daytona_sandbox( - state: &Arc, - run_id: &RunId, -) -> Result { - let record = load_run_sandbox_record(state, run_id).await?; - if record.provider != SandboxProvider::Daytona.to_string() { - return Err(ApiError::new( - StatusCode::CONFLICT, - "Sandbox provider does not support this capability.", - ) - .into_response()); - } - let Some(name) = record.identifier.as_deref() else { - return Err(ApiError::new( - StatusCode::CONFLICT, - "Sandbox record is missing the Daytona identifier.", - ) - .into_response()); - }; - let Some(repo_cloned) = record.repo_cloned else { - return Err(ApiError::new( - StatusCode::CONFLICT, - "Sandbox record is missing clone metadata.", - ) - .into_response()); - }; - let daytona_api_key = state.vault_or_env(EnvVars::DAYTONA_API_KEY); - DaytonaSandbox::reconnect( - name, - daytona_api_key, - repo_cloned, - record.clone_origin_url.clone(), - record.clone_branch.clone(), - ) - .await - .map_err(|err| ApiError::new(StatusCode::CONFLICT, err.display_with_causes()).into_response()) -} - -async fn load_run_sandbox_record( - state: &Arc, - run_id: &RunId, -) -> Result { - match state.store.open_run_reader(run_id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => run_state.sandbox.ok_or_else(|| { - ApiError::new(StatusCode::CONFLICT, "Run has no active sandbox.").into_response() - }), - Err(err) => Err( - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - ), - }, - Err(_) => Err(ApiError::not_found("Run not found.").into_response()), - } -} - async fn append_control_request( state: &AppState, run_id: RunId, @@ -6688,23 +3191,6 @@ async fn append_control_request( workflow_event::append_event(&run_store, &run_id, &event).await } -/// Returns the wire event name if the given body has a dedicated operation -/// endpoint that clients must use instead of injecting via `append_run_event`. -/// These endpoints enforce authorization and status-transition preconditions -/// (e.g. "archive only from terminal") that a direct event append would -/// bypass. Other run-lifecycle events flow through this endpoint legitimately: -/// the worker subprocess emits state transitions during execution. -fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> { - match body { - EventBody::RunArchived(_) => Some("run.archived"), - EventBody::RunUnarchived(_) => Some("run.unarchived"), - EventBody::RunCancelRequested(_) => Some("run.cancel.requested"), - EventBody::RunPauseRequested(_) => Some("run.pause.requested"), - EventBody::RunUnpauseRequested(_) => Some("run.unpause.requested"), - _ => None, - } -} - /// Returns a 409 response with an actionable "unarchive first" message if the /// run is currently archived. Returns `None` otherwise (including when the run /// doesn't exist — the caller's own not-found handling will surface that). @@ -6721,1271 +3207,6 @@ async fn reject_if_archived(state: &AppState, run_id: &RunId) -> Option, run_id: RunId, worker_pid: u32) { - tokio::spawn(async move { - sleep(WORKER_CANCEL_GRACE).await; - let current_pid = { - let runs = state.runs.lock().expect("runs lock poisoned"); - runs.get(&run_id).and_then(|run| run.worker_pid) - }; - if current_pid == Some(worker_pid) && fabro_proc::process_group_alive(worker_pid) { - #[cfg(unix)] - fabro_proc::sigkill_process_group(worker_pid); - } - }); -} - -async fn cancel_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let ( - created_at, - response_status, - persist_cancelled_status, - answer_transport, - cancel_token, - cancel_tx, - worker_pid, - ) = { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get_mut(&id) { - Some(managed_run) => match managed_run.status { - RunStatus::Submitted - | RunStatus::Queued - | RunStatus::Starting - | RunStatus::Running - | RunStatus::Blocked { .. } - | RunStatus::Paused { .. } => { - let use_cancel_signal = !matches!( - managed_run.answer_transport, - Some(RunAnswerTransport::InProcess { .. }) - ); - let persist_cancelled_status = - matches!(managed_run.status, RunStatus::Submitted | RunStatus::Queued); - let response_status = if persist_cancelled_status { - let cancelled = RunStatus::Failed { - reason: FailureReason::Cancelled, - }; - managed_run.status = cancelled; - cancelled - } else { - managed_run.status - }; - ( - managed_run.created_at, - response_status, - persist_cancelled_status, - managed_run.answer_transport.clone(), - managed_run.cancel_token.clone(), - use_cancel_signal - .then(|| managed_run.cancel_tx.take()) - .flatten(), - managed_run.worker_pid, - ) - } - _ => { - return ApiError::new(StatusCode::CONFLICT, "Run is not cancellable.") - .into_response(); - } - }, - None => return ApiError::not_found("Run not found.").into_response(), - } - }; - - if pending_control != Some(RunControlAction::Cancel) { - if let Err(err) = append_control_request( - state.as_ref(), - id, - RunControlAction::Cancel, - Some(Principal::User(subject.0.clone())), - ) - .await - { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - - if let Some(token) = &cancel_token { - token.store(true, Ordering::SeqCst); - } - let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx { - let _ = cancel_tx.send(()); - true - } else { - false - }; - if let Some(answer_transport) = answer_transport { - if !(sent_cancel_signal && matches!(answer_transport, RunAnswerTransport::InProcess { .. })) - { - let _ = answer_transport.cancel_run().await; - } - } - if let Some(worker_pid) = worker_pid { - #[cfg(unix)] - fabro_proc::sigterm(worker_pid); - schedule_worker_kill(Arc::clone(&state), id, worker_pid); - } - - if persist_cancelled_status { - if let Err(err) = persist_cancelled_run_status(state.as_ref(), id).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - } - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - ( - StatusCode::OK, - Json(RunStatusResponse { - id: id.to_string(), - status: response_status, - error: None, - queue_position: None, - pending_control, - created_at, - }), - ) - .into_response() -} - -/// How `pause_run` should enact the transition, chosen from the current run -/// status. -enum PauseMode { - /// Worker is running; ask it to pause via SIGUSR1. Status flips to - /// `Paused` once the worker acknowledges. - Signal { worker_pid: u32 }, - /// Worker is blocked on a human gate; flip to `Paused` directly by - /// appending `RunPaused` ourselves. - AppendEvent, -} - -/// How `unpause_run` should enact the transition. -enum UnpauseMode { - /// No outstanding block; ask the worker to resume via SIGUSR2. - Signal { worker_pid: u32 }, - /// Was paused while blocked; append `RunUnpaused` and let the reducer - /// restore the underlying blocked state from `Paused { prior_block }`. - AppendEvent, -} - -async fn pause_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let (created_at, mode) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) if managed_run.status == RunStatus::Running => { - let Some(worker_pid) = managed_run.worker_pid else { - return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") - .into_response(); - }; - (managed_run.created_at, PauseMode::Signal { worker_pid }) - } - Some(managed_run) if matches!(managed_run.status, RunStatus::Blocked { .. }) => { - (managed_run.created_at, PauseMode::AppendEvent) - } - Some(_) => { - return ApiError::new(StatusCode::CONFLICT, "Run is not pausable.").into_response(); - } - None => return ApiError::not_found("Run not found.").into_response(), - } - }; - - if pending_control.is_some() { - return ApiError::new( - StatusCode::CONFLICT, - "Run control request is already pending.", - ) - .into_response(); - } - if let Err(err) = append_control_request( - state.as_ref(), - id, - RunControlAction::Pause, - Some(Principal::User(subject.0.clone())), - ) - .await - { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - let response_status = match mode { - PauseMode::Signal { worker_pid } => { - #[cfg(unix)] - fabro_proc::sigusr1(worker_pid); - #[cfg(not(unix))] - let _ = worker_pid; - RunStatus::Running - } - PauseMode::AppendEvent => { - if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { - events.push(workflow_event::Event::RunPaused); - }) - .await - { - return response; - } - state - .runs - .lock() - .expect("runs lock poisoned") - .get(&id) - .map_or(RunStatus::Paused { prior_block: None }, |run| run.status) - } - }; - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - ( - StatusCode::OK, - Json(RunStatusResponse { - id: id.to_string(), - status: response_status, - error: None, - queue_position: None, - pending_control, - created_at, - }), - ) - .into_response() -} - -async fn unpause_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let (created_at, mode) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => match managed_run.status { - RunStatus::Paused { - prior_block: Some(_), - } => (managed_run.created_at, UnpauseMode::AppendEvent), - RunStatus::Paused { prior_block: None } => { - let Some(worker_pid) = managed_run.worker_pid else { - return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") - .into_response(); - }; - (managed_run.created_at, UnpauseMode::Signal { worker_pid }) - } - _ => { - return ApiError::new(StatusCode::CONFLICT, "Run is not paused.") - .into_response(); - } - }, - None => return ApiError::not_found("Run not found.").into_response(), - } - }; - - if pending_control.is_some() { - return ApiError::new( - StatusCode::CONFLICT, - "Run control request is already pending.", - ) - .into_response(); - } - if let Err(err) = append_control_request( - state.as_ref(), - id, - RunControlAction::Unpause, - Some(Principal::User(subject.0.clone())), - ) - .await - { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - let response_status = match mode { - UnpauseMode::Signal { worker_pid } => { - #[cfg(unix)] - fabro_proc::sigusr2(worker_pid); - #[cfg(not(unix))] - let _ = worker_pid; - RunStatus::Paused { prior_block: None } - } - UnpauseMode::AppendEvent => { - if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { - events.push(workflow_event::Event::RunUnpaused); - }) - .await - { - return response; - } - state - .runs - .lock() - .expect("runs lock poisoned") - .get(&id) - .map_or(RunStatus::Running, |run| run.status) - } - }; - let pending_control = match load_pending_control(state.as_ref(), id).await { - Ok(pending_control) => pending_control, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - ( - StatusCode::OK, - Json(RunStatusResponse { - id: id.to_string(), - status: response_status, - error: None, - queue_position: None, - pending_control, - created_at, - }), - ) - .into_response() -} - -async fn archive_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - run_archive_action(state, subject, id, ArchiveAction::Archive).await -} - -async fn unarchive_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - run_archive_action(state, subject, id, ArchiveAction::Unarchive).await -} - -async fn rewind_run( - subject: RequiredUser, - State(state): State>, - Path(id): Path, - body: Option>, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let request = body.map(|Json(body)| body).unwrap_or_default(); - let target = match parse_fork_target(request.target) { - Ok(target) => target, - Err(err) => return err.into_response(), - }; - let input = operations::RewindInput { run_id: id, target }; - match Box::pin(operations::rewind( - &state.store, - &input, - Some(Principal::User(subject.0.clone())), - )) - .await - { - Ok(operations::RewindOutcome::Full { - source_run_id, - new_run_id, - target, - }) => ( - StatusCode::OK, - Json(RewindResponse { - source_run_id: source_run_id.to_string(), - new_run_id: new_run_id.to_string(), - target: target.response_target(), - archived: true, - archive_error: None, - }), - ) - .into_response(), - Ok(operations::RewindOutcome::Partial { - source_run_id, - new_run_id, - target, - archive_error, - }) => ( - StatusCode::MULTI_STATUS, - Json(RewindResponse { - source_run_id: source_run_id.to_string(), - new_run_id: new_run_id.to_string(), - target: target.response_target(), - archived: false, - archive_error: Some(archive_error), - }), - ) - .into_response(), - Err(err) => workflow_operation_error_response(err), - } -} - -async fn fork_run( - _subject: RequiredUser, - State(state): State>, - Path(id): Path, - body: Option>, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let request = body.map(|Json(body)| body).unwrap_or_default(); - let target = match parse_fork_target(request.target) { - Ok(target) => target, - Err(err) => return err.into_response(), - }; - let input = operations::ForkRunInput { - source_run_id: id, - target, - }; - match Box::pin(operations::fork_run(&state.store, &input)).await { - Ok(outcome) => ( - StatusCode::OK, - Json(ForkResponse { - source_run_id: outcome.source_run_id.to_string(), - new_run_id: outcome.new_run_id.to_string(), - target: outcome.target.response_target(), - }), - ) - .into_response(), - Err(err) => workflow_operation_error_response(err), - } -} - -async fn run_timeline( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - match operations::timeline(&state.store, &id).await { - Ok(entries) => Json( - entries - .into_iter() - .map(|entry| TimelineEntryResponse { - ordinal: std::num::NonZeroU64::new(entry.ordinal as u64) - .expect("timeline ordinals start at 1"), - node_name: entry.node_name, - visit: std::num::NonZeroU64::new(entry.visit as u64) - .expect("timeline visits start at 1"), - checkpoint_seq: std::num::NonZeroU64::new(u64::from(entry.checkpoint_seq)) - .expect("checkpoint event sequence starts at 1"), - run_commit_sha: entry.run_commit_sha, - }) - .collect::>(), - ) - .into_response(), - Err(err) => workflow_operation_error_response(err), - } -} - -fn parse_fork_target(target: Option) -> Result, ApiError> { - target - .map(|target| { - target - .parse::() - .map_err(|err| ApiError::bad_request(err.to_string())) - }) - .transpose() -} - -fn workflow_operation_error_response(err: WorkflowError) -> Response { - match err { - WorkflowError::Parse(message) | WorkflowError::Validation(message) => { - ApiError::bad_request(message).into_response() - } - WorkflowError::ValidationFailed { .. } => { - ApiError::bad_request("Validation failed").into_response() - } - WorkflowError::Precondition(message) => { - ApiError::new(StatusCode::CONFLICT, message).into_response() - } - WorkflowError::RunNotFound(_) => ApiError::not_found("Run not found.").into_response(), - WorkflowError::Unsupported(message) => { - ApiError::new(StatusCode::NOT_IMPLEMENTED, message).into_response() - } - err => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - } -} - -#[derive(Clone, Copy)] -enum ArchiveAction { - Archive, - Unarchive, -} - -async fn run_archive_action( - state: Arc, - subject: RequiredUser, - id: String, - action: ArchiveAction, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - let actor = Some(Principal::User(subject.0.clone())); - let result = match action { - ArchiveAction::Archive => operations::archive(&state.store, &id, actor) - .await - .map(|_| ()), - ArchiveAction::Unarchive => operations::unarchive(&state.store, &id, actor) - .await - .map(|_| ()), - }; - match result { - Ok(()) => archive_status_response(state.as_ref(), id).await, - Err(WorkflowError::Precondition(message)) => { - ApiError::new(StatusCode::CONFLICT, message).into_response() - } - Err(WorkflowError::RunNotFound(_)) => ApiError::not_found("Run not found.").into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -/// Build a `RunStatusResponse` reflecting the durable projection after an -/// archive/unarchive transition. The run is terminal in both directions, so no -/// live queue position or worker-only fields apply. -async fn archive_status_response(state: &AppState, id: RunId) -> Response { - let Ok(run_store) = state.store.open_run_reader(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let projection = match run_store.state().await { - Ok(projection) => projection, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let Some(status) = projection.status else { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - "run has no status after archive/unarchive", - ) - .into_response(); - }; - ( - StatusCode::OK, - Json(RunStatusResponse { - id: id.to_string(), - status, - error: None, - queue_position: None, - pending_control: None, - created_at: id.created_at(), - }), - ) - .into_response() -} - -/// Persist a synchronous pause/unpause transition: append the caller-supplied -/// events to the run store and mirror the new status in the in-memory run map. -/// Returns `Some(Response)` on error, `None` on success. -async fn synchronous_transition( - state: &AppState, - id: RunId, - append_events: impl FnOnce(&mut Vec), -) -> Option { - let run_store = match state.store.open_run(&id).await { - Ok(run_store) => run_store, - Err(err) => { - return Some( - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - ); - } - }; - let mut events = Vec::new(); - append_events(&mut events); - for event in events { - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { - return Some( - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - ); - } - let stored = workflow_event::to_run_event(&id, &event); - update_live_run_from_event(state, id, &stored); - } - None -} - -async fn list_models( - _auth: RequiredUser, - State(state): State>, - Query(params): Query, -) -> Response { - let provider = match params.provider.as_deref() { - Some(value) => match Provider::from_str(value) { - Ok(provider) => Some(provider), - Err(_) => { - return ApiError::new( - StatusCode::BAD_REQUEST, - format!("unknown provider: {value}"), - ) - .into_response(); - } - }, - None => None, - }; - - let query = params.query.as_ref().map(|value| value.to_lowercase()); - let limit = params.limit.clamp(1, 100) as usize; - let offset = params.offset.min(MAX_PAGE_OFFSET) as usize; - let configured: HashSet = state - .llm_source - .configured_providers() - .await - .into_iter() - .collect(); - - let mut models = fabro_model::Catalog::builtin() - .list(provider) - .into_iter() - .filter(|model| match &query { - Some(query) => { - model.id.to_lowercase().contains(query) - || model.display_name.to_lowercase().contains(query) - || model - .aliases - .iter() - .any(|alias| alias.to_lowercase().contains(query)) - } - None => true, - }) - .cloned() - .map(|mut model| { - model.configured = configured.contains(&model.provider); - model - }) - .collect::>(); - - let has_more = models.len() > offset.saturating_add(limit); - let data = models.drain(offset..models.len().min(offset.saturating_add(limit))); - - ( - StatusCode::OK, - Json(serde_json::json!({ - "data": data.collect::>(), - "meta": { "has_more": has_more } - })), - ) - .into_response() -} - -async fn test_model( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, -) -> Response { - let mode = match params.mode.as_deref() { - Some(value) => match ModelTestMode::from_str(value) { - Ok(mode) => mode, - Err(_) => { - return ApiError::new( - StatusCode::BAD_REQUEST, - format!("invalid model test mode: {value}"), - ) - .into_response(); - } - }, - None => ModelTestMode::Basic, - }; - let Some(info) = fabro_model::Catalog::builtin().get(&id) else { - return ApiError::not_found(format!("Model not found: {id}")).into_response(); - }; - - let llm_result = match state.resolve_llm_client().await { - Ok(result) => result, - Err(err) => { - error!(error = ?err, "Failed to resolve LLM client"); - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to resolve LLM client: {err}"), - ) - .into_response(); - } - }; - if let Some((_, issue)) = llm_result - .auth_issues - .iter() - .find(|(provider, _)| *provider == info.provider) - { - return ApiError::bad_request(auth_issue_message(info.provider, issue)).into_response(); - } - let provider_name = <&'static str>::from(info.provider); - if !llm_result.client.provider_names().contains(&provider_name) { - return Json(serde_json::json!({ - "model_id": info.id, - "status": "skip", - })) - .into_response(); - } - let client = Arc::new(llm_result.client); - - let outcome = run_model_test(info, mode, client).await; - Json(serde_json::json!({ - "model_id": info.id, - "status": <&'static str>::from(outcome.status), - "error_message": outcome.error_message, - })) - .into_response() -} - -fn finish_reason_to_api_stop_reason(reason: &FinishReason) -> String { - match reason { - FinishReason::Stop => "end_turn".to_string(), - FinishReason::Length => "max_tokens".to_string(), - FinishReason::ToolCalls => "tool_calls".to_string(), - FinishReason::ContentFilter => "content_filter".to_string(), - FinishReason::Error => "error".to_string(), - FinishReason::Other(s) => s.clone(), - } -} - -fn convert_api_message(msg: &CompletionMessage) -> LlmMessage { - let role = match msg.role { - CompletionMessageRole::System => Role::System, - CompletionMessageRole::User => Role::User, - CompletionMessageRole::Assistant => Role::Assistant, - CompletionMessageRole::Tool => Role::Tool, - CompletionMessageRole::Developer => Role::Developer, - }; - let content: Vec = msg - .content - .iter() - .filter_map(|part| { - let json = serde_json::to_value(part).ok()?; - serde_json::from_value(json).ok() - }) - .collect(); - LlmMessage { - role, - content, - name: msg.name.clone(), - tool_call_id: msg.tool_call_id.clone(), - } -} - -fn convert_llm_message(msg: &LlmMessage) -> CompletionMessage { - let role = match msg.role { - Role::System => CompletionMessageRole::System, - Role::User => CompletionMessageRole::User, - Role::Assistant => CompletionMessageRole::Assistant, - Role::Tool => CompletionMessageRole::Tool, - Role::Developer => CompletionMessageRole::Developer, - }; - let content: Vec = msg - .content - .iter() - .filter_map(|part| { - let json = serde_json::to_value(part).ok()?; - serde_json::from_value(json).ok() - }) - .collect(); - CompletionMessage { - role, - content, - name: msg.name.clone(), - tool_call_id: msg.tool_call_id.clone(), - } -} - -async fn create_completion( - _auth: RequiredUser, - State(state): State>, - Json(req): Json, -) -> Response { - // Resolve model - let model_id = req.model.unwrap_or_else(|| { - fabro_model::Catalog::builtin() - .list(None) - .first() - .map_or_else(|| "claude-sonnet-4-5".to_string(), |m| m.id.clone()) - }); - - let catalog_info = fabro_model::Catalog::builtin().get(&model_id); - - // Resolve provider: explicit request > catalog > None - let provider_name = req - .provider - .or_else(|| catalog_info.map(|i| i.provider.to_string())); - - info!(model = %model_id, provider = ?provider_name, "Completion request received"); - - // Build messages list - let mut messages: Vec = Vec::new(); - if let Some(system) = req.system { - messages.push(LlmMessage::system(system)); - } - for msg in &req.messages { - messages.push(convert_api_message(msg)); - } - - // Convert tools - let tools: Option> = if req.tools.is_empty() { - None - } else { - Some( - req.tools - .into_iter() - .map(|t| ToolDefinition { - name: t.name, - description: t.description, - parameters: t.parameters, - }) - .collect(), - ) - }; - - // Convert tool_choice - let tool_choice: Option = req.tool_choice.map(|tc| match tc.mode { - CompletionToolChoiceMode::Auto => ToolChoice::Auto, - CompletionToolChoiceMode::None => ToolChoice::None, - CompletionToolChoiceMode::Required => ToolChoice::Required, - CompletionToolChoiceMode::Named => ToolChoice::named(tc.tool_name.unwrap_or_default()), - }); - - // Build the LLM request - let request = LlmRequest { - model: model_id.clone(), - messages, - provider: provider_name, - tools, - tool_choice, - response_format: None, - temperature: req.temperature, - top_p: req.top_p, - max_tokens: req.max_tokens, - stop_sequences: if req.stop_sequences.is_empty() { - None - } else { - Some(req.stop_sequences) - }, - reasoning_effort: req.reasoning_effort.as_deref().and_then(|s| s.parse().ok()), - speed: None, - metadata: None, - provider_options: req.provider_options, - }; - - // Force non-streaming for structured output - let use_stream = req.stream && req.schema.is_none(); - - let llm_result = match state.resolve_llm_client().await { - Ok(result) => result, - Err(err) => { - error!(error = ?err, "Failed to create LLM client"); - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create LLM client: {err}"), - ) - .into_response(); - } - }; - for (provider, issue) in &llm_result.auth_issues { - warn!(provider = %provider, error = %issue, "LLM provider unavailable due to auth issue"); - } - let client = llm_result.client; - - if use_stream { - // Streaming path: forward all StreamEvents as SSE - let stream_result = match client.stream(&request).await { - Ok(s) => s, - Err(e) => { - return ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(); - } - }; - - let sse_stream = tokio_stream::StreamExt::filter_map(stream_result, |event| match event { - Ok(ref evt) => match serde_json::to_string(evt) { - Ok(json) => Some(Ok::<_, std::convert::Infallible>( - Event::default().event("stream_event").data(json), - )), - Err(e) => Some(Ok(Event::default().event("stream_event").data( - serde_json::json!({ - "type": "error", - "error": {"Stream": {"message": format!("failed to serialize event: {e}")}}, - "raw": null - }) - .to_string(), - ))), - }, - Err(e) => Some(Ok(Event::default().event("stream_event").data( - serde_json::json!({ - "type": "error", - "error": {"Stream": {"message": e.to_string()}}, - "raw": null - }) - .to_string(), - ))), - }); - - Sse::new(sse_stream) - .keep_alive( - KeepAlive::new().interval(Duration::from_secs(15)).event( - Event::default() - .event("ping") - .data(serde_json::json!({"type": "ping"}).to_string()), - ), - ) - .into_response() - } else { - // Non-streaming path - let msg_id = Ulid::new().to_string(); - - if let Some(schema) = req.schema { - // Structured output uses generate_object for JSON parsing logic - let mut params = - GenerateParams::new(&request.model, std::sync::Arc::new(client.clone())) - .messages(request.messages); - if let Some(ref p) = request.provider { - params = params.provider(p); - } - if let Some(temp) = request.temperature { - params = params.temperature(temp); - } - if let Some(max_tokens) = request.max_tokens { - params = params.max_tokens(max_tokens); - } - if let Some(top_p) = request.top_p { - params = params.top_p(top_p); - } - match generate_object(params, schema).await { - Ok(result) => Json(CompletionResponse { - id: msg_id, - model: model_id, - message: convert_llm_message(&result.response.message), - stop_reason: finish_reason_to_api_stop_reason(&result.finish_reason), - usage: CompletionUsage { - input_tokens: result.usage.input_tokens, - output_tokens: result.usage.output_tokens, - }, - output: result.output, - }) - .into_response(), - Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(), - } - } else { - match client.complete(&request).await { - Ok(response) => Json(CompletionResponse { - id: response.id, - model: response.model, - message: convert_llm_message(&response.message), - stop_reason: finish_reason_to_api_stop_reason(&response.finish_reason), - usage: CompletionUsage { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - }, - output: None, - }) - .into_response(), - Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(), - } - } - } -} - -#[expect( - clippy::disallowed_methods, - reason = "Render-graph subprocess startup resolves Cargo's test binary env override when present." -)] -fn render_graph_subprocess_exe( - exe_override: Option<&std::path::Path>, -) -> Result { - if let Some(path) = exe_override { - Ok(path.to_path_buf()) - } else { - if let Some(path) = std::env::var_os(EnvVars::CARGO_BIN_EXE_FABRO).map(PathBuf::from) { - return Ok(path); - } - - let current = std::env::current_exe() - .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; - let current_name = current.file_stem().and_then(|name| name.to_str()); - if current_name == Some("fabro") { - return Ok(current); - } - - let candidate = current - .parent() - .and_then(|parent| parent.parent()) - .map(|parent| parent.join(if cfg!(windows) { "fabro.exe" } else { "fabro" })); - if let Some(candidate) = candidate.filter(|path| path.is_file()) { - return Ok(candidate); - } - - Ok(current) - } -} - -fn render_subprocess_failure( - status: std::process::ExitStatus, - stderr: &[u8], -) -> RenderSubprocessError { - #[cfg(unix)] - { - use std::os::unix::process::ExitStatusExt; - - if let Some(signal) = status.signal() { - let stderr = String::from_utf8_lossy(stderr).trim().to_string(); - let detail = if stderr.is_empty() { - format!("terminated by signal {signal}") - } else { - format!("terminated by signal {signal}: {stderr}") - }; - return RenderSubprocessError::ChildCrashed(detail); - } - } - - let stderr = String::from_utf8_lossy(stderr).trim().to_string(); - let detail = match status.code() { - Some(code) if stderr.is_empty() => format!("exited with status {code}"), - Some(code) => format!("exited with status {code}: {stderr}"), - None if stderr.is_empty() => "child exited unsuccessfully".to_string(), - None => format!("child exited unsuccessfully: {stderr}"), - }; - RenderSubprocessError::ChildCrashed(detail) -} - -async fn render_dot_subprocess( - styled_source: &str, - exe_override: Option<&std::path::Path>, -) -> Result, RenderSubprocessError> { - let _permit = GRAPHVIZ_RENDER_SEMAPHORE - .acquire() - .await - .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; - let exe = render_graph_subprocess_exe(exe_override)?; - let mut cmd = Command::new(exe); - apply_render_graph_env(&mut cmd); - cmd.arg("__render-graph") - .env(EnvVars::FABRO_TELEMETRY, "off") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let mut child = cmd - .spawn() - .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; - let mut stdin = child.stdin.take().ok_or_else(|| { - RenderSubprocessError::SpawnFailed("render subprocess stdin was not piped".to_string()) - })?; - if let Err(err) = stdin.write_all(styled_source.as_bytes()).await { - drop(stdin); - let output = child - .wait_with_output() - .await - .map_err(|wait_err| RenderSubprocessError::SpawnFailed(wait_err.to_string()))?; - return Err(RenderSubprocessError::ChildCrashed(format!( - "failed writing DOT to child stdin: {err}; {}", - render_subprocess_failure(output.status, &output.stderr) - ))); - } - drop(stdin); - - let output = child - .wait_with_output() - .await - .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; - - if !output.status.success() { - return Err(render_subprocess_failure(output.status, &output.stderr)); - } - - if let Some(error) = output.stdout.strip_prefix(RENDER_ERROR_PREFIX) { - return Err(RenderSubprocessError::RenderFailed( - String::from_utf8_lossy(error).trim().to_string(), - )); - } - - if output.stdout.starts_with(b", -) -> Response { - use fabro_graphviz::render::{inject_dot_style_defaults, postprocess_svg}; - - let styled_source = inject_dot_style_defaults(dot_source); - match render_dot_subprocess(&styled_source, exe_override).await { - Ok(raw) => { - let bytes = postprocess_svg(raw); - (StatusCode::OK, [("content-type", "image/svg+xml")], bytes).into_response() - } - Err(RenderSubprocessError::RenderFailed(err)) => { - ApiError::new(StatusCode::BAD_REQUEST, err).into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - } -} - -pub(crate) async fn render_graph_bytes(dot_source: &str) -> Response { - render_graph_response(dot_source, None).await -} - -#[cfg(test)] -async fn render_graph_bytes_with_exe_override( - dot_source: &str, - exe_override: Option<&std::path::Path>, -) -> Response { - render_graph_response(dot_source, exe_override).await -} - -#[derive(serde::Deserialize)] -struct GraphParams { - #[serde(default)] - direction: Option, -} - -async fn load_run_dot_source(state: &AppState, id: &RunId) -> Result { - let live_dot_source = { - let runs = state.runs.lock().expect("runs lock poisoned"); - runs.get(id) - .map(|managed_run| managed_run.dot_source.clone()) - }; - - let dot_source = if let Some(dot) = live_dot_source.filter(|d| !d.is_empty()) { - Some(dot) - } else { - match state.store.open_run_reader(id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => run_state.graph_source, - Err(err) => { - return Err( - ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response() - ); - } - }, - Err(_) => return Err(ApiError::not_found("Run not found.").into_response()), - } - }; - - dot_source - .ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "Graph not found.").into_response()) -} - -async fn get_graph( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, - Query(params): Query, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - - let dot = match load_run_dot_source(&state, &id).await { - Ok(dot) => dot, - Err(response) => return response, - }; - - let dot = match params.direction.as_deref() { - Some(dir @ ("LR" | "TB" | "BT" | "RL")) => { - use fabro_graphviz::render; - render::apply_direction(&dot, dir).into_owned() - } - _ => dot, - }; - - render_graph_bytes(&dot).await -} - -async fn get_graph_source( - _auth: RequiredUser, - State(state): State>, - Path(id): Path, -) -> Response { - let id = match parse_run_id_path(&id) { - Ok(id) => id, - Err(response) => return response, - }; - - match load_run_dot_source(&state, &id).await { - Ok(dot) => (StatusCode::OK, [("content-type", "text/vnd.graphviz")], dot).into_response(), - Err(response) => response, - } -} - #[cfg(test)] #[expect( clippy::disallowed_methods, diff --git a/lib/crates/fabro-server/src/server/handler/artifacts.rs b/lib/crates/fabro-server/src/server/handler/artifacts.rs new file mode 100644 index 000000000..abab10948 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/artifacts.rs @@ -0,0 +1,679 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, ArtifactEntry, ArtifactKey, ArtifactListResponse, AsyncWriteExt, Body, + Bytes, DefaultBodyLimit, Digest, HashMap, HashSet, HeaderMap, IntoResponse, Json, NodeArtifact, + Path, Query, RequireRunBlob, RequireRunScoped, RequireStageArtifact, RequiredUser, Response, + Router, RunArtifactEntry, RunArtifactListResponse, RunId, Sha256, StageArtifactEntry, StageId, + State, StatusCode, StreamExt, WriteBlobResponse, axum_extract, bad_request_response, get, + header, octet_stream_response, parse_run_id_path, parse_stage_id_path, + payload_too_large_response, post, reject_if_archived, required_query_param, + validate_relative_artifact_path, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/runs/{id}/checkpoint", get(get_checkpoint)) + .route("/runs/{id}/blobs", post(write_run_blob)) + .route("/runs/{id}/blobs/{blobId}", get(read_run_blob)) + .route("/runs/{id}/artifacts", get(list_run_artifacts)) + .route( + "/runs/{id}/stages/{stageId}/artifacts", + get(list_stage_artifacts) + .post(put_stage_artifact) + .layer(DefaultBodyLimit::disable()), + ) + .route( + "/runs/{id}/stages/{stageId}/artifacts/download", + get(get_stage_artifact), + ) +} + +#[derive(serde::Deserialize)] +struct ArtifactFilenameParams { + #[serde(default)] + filename: Option, + #[serde(default)] + retry: Option, +} + +const MAX_SINGLE_ARTIFACT_BYTES: u64 = 10 * 1024 * 1024; +const MAX_MULTIPART_ARTIFACTS: usize = 100; +const MAX_MULTIPART_REQUEST_BYTES: u64 = 50 * 1024 * 1024; +const MAX_MULTIPART_MANIFEST_BYTES: usize = 256 * 1024; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct ArtifactBatchUploadManifest { + entries: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct ArtifactBatchUploadEntry { + part: String, + path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + sha256: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + expected_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + content_type: Option, +} + +async fn get_checkpoint( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let live_checkpoint = { + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) => managed_run.checkpoint.clone(), + None => return ApiError::not_found("Run not found.").into_response(), + } + }; + if let Some(cp) = live_checkpoint { + return (StatusCode::OK, Json(cp)).into_response(); + } + + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => match run_state.checkpoint { + Some(cp) => (StatusCode::OK, Json(cp)).into_response(), + None => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + }, + Err(err) => { + tracing::warn!(run_id = %id, error = %err, "Failed to load checkpoint state from store"); + (StatusCode::OK, Json(serde_json::json!(null))).into_response() + } + }, + Err(err) => { + tracing::warn!(run_id = %id, error = %err, "Failed to open run store reader"); + ApiError::not_found("Run not found.").into_response() + } + } +} + +async fn write_run_blob( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, + body: Bytes, +) -> Response { + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + match state.store.open_run(&id).await { + Ok(run_store) => match run_store.write_blob(&body).await { + Ok(blob_id) => Json(WriteBlobResponse { + id: blob_id.to_string(), + }) + .into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + +async fn read_run_blob( + RequireRunBlob(id, blob_id): RequireRunBlob, + State(state): State>, +) -> Response { + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.read_blob(&blob_id).await { + Ok(Some(bytes)) => octet_stream_response(bytes), + Ok(None) => ApiError::not_found("Blob not found.").into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + +async fn load_run_spec(state: &AppState, run_id: &RunId) -> Result { + let run_store = state + .store + .open_run_reader(run_id) + .await + .map_err(|_| ApiError::not_found("Run not found.").into_response())?; + let run_state = run_store.state().await.map_err(|err| { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + })?; + run_state.spec.ok_or_else(|| { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "run spec missing from store", + ) + .into_response() + }) +} + +async fn list_run_artifacts( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Err(response) = load_run_spec(state.as_ref(), &id).await { + return response; + } + + match state.artifact_store.list_for_run(&id).await { + Ok(entries) => Json(RunArtifactListResponse { + data: entries.into_iter().map(run_artifact_entry_from).collect(), + }) + .into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +fn run_artifact_entry_from(entry: NodeArtifact) -> RunArtifactEntry { + RunArtifactEntry { + stage_id: entry.node.to_string(), + node_slug: entry.node.node_id().to_string(), + retry: entry.retry.cast_signed(), + relative_path: entry.filename, + size: entry.size.cast_signed(), + } +} + +fn artifact_entry_from(entry: StageArtifactEntry) -> ArtifactEntry { + ArtifactEntry { + filename: entry.filename, + retry: entry.retry.cast_signed(), + size: entry.size.cast_signed(), + } +} + +async fn list_stage_artifacts( + _auth: RequiredUser, + State(state): State>, + Path((id, stage_id)): Path<(String, String)>, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + if let Err(response) = load_run_spec(state.as_ref(), &id).await { + return response; + } + + match state.artifact_store.list_for_node(&id, &stage_id).await { + Ok(entries) => Json(ArtifactListResponse { + data: entries.into_iter().map(artifact_entry_from).collect(), + }) + .into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +enum ArtifactUploadContentType { + OctetStream, + Multipart { boundary: String }, +} + +struct ValidatedArtifactBatchEntry { + path: String, + sha256: Option, + expected_bytes: Option, +} + +#[allow( + clippy::result_large_err, + reason = "Upload content-type parsing returns HTTP client errors directly." +)] +fn artifact_upload_content_type( + headers: &HeaderMap, +) -> Result { + let value = headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| { + ApiError::new( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "artifact uploads require a supported Content-Type", + ) + .into_response() + })?; + + let mime = value.split(';').next().unwrap_or(value).trim(); + match mime { + "application/octet-stream" => Ok(ArtifactUploadContentType::OctetStream), + "multipart/form-data" => multer::parse_boundary(value) + .map(|boundary| ArtifactUploadContentType::Multipart { boundary }) + .map_err(|err| bad_request_response(format!("invalid multipart boundary: {err}"))), + _ => Err(ApiError::new( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "artifact uploads only support application/octet-stream or multipart/form-data", + ) + .into_response()), + } +} + +#[allow( + clippy::result_large_err, + reason = "Content-Length parsing returns HTTP client errors directly." +)] +fn content_length_from_headers(headers: &HeaderMap) -> Result, Response> { + headers + .get(header::CONTENT_LENGTH) + .map(|value| { + value + .to_str() + .map_err(|err| { + bad_request_response(format!("invalid content-length header: {err}")) + }) + .and_then(|value| { + value.parse::().map_err(|err| { + bad_request_response(format!("invalid content-length header: {err}")) + }) + }) + }) + .transpose() +} + +#[allow( + clippy::result_large_err, + reason = "Multipart manifest parsing returns HTTP client errors directly." +)] +async fn read_multipart_manifest( + field: &mut multer::Field<'_>, +) -> Result { + let mut manifest_bytes = Vec::new(); + while let Some(chunk) = field + .chunk() + .await + .map_err(|err| bad_request_response(format!("invalid multipart body: {err}")))? + { + manifest_bytes.extend_from_slice(&chunk); + if manifest_bytes.len() > MAX_MULTIPART_MANIFEST_BYTES { + return Err(payload_too_large_response( + "multipart manifest exceeds the server limit", + )); + } + } + + serde_json::from_slice(&manifest_bytes) + .map_err(|err| bad_request_response(format!("invalid multipart manifest: {err}"))) +} + +#[allow( + clippy::result_large_err, + reason = "Artifact batch validation returns HTTP client errors directly." +)] +fn validate_artifact_batch_manifest( + manifest: ArtifactBatchUploadManifest, +) -> Result, Response> { + if manifest.entries.is_empty() { + return Err(bad_request_response( + "multipart manifest must include at least one artifact entry", + )); + } + if manifest.entries.len() > MAX_MULTIPART_ARTIFACTS { + return Err(payload_too_large_response(format!( + "multipart upload exceeds the {MAX_MULTIPART_ARTIFACTS} artifact limit" + ))); + } + + let mut entries = HashMap::with_capacity(manifest.entries.len()); + let mut seen_paths = HashSet::new(); + let mut expected_total_bytes = 0_u64; + + for entry in manifest.entries { + if entry.part.is_empty() { + return Err(bad_request_response( + "multipart manifest part names must not be empty", + )); + } + if entry.part == "manifest" { + return Err(bad_request_response( + "multipart manifest part name 'manifest' is reserved", + )); + } + let path = validate_relative_artifact_path("manifest path", &entry.path)?; + if !seen_paths.insert(path.clone()) { + return Err(bad_request_response(format!( + "duplicate artifact path in multipart manifest: {path}" + ))); + } + if let Some(sha256) = entry.sha256.as_ref() { + if sha256.len() != 64 || !sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(bad_request_response(format!( + "invalid sha256 for multipart part {}", + entry.part + ))); + } + } + if let Some(expected_bytes) = entry.expected_bytes { + if expected_bytes > MAX_SINGLE_ARTIFACT_BYTES { + return Err(payload_too_large_response(format!( + "artifact {path} exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" + ))); + } + expected_total_bytes = expected_total_bytes.saturating_add(expected_bytes); + if expected_total_bytes > MAX_MULTIPART_REQUEST_BYTES { + return Err(payload_too_large_response(format!( + "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" + ))); + } + } + if entries + .insert(entry.part.clone(), ValidatedArtifactBatchEntry { + path, + sha256: entry.sha256.map(|value| value.to_ascii_lowercase()), + expected_bytes: entry.expected_bytes, + }) + .is_some() + { + return Err(bad_request_response(format!( + "duplicate multipart part name in manifest: {}", + entry.part + ))); + } + } + + Ok(entries) +} + +async fn upload_stage_artifact_octet_stream( + state: &AppState, + run_id: &RunId, + stage_id: &StageId, + retry: u32, + filename: String, + body: Body, + content_length: Option, +) -> Response { + let relative_path = match validate_relative_artifact_path("filename", &filename) { + Ok(path) => path, + Err(response) => return response, + }; + + if content_length.is_some_and(|length| length > MAX_SINGLE_ARTIFACT_BYTES) { + return payload_too_large_response(format!( + "artifact exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" + )); + } + + let mut writer = match state.artifact_store.writer( + run_id, + &ArtifactKey::new(stage_id.clone(), retry, relative_path), + ) { + Ok(writer) => writer, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + let mut bytes_written = 0_u64; + let mut data_stream = body.into_data_stream(); + while let Some(chunk) = data_stream.next().await { + let chunk = match chunk + .map_err(|err| bad_request_response(format!("invalid request body: {err}"))) + { + Ok(chunk) => chunk, + Err(response) => return response, + }; + bytes_written = + bytes_written.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX)); + if bytes_written > MAX_SINGLE_ARTIFACT_BYTES { + return payload_too_large_response(format!( + "artifact exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit" + )); + } + if let Err(err) = writer.write_all(&chunk).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + + match writer.shutdown().await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +async fn upload_stage_artifact_multipart( + state: &AppState, + run_id: &RunId, + stage_id: &StageId, + retry: u32, + boundary: String, + body: Body, +) -> Response { + let mut multipart = multer::Multipart::new(body.into_data_stream(), boundary); + let Some(mut manifest_field) = (match multipart + .next_field() + .await + .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) + { + Ok(field) => field, + Err(response) => return response, + }) else { + return bad_request_response("multipart upload must begin with a manifest part"); + }; + + if manifest_field.name() != Some("manifest") { + return bad_request_response("multipart upload must begin with a manifest part"); + } + + let manifest = match read_multipart_manifest(&mut manifest_field).await { + Ok(manifest) => manifest, + Err(response) => return response, + }; + drop(manifest_field); + let mut expected_parts = match validate_artifact_batch_manifest(manifest) { + Ok(entries) => entries, + Err(response) => return response, + }; + let mut total_bytes = 0_u64; + + while let Some(mut field) = match multipart + .next_field() + .await + .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) + { + Ok(field) => field, + Err(response) => return response, + } { + let Some(part_name) = field.name().map(ToOwned::to_owned) else { + return bad_request_response("multipart file parts must be named"); + }; + let Some(entry) = expected_parts.remove(&part_name) else { + return bad_request_response(format!("unexpected multipart part: {part_name}")); + }; + + let mut writer = match state.artifact_store.writer( + run_id, + &ArtifactKey::new(stage_id.clone(), retry, entry.path.clone()), + ) { + Ok(writer) => writer, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let mut bytes_written = 0_u64; + let mut sha256 = Sha256::new(); + + while let Some(chunk) = match field + .chunk() + .await + .map_err(|err| bad_request_response(format!("invalid multipart body: {err}"))) + { + Ok(chunk) => chunk, + Err(response) => return response, + } { + let chunk_len = u64::try_from(chunk.len()).unwrap_or(u64::MAX); + bytes_written = bytes_written.saturating_add(chunk_len); + total_bytes = total_bytes.saturating_add(chunk_len); + + if bytes_written > MAX_SINGLE_ARTIFACT_BYTES { + return payload_too_large_response(format!( + "artifact {} exceeds the {MAX_SINGLE_ARTIFACT_BYTES} byte limit", + entry.path + )); + } + if total_bytes > MAX_MULTIPART_REQUEST_BYTES { + return payload_too_large_response(format!( + "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" + )); + } + + sha256.update(&chunk); + if let Err(err) = writer.write_all(&chunk).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + + if let Some(expected_bytes) = entry.expected_bytes { + if bytes_written != expected_bytes { + return bad_request_response(format!( + "multipart part {part_name} expected {expected_bytes} bytes but received {bytes_written}" + )); + } + } + if let Some(expected_sha256) = entry.sha256.as_ref() { + let actual_sha256 = hex::encode(sha256.finalize()); + if actual_sha256 != *expected_sha256 { + return bad_request_response(format!( + "multipart part {part_name} sha256 did not match manifest" + )); + } + } + + if let Err(err) = writer.shutdown().await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + + if !expected_parts.is_empty() { + let mut missing = expected_parts.into_keys().collect::>(); + missing.sort(); + return bad_request_response(format!( + "multipart upload is missing part(s): {}", + missing.join(", ") + )); + } + + StatusCode::NO_CONTENT.into_response() +} + +async fn put_stage_artifact( + State(state): State>, + RequireStageArtifact(id, stage_id): RequireStageArtifact, + Query(params): Query, + request: axum_extract::Request, +) -> Response { + let (parts, body) = request.into_parts(); + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + if let Err(response) = load_run_spec(state.as_ref(), &id).await.map(|_| ()) { + return response; + } + let retry = match required_query_param(params.retry.as_ref(), "retry") { + Ok(retry) => retry, + Err(response) => return response, + }; + + let content_length = match content_length_from_headers(&parts.headers) { + Ok(length) => length, + Err(response) => return response, + }; + match artifact_upload_content_type(&parts.headers) { + Ok(ArtifactUploadContentType::OctetStream) => { + let filename = match required_query_param(params.filename.as_ref(), "filename") { + Ok(filename) => filename, + Err(response) => return response, + }; + upload_stage_artifact_octet_stream( + state.as_ref(), + &id, + &stage_id, + retry, + filename, + body, + content_length, + ) + .await + } + Ok(ArtifactUploadContentType::Multipart { boundary }) => { + if content_length.is_some_and(|length| length > MAX_MULTIPART_REQUEST_BYTES) { + return payload_too_large_response(format!( + "multipart upload exceeds the {MAX_MULTIPART_REQUEST_BYTES} byte limit" + )); + } + upload_stage_artifact_multipart(state.as_ref(), &id, &stage_id, retry, boundary, body) + .await + } + Err(response) => response, + } +} + +async fn get_stage_artifact( + _auth: RequiredUser, + State(state): State>, + Path((id, stage_id)): Path<(String, String)>, + Query(params): Query, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let filename = match required_query_param(params.filename.as_ref(), "filename") { + Ok(filename) => filename, + Err(response) => return response, + }; + let retry = match required_query_param(params.retry.as_ref(), "retry") { + Ok(retry) => retry, + Err(response) => return response, + }; + let relative_path = match validate_relative_artifact_path("filename", &filename) { + Ok(path) => path, + Err(response) => return response, + }; + if let Err(response) = load_run_spec(state.as_ref(), &id).await { + return response; + } + + match state + .artifact_store + .get( + &id, + &ArtifactKey::new(stage_id.clone(), retry, relative_path), + ) + .await + { + Ok(Some(bytes)) => octet_stream_response(bytes), + Ok(None) => ApiError::not_found("Artifact not found.").into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs new file mode 100644 index 000000000..52677f66e --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -0,0 +1,236 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap, + IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path, + Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, + RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get, + parse_run_id_path, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/runs/{id}/stages", get(list_run_stages)) + .route("/runs/{id}/billing", get(get_run_billing)) +} + +fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState { + let latest = events.iter().rev().find(|envelope| { + envelope.event.node_id.as_deref() == Some(node_id) + && matches!( + envelope.event.event_name(), + "stage.retrying" | "stage.started" | "stage.completed" | "stage.failed" + ) + }); + + if latest.is_some_and(|e| e.event.event_name() == "stage.retrying") { + StageState::Retrying + } else { + StageState::Running + } +} + +async fn list_run_stages( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(_pagination): Query, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + + // Try live run first. + let (checkpoint, run_is_active) = { + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) => { + let active = !matches!( + managed_run.status, + RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead + ); + (managed_run.checkpoint.clone(), active) + } + None => (None, false), + } + }; + + // Fall back to stored run. + let (checkpoint, run_is_active) = if checkpoint.is_some() { + (checkpoint, run_is_active) + } else { + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => { + let active = run_state.status.is_some_and(|status| !status.is_terminal()); + (run_state.checkpoint, active) + } + Err(_) => (None, false), + }, + Err(_) => return ApiError::not_found("Run not found.").into_response(), + } + }; + + let Some(checkpoint) = checkpoint else { + return ( + StatusCode::OK, + Json(ListResponse::new(Vec::::new())), + ) + .into_response(); + }; + + let events = match state.store.open_run_reader(&id).await { + Ok(run_store) => run_store.list_events().await.unwrap_or_default(), + Err(_) => Vec::new(), + }; + let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events); + + let mut stages = Vec::new(); + for node_id in &checkpoint.completed_nodes { + let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); + let status = match checkpoint.node_outcomes.get(node_id) { + Some(outcome) => StageState::from(outcome.status), + None => StageState::Succeeded, + }; + stages.push(RunStage { + id: node_id.clone(), + name: node_id.clone(), + status, + duration_secs: Some(duration_ms as f64 / 1000.0), + dot_id: Some(node_id.clone()), + }); + } + + // Add next node as running if the run is still active. + // The checkpoint's current_node is the last *completed* stage; next_node_id + // is the stage that is currently executing. + if let Some(next_id) = &checkpoint.next_node_id { + if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) { + stages.push(RunStage { + id: next_id.clone(), + name: next_id.clone(), + status: active_stage_state_from_events(&events, next_id), + duration_secs: None, + dot_id: Some(next_id.clone()), + }); + } + } + + (StatusCode::OK, Json(ListResponse::new(stages))).into_response() +} + +async fn get_run_billing( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let run_store = match state.store.open_run_reader(&id).await { + Ok(run_store) => run_store, + Err(err) => { + return ApiError::new(StatusCode::NOT_FOUND, err.to_string()).into_response(); + } + }; + + let checkpoint = match run_store.state().await { + Ok(state) => state.checkpoint, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + let Some(checkpoint) = checkpoint else { + let empty = RunBilling { + by_model: Vec::new(), + stages: Vec::new(), + totals: RunBillingTotals { + cache_read_tokens: 0, + cache_write_tokens: 0, + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + runtime_secs: 0.0, + total_tokens: 0, + total_usd_micros: None, + }, + }; + return (StatusCode::OK, Json(empty)).into_response(); + }; + + let stage_durations = match run_store.list_events().await { + Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + let mut by_model_totals = HashMap::::new(); + let mut billed_usages = Vec::new(); + let mut runtime_secs = 0.0_f64; + let mut stages = Vec::new(); + + for node_id in &checkpoint.completed_nodes { + let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); + runtime_secs += duration_ms as f64 / 1000.0; + + let Some(usage) = checkpoint + .node_outcomes + .get(node_id) + .and_then(|outcome| outcome.usage.as_ref()) + else { + continue; + }; + + billed_usages.push(usage.clone()); + let tokens = usage.tokens(); + let billing = BilledTokenCounts { + cache_read_tokens: tokens.cache_read_tokens, + cache_write_tokens: tokens.cache_write_tokens, + input_tokens: tokens.input_tokens, + output_tokens: tokens.output_tokens, + reasoning_tokens: tokens.reasoning_tokens, + total_tokens: tokens.total_tokens(), + total_usd_micros: usage.total_usd_micros, + }; + let model_id = usage.model_id().to_string(); + accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); + stages.push(RunBillingStage { + billing, + model: ModelReference { id: model_id }, + runtime_secs: duration_ms as f64 / 1000.0, + stage: BillingStageRef { + id: node_id.clone(), + name: node_id.clone(), + }, + }); + } + + let totals = BilledTokenCounts::from_billed_usage(&billed_usages); + let by_model = by_model_totals + .into_iter() + .map(|(model, totals)| BillingByModel { + billing: totals.billing, + model: ModelReference { id: model }, + stages: totals.stages, + }) + .collect::>(); + + let response = RunBilling { + by_model, + stages, + totals: RunBillingTotals { + cache_read_tokens: totals.cache_read_tokens, + cache_write_tokens: totals.cache_write_tokens, + input_tokens: totals.input_tokens, + output_tokens: totals.output_tokens, + reasoning_tokens: totals.reasoning_tokens, + runtime_secs, + total_tokens: totals.total_tokens, + total_usd_micros: totals.total_usd_micros, + }, + }; + + (StatusCode::OK, Json(response)).into_response() +} diff --git a/lib/crates/fabro-server/src/server/handler/completions.rs b/lib/crates/fabro-server/src/server/handler/completions.rs new file mode 100644 index 000000000..0c0a9d8a8 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/completions.rs @@ -0,0 +1,269 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, CompletionContentPart, CompletionMessage, CompletionMessageRole, + CompletionResponse, CompletionToolChoiceMode, CompletionUsage, ContentPart, + CreateCompletionRequest, Duration, Event, FinishReason, GenerateParams, IntoResponse, Json, + KeepAlive, LlmMessage, LlmRequest, RequiredUser, Response, Role, Router, Sse, State, + StatusCode, ToolChoice, ToolDefinition, Ulid, error, generate_object, info, post, warn, +}; + +pub(super) fn routes() -> Router> { + Router::new().route("/completions", post(create_completion)) +} + +fn finish_reason_to_api_stop_reason(reason: &FinishReason) -> String { + match reason { + FinishReason::Stop => "end_turn".to_string(), + FinishReason::Length => "max_tokens".to_string(), + FinishReason::ToolCalls => "tool_calls".to_string(), + FinishReason::ContentFilter => "content_filter".to_string(), + FinishReason::Error => "error".to_string(), + FinishReason::Other(s) => s.clone(), + } +} + +fn convert_api_message(msg: &CompletionMessage) -> LlmMessage { + let role = match msg.role { + CompletionMessageRole::System => Role::System, + CompletionMessageRole::User => Role::User, + CompletionMessageRole::Assistant => Role::Assistant, + CompletionMessageRole::Tool => Role::Tool, + CompletionMessageRole::Developer => Role::Developer, + }; + let content: Vec = msg + .content + .iter() + .filter_map(|part| { + let json = serde_json::to_value(part).ok()?; + serde_json::from_value(json).ok() + }) + .collect(); + LlmMessage { + role, + content, + name: msg.name.clone(), + tool_call_id: msg.tool_call_id.clone(), + } +} + +fn convert_llm_message(msg: &LlmMessage) -> CompletionMessage { + let role = match msg.role { + Role::System => CompletionMessageRole::System, + Role::User => CompletionMessageRole::User, + Role::Assistant => CompletionMessageRole::Assistant, + Role::Tool => CompletionMessageRole::Tool, + Role::Developer => CompletionMessageRole::Developer, + }; + let content: Vec = msg + .content + .iter() + .filter_map(|part| { + let json = serde_json::to_value(part).ok()?; + serde_json::from_value(json).ok() + }) + .collect(); + CompletionMessage { + role, + content, + name: msg.name.clone(), + tool_call_id: msg.tool_call_id.clone(), + } +} + +async fn create_completion( + _auth: RequiredUser, + State(state): State>, + Json(req): Json, +) -> Response { + // Resolve model + let model_id = req.model.unwrap_or_else(|| { + fabro_model::Catalog::builtin() + .list(None) + .first() + .map_or_else(|| "claude-sonnet-4-5".to_string(), |m| m.id.clone()) + }); + + let catalog_info = fabro_model::Catalog::builtin().get(&model_id); + + // Resolve provider: explicit request > catalog > None + let provider_name = req + .provider + .or_else(|| catalog_info.map(|i| i.provider.to_string())); + + info!(model = %model_id, provider = ?provider_name, "Completion request received"); + + // Build messages list + let mut messages: Vec = Vec::new(); + if let Some(system) = req.system { + messages.push(LlmMessage::system(system)); + } + for msg in &req.messages { + messages.push(convert_api_message(msg)); + } + + // Convert tools + let tools: Option> = if req.tools.is_empty() { + None + } else { + Some( + req.tools + .into_iter() + .map(|t| ToolDefinition { + name: t.name, + description: t.description, + parameters: t.parameters, + }) + .collect(), + ) + }; + + // Convert tool_choice + let tool_choice: Option = req.tool_choice.map(|tc| match tc.mode { + CompletionToolChoiceMode::Auto => ToolChoice::Auto, + CompletionToolChoiceMode::None => ToolChoice::None, + CompletionToolChoiceMode::Required => ToolChoice::Required, + CompletionToolChoiceMode::Named => ToolChoice::named(tc.tool_name.unwrap_or_default()), + }); + + // Build the LLM request + let request = LlmRequest { + model: model_id.clone(), + messages, + provider: provider_name, + tools, + tool_choice, + response_format: None, + temperature: req.temperature, + top_p: req.top_p, + max_tokens: req.max_tokens, + stop_sequences: if req.stop_sequences.is_empty() { + None + } else { + Some(req.stop_sequences) + }, + reasoning_effort: req.reasoning_effort.as_deref().and_then(|s| s.parse().ok()), + speed: None, + metadata: None, + provider_options: req.provider_options, + }; + + // Force non-streaming for structured output + let use_stream = req.stream && req.schema.is_none(); + + let llm_result = match state.resolve_llm_client().await { + Ok(result) => result, + Err(err) => { + error!(error = ?err, "Failed to create LLM client"); + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create LLM client: {err}"), + ) + .into_response(); + } + }; + for (provider, issue) in &llm_result.auth_issues { + warn!(provider = %provider, error = %issue, "LLM provider unavailable due to auth issue"); + } + let client = llm_result.client; + + if use_stream { + // Streaming path: forward all StreamEvents as SSE + let stream_result = match client.stream(&request).await { + Ok(s) => s, + Err(e) => { + return ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) + .into_response(); + } + }; + + let sse_stream = tokio_stream::StreamExt::filter_map(stream_result, |event| match event { + Ok(ref evt) => match serde_json::to_string(evt) { + Ok(json) => Some(Ok::<_, std::convert::Infallible>( + Event::default().event("stream_event").data(json), + )), + Err(e) => Some(Ok(Event::default().event("stream_event").data( + serde_json::json!({ + "type": "error", + "error": {"Stream": {"message": format!("failed to serialize event: {e}")}}, + "raw": null + }) + .to_string(), + ))), + }, + Err(e) => Some(Ok(Event::default().event("stream_event").data( + serde_json::json!({ + "type": "error", + "error": {"Stream": {"message": e.to_string()}}, + "raw": null + }) + .to_string(), + ))), + }); + + Sse::new(sse_stream) + .keep_alive( + KeepAlive::new().interval(Duration::from_secs(15)).event( + Event::default() + .event("ping") + .data(serde_json::json!({"type": "ping"}).to_string()), + ), + ) + .into_response() + } else { + // Non-streaming path + let msg_id = Ulid::new().to_string(); + + if let Some(schema) = req.schema { + // Structured output uses generate_object for JSON parsing logic + let mut params = + GenerateParams::new(&request.model, std::sync::Arc::new(client.clone())) + .messages(request.messages); + if let Some(ref p) = request.provider { + params = params.provider(p); + } + if let Some(temp) = request.temperature { + params = params.temperature(temp); + } + if let Some(max_tokens) = request.max_tokens { + params = params.max_tokens(max_tokens); + } + if let Some(top_p) = request.top_p { + params = params.top_p(top_p); + } + match generate_object(params, schema).await { + Ok(result) => Json(CompletionResponse { + id: msg_id, + model: model_id, + message: convert_llm_message(&result.response.message), + stop_reason: finish_reason_to_api_stop_reason(&result.finish_reason), + usage: CompletionUsage { + input_tokens: result.usage.input_tokens, + output_tokens: result.usage.output_tokens, + }, + output: result.output, + }) + .into_response(), + Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) + .into_response(), + } + } else { + match client.complete(&request).await { + Ok(response) => Json(CompletionResponse { + id: response.id, + model: response.model, + message: convert_llm_message(&response.message), + stop_reason: finish_reason_to_api_stop_reason(&response.finish_reason), + usage: CompletionUsage { + input_tokens: response.usage.input_tokens, + output_tokens: response.usage.output_tokens, + }, + output: None, + }) + .into_response(), + Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) + .into_response(), + } + } + } +} diff --git a/lib/crates/fabro-server/src/server/handler/events.rs b/lib/crates/fabro-server/src/server/handler/events.rs new file mode 100644 index 000000000..ff3889cd5 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/events.rs @@ -0,0 +1,344 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope, + EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path, + Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse, + State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path, + redact_jsonl_line, reject_if_archived, update_live_run_from_event, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/attach", get(attach_events)) + .route( + "/runs/{id}/events", + get(list_run_events).post(append_run_event), + ) + .route("/runs/{id}/attach", get(attach_run_events)) +} + +#[derive(serde::Deserialize)] +struct EventListParams { + #[serde(default)] + since_seq: Option, + #[serde(default)] + limit: Option, +} + +impl EventListParams { + fn since_seq(&self) -> u32 { + self.since_seq.unwrap_or(1).max(1) + } + + fn limit(&self) -> usize { + self.limit.unwrap_or(100).clamp(1, 1000) + } +} + +#[derive(serde::Deserialize)] +struct AttachParams { + #[serde(default)] + since_seq: Option, +} + +#[derive(serde::Deserialize)] +struct GlobalAttachParams { + #[serde(default)] + run_id: Option, +} + +async fn attach_events( + _auth: RequiredUser, + State(state): State>, + Query(params): Query, +) -> Response { + let run_filter = match parse_global_run_filter(params.run_id.as_deref()) { + Ok(filter) => filter, + Err(err) => return ApiError::new(StatusCode::BAD_REQUEST, err).into_response(), + }; + + let stream = + filtered_global_events(state.global_event_tx.subscribe(), run_filter).filter_map(|event| { + sse_event_from_store(&event).map(Ok::) + }); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +pub(in crate::server) fn filtered_global_events( + event_rx: broadcast::Receiver, + run_filter: Option>, +) -> impl tokio_stream::Stream { + BroadcastStream::new(event_rx).filter_map(move |result| match result { + Ok(event) if event_matches_run_filter(&event, run_filter.as_ref()) => Some(event), + Ok(_) | Err(_) => None, + }) +} + +fn parse_global_run_filter(raw: Option<&str>) -> Result>, String> { + let Some(raw) = raw else { + return Ok(None); + }; + + let mut run_ids = HashSet::new(); + for part in raw + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + { + let run_id = part + .parse::() + .map_err(|err| format!("invalid run_id '{part}': {err}"))?; + run_ids.insert(run_id); + } + + if run_ids.is_empty() { + Ok(None) + } else { + Ok(Some(run_ids)) + } +} + +fn event_matches_run_filter(event: &EventEnvelope, run_filter: Option<&HashSet>) -> bool { + let Some(run_filter) = run_filter else { + return true; + }; + run_filter.contains(&event.event.run_id) +} + +fn sse_event_from_store(event: &EventEnvelope) -> Option { + let data = serde_json::to_string(event).ok()?; + let data = redact_jsonl_line(&data); + Some(Event::default().data(data)) +} + +fn attach_event_is_terminal(event: &EventEnvelope) -> bool { + matches!( + &event.event.body, + EventBody::RunCompleted(_) | EventBody::RunFailed(_) + ) +} + +fn run_projection_is_active(state: &fabro_store::RunProjection) -> bool { + state.status.is_some_and(RunStatus::is_active) +} + +async fn append_run_event( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, + Json(value): Json, +) -> Response { + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let event = match RunEvent::from_value(value.clone()) { + Ok(event) => event, + Err(err) => { + return ApiError::bad_request(format!("Invalid run event: {err}")).into_response(); + } + }; + if event.run_id != id { + return ApiError::bad_request("Event run_id does not match path run ID.").into_response(); + } + if let Some(denied) = denied_lifecycle_event_name(&event.body) { + return ApiError::bad_request(format!( + "{denied} is a lifecycle event; clients must call the corresponding operation endpoint instead of injecting it via append_run_event" + )) + .into_response(); + } + let payload = match EventPayload::new(value, &id) { + Ok(payload) => payload, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + + match state.store.open_run(&id).await { + Ok(run_store) => match run_store.append_event(&payload).await { + Ok(seq) => { + update_live_run_from_event(&state, id, &event); + Json(AppendEventResponse { + seq: i64::from(seq), + }) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + +async fn list_run_events( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, + Query(params): Query, +) -> Response { + let since_seq = params.since_seq(); + let limit = params.limit(); + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store + .list_events_from_with_limit(since_seq, limit) + .await + { + Ok(mut events) => { + let has_more = events.len() > limit; + events.truncate(limit); + Json(PaginatedEventList { + data: events, + meta: PaginationMeta { has_more }, + }) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + +async fn attach_run_events( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + const ATTACH_REPLAY_BATCH_LIMIT: usize = 256; + + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let start_seq = match params.since_seq { + Some(seq) if seq >= 1 => seq, + Some(_) => 1, + None => match run_store.list_events().await { + Ok(events) => events.last().map_or(1, |event| event.seq.saturating_add(1)), + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }, + }; + let (sender, receiver) = mpsc::unbounded_channel(); + tokio::spawn(async move { + let mut next_seq = start_seq; + + loop { + let Ok(replay_batch) = run_store + .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) + .await + else { + return; + }; + let replay_has_more = replay_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; + + for event in replay_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { + next_seq = event.seq.saturating_add(1); + let terminal = attach_event_is_terminal(&event); + if let Some(sse_event) = sse_event_from_store(&event) { + if sender + .send(Ok::(sse_event)) + .is_err() + { + return; + } + } + if terminal { + return; + } + } + + if replay_has_more { + continue; + } + + let Ok(state) = run_store.state().await else { + return; + }; + + if run_projection_is_active(&state) { + break; + } + + let Ok(tail_batch) = run_store + .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) + .await + else { + return; + }; + let tail_has_more = tail_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; + + for event in tail_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { + next_seq = event.seq.saturating_add(1); + let terminal = attach_event_is_terminal(&event); + if let Some(sse_event) = sse_event_from_store(&event) { + if sender + .send(Ok::(sse_event)) + .is_err() + { + return; + } + } + if terminal { + return; + } + } + + if tail_has_more { + continue; + } + + return; + } + + let Ok(mut live_stream) = run_store.watch_events_from(next_seq) else { + return; + }; + + while let Some(result) = live_stream.next().await { + let Ok(event) = result else { + return; + }; + let terminal = attach_event_is_terminal(&event); + if let Some(sse_event) = sse_event_from_store(&event) { + if sender + .send(Ok::(sse_event)) + .is_err() + { + return; + } + } + if terminal { + return; + } + } + }); + + Sse::new(UnboundedReceiverStream::new(receiver)) + .keep_alive(KeepAlive::default()) + .into_response() +} + +/// Returns the wire event name if the given body has a dedicated operation +/// endpoint that clients must use instead of injecting via `append_run_event`. +/// These endpoints enforce authorization and status-transition preconditions +/// (e.g. "archive only from terminal") that a direct event append would +/// bypass. Other run-lifecycle events flow through this endpoint legitimately: +/// the worker subprocess emits state transitions during execution. +fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> { + match body { + EventBody::RunArchived(_) => Some("run.archived"), + EventBody::RunUnarchived(_) => Some("run.unarchived"), + EventBody::RunCancelRequested(_) => Some("run.cancel.requested"), + EventBody::RunPauseRequested(_) => Some("run.pause.requested"), + EventBody::RunUnpauseRequested(_) => Some("run.unpause.requested"), + _ => None, + } +} diff --git a/lib/crates/fabro-server/src/server/handler/graph.rs b/lib/crates/fabro-server/src/server/handler/graph.rs new file mode 100644 index 000000000..ea62bf5dd --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/graph.rs @@ -0,0 +1,298 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, AsyncWriteExt, Command, EnvVars, IntoResponse, Json, LazyLock, Path, + PathBuf, Query, RenderWorkflowGraphDirection, RenderWorkflowGraphRequest, RequiredUser, + Response, Router, RunId, Semaphore, State, StatusCode, Stdio, apply_render_graph_env, get, + parse_run_id_path, post, run_manifest, +}; + +pub(super) fn manifest_routes() -> Router> { + Router::new().route("/graph/render", post(render_graph_from_manifest)) +} + +pub(super) fn run_routes() -> Router> { + Router::new() + .route("/runs/{id}/graph", get(get_graph)) + .route("/runs/{id}/graph/source", get(get_graph_source)) +} + +const RENDER_ERROR_PREFIX: &[u8] = b"RENDER_ERROR:"; +const GRAPHVIZ_RENDER_CONCURRENCY_LIMIT: usize = 4; + +static GRAPHVIZ_RENDER_SEMAPHORE: LazyLock = + LazyLock::new(|| Semaphore::new(GRAPHVIZ_RENDER_CONCURRENCY_LIMIT)); + +#[derive(Debug, thiserror::Error)] +pub(in crate::server) enum RenderSubprocessError { + #[error("failed to spawn render subprocess: {0}")] + SpawnFailed(String), + #[error("render subprocess crashed: {0}")] + ChildCrashed(String), + #[error("render subprocess returned invalid output: {0}")] + ProtocolViolation(String), + #[error("{0}")] + RenderFailed(String), +} + +async fn render_graph_from_manifest( + _auth: RequiredUser, + State(state): State>, + Json(req): Json, +) -> Response { + let manifest_run_defaults = state.manifest_run_defaults(); + let prepared = + match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req.manifest) { + Ok(prepared) => prepared, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let validated = match run_manifest::validate_prepared_manifest(&prepared) { + Ok(validated) => validated, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + if validated.has_errors() { + return ApiError::bad_request("Validation failed").into_response(); + } + + let direction = req.direction.as_ref().map(|direction| match direction { + RenderWorkflowGraphDirection::Lr => "LR", + RenderWorkflowGraphDirection::Tb => "TB", + }); + let dot_source = run_manifest::graph_source(&prepared, direction); + render_graph_bytes(&dot_source).await +} + +#[expect( + clippy::disallowed_methods, + reason = "Render-graph subprocess startup resolves Cargo's test binary env override when present." +)] +fn render_graph_subprocess_exe( + exe_override: Option<&std::path::Path>, +) -> Result { + if let Some(path) = exe_override { + Ok(path.to_path_buf()) + } else { + if let Some(path) = std::env::var_os(EnvVars::CARGO_BIN_EXE_FABRO).map(PathBuf::from) { + return Ok(path); + } + + let current = std::env::current_exe() + .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; + let current_name = current.file_stem().and_then(|name| name.to_str()); + if current_name == Some("fabro") { + return Ok(current); + } + + let candidate = current + .parent() + .and_then(|parent| parent.parent()) + .map(|parent| parent.join(if cfg!(windows) { "fabro.exe" } else { "fabro" })); + if let Some(candidate) = candidate.filter(|path| path.is_file()) { + return Ok(candidate); + } + + Ok(current) + } +} + +fn render_subprocess_failure( + status: std::process::ExitStatus, + stderr: &[u8], +) -> RenderSubprocessError { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + + if let Some(signal) = status.signal() { + let stderr = String::from_utf8_lossy(stderr).trim().to_string(); + let detail = if stderr.is_empty() { + format!("terminated by signal {signal}") + } else { + format!("terminated by signal {signal}: {stderr}") + }; + return RenderSubprocessError::ChildCrashed(detail); + } + } + + let stderr = String::from_utf8_lossy(stderr).trim().to_string(); + let detail = match status.code() { + Some(code) if stderr.is_empty() => format!("exited with status {code}"), + Some(code) => format!("exited with status {code}: {stderr}"), + None if stderr.is_empty() => "child exited unsuccessfully".to_string(), + None => format!("child exited unsuccessfully: {stderr}"), + }; + RenderSubprocessError::ChildCrashed(detail) +} + +pub(in crate::server) async fn render_dot_subprocess( + styled_source: &str, + exe_override: Option<&std::path::Path>, +) -> Result, RenderSubprocessError> { + let _permit = GRAPHVIZ_RENDER_SEMAPHORE + .acquire() + .await + .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; + let exe = render_graph_subprocess_exe(exe_override)?; + let mut cmd = Command::new(exe); + apply_render_graph_env(&mut cmd); + cmd.arg("__render-graph") + .env(EnvVars::FABRO_TELEMETRY, "off") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = cmd + .spawn() + .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; + let mut stdin = child.stdin.take().ok_or_else(|| { + RenderSubprocessError::SpawnFailed("render subprocess stdin was not piped".to_string()) + })?; + if let Err(err) = stdin.write_all(styled_source.as_bytes()).await { + drop(stdin); + let output = child + .wait_with_output() + .await + .map_err(|wait_err| RenderSubprocessError::SpawnFailed(wait_err.to_string()))?; + return Err(RenderSubprocessError::ChildCrashed(format!( + "failed writing DOT to child stdin: {err}; {}", + render_subprocess_failure(output.status, &output.stderr) + ))); + } + drop(stdin); + + let output = child + .wait_with_output() + .await + .map_err(|err| RenderSubprocessError::SpawnFailed(err.to_string()))?; + + if !output.status.success() { + return Err(render_subprocess_failure(output.status, &output.stderr)); + } + + if let Some(error) = output.stdout.strip_prefix(RENDER_ERROR_PREFIX) { + return Err(RenderSubprocessError::RenderFailed( + String::from_utf8_lossy(error).trim().to_string(), + )); + } + + if output.stdout.starts_with(b", +) -> Response { + use fabro_graphviz::render::{inject_dot_style_defaults, postprocess_svg}; + + let styled_source = inject_dot_style_defaults(dot_source); + match render_dot_subprocess(&styled_source, exe_override).await { + Ok(raw) => { + let bytes = postprocess_svg(raw); + (StatusCode::OK, [("content-type", "image/svg+xml")], bytes).into_response() + } + Err(RenderSubprocessError::RenderFailed(err)) => { + ApiError::new(StatusCode::BAD_REQUEST, err).into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +pub(crate) async fn render_graph_bytes(dot_source: &str) -> Response { + render_graph_response(dot_source, None).await +} + +#[cfg(test)] +pub(in crate::server) async fn render_graph_bytes_with_exe_override( + dot_source: &str, + exe_override: Option<&std::path::Path>, +) -> Response { + render_graph_response(dot_source, exe_override).await +} + +#[derive(serde::Deserialize)] +struct GraphParams { + #[serde(default)] + direction: Option, +} + +async fn load_run_dot_source(state: &AppState, id: &RunId) -> Result { + let live_dot_source = { + let runs = state.runs.lock().expect("runs lock poisoned"); + runs.get(id) + .map(|managed_run| managed_run.dot_source.clone()) + }; + + let dot_source = if let Some(dot) = live_dot_source.filter(|d| !d.is_empty()) { + Some(dot) + } else { + match state.store.open_run_reader(id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => run_state.graph_source, + Err(err) => { + return Err( + ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response() + ); + } + }, + Err(_) => return Err(ApiError::not_found("Run not found.").into_response()), + } + }; + + dot_source + .ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "Graph not found.").into_response()) +} + +async fn get_graph( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + + let dot = match load_run_dot_source(&state, &id).await { + Ok(dot) => dot, + Err(response) => return response, + }; + + let dot = match params.direction.as_deref() { + Some(dir @ ("LR" | "TB" | "BT" | "RL")) => { + use fabro_graphviz::render; + render::apply_direction(&dot, dir).into_owned() + } + _ => dot, + }; + + render_graph_bytes(&dot).await +} + +async fn get_graph_source( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + + match load_run_dot_source(&state, &id).await { + Ok(dot) => (StatusCode::OK, [("content-type", "text/vnd.graphviz")], dot).into_response(), + Err(response) => response, + } +} diff --git a/lib/crates/fabro-server/src/server/handler/lifecycle.rs b/lib/crates/fabro-server/src/server/handler/lifecycle.rs new file mode 100644 index 000000000..c0e4ede05 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/lifecycle.rs @@ -0,0 +1,797 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, FailureReason, ForkRequest, ForkResponse, IntoResponse, Json, Ordering, + Path, Principal, RequiredUser, Response, RewindRequest, RewindResponse, Router, + RunAnswerTransport, RunControlAction, RunExecutionMode, RunId, RunStatus, RunStatusResponse, + StartRunRequest, State, StatusCode, Storage, TimelineEntryResponse, WORKER_CANCEL_GRACE, + WorkflowError, append_control_request, get, load_pending_control, managed_run, operations, + parse_run_id_path, persist_cancelled_run_status, post, reject_if_archived, sleep, + update_live_run_from_event, workflow_event, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/runs/{id}/cancel", post(cancel_run)) + .route("/runs/{id}/start", post(start_run)) + .route("/runs/{id}/pause", post(pause_run)) + .route("/runs/{id}/unpause", post(unpause_run)) + .route("/runs/{id}/archive", post(archive_run)) + .route("/runs/{id}/rewind", post(rewind_run)) + .route("/runs/{id}/fork", post(fork_run)) + .route("/runs/{id}/timeline", get(run_timeline)) + .route("/runs/{id}/unarchive", post(unarchive_run)) +} + +async fn start_run( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + body: Option>, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let resume = body.is_some_and(|Json(req)| req.resume); + + { + let runs = state.runs.lock().expect("runs lock poisoned"); + if let Some(managed_run) = runs.get(&id) { + if matches!( + managed_run.status, + RunStatus::Queued + | RunStatus::Starting + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } + ) { + return ApiError::new( + StatusCode::CONFLICT, + if resume { + "an engine process is still running for this run — cannot resume" + } else { + "an engine process is still running for this run — cannot start" + }, + ) + .into_response(); + } + } + } + + let Ok(run_store) = state.store.open_run(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let run_state = match run_store.state().await { + Ok(state) => state, + Err(err) => { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to load run state: {err}"), + ) + .into_response(); + } + }; + + if resume { + if run_state.checkpoint.is_none() { + return ApiError::new(StatusCode::CONFLICT, "no checkpoint to resume from") + .into_response(); + } + } else if let Some(status) = run_state.status { + if !matches!( + status, + RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting + ) { + return ApiError::new( + StatusCode::CONFLICT, + format!("cannot start run: status is {status}, expected submitted"), + ) + .into_response(); + } + } + + if run_state.spec.is_none() { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "run spec missing from store", + ) + .into_response(); + } + let run_dir = Storage::new(state.server_storage_dir()) + .run_scratch(&id) + .root() + .to_path_buf(); + let dot_source = run_state.graph_source.unwrap_or_default(); + if let Err(err) = + workflow_event::append_event(&run_store, &id, &workflow_event::Event::RunQueued).await + { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + runs.insert( + id, + managed_run( + dot_source, + RunStatus::Queued, + id.created_at(), + run_dir, + if resume { + RunExecutionMode::Resume + } else { + RunExecutionMode::Start + }, + ), + ); + } + + state.scheduler_notify.notify_one(); + ( + StatusCode::OK, + Json(RunStatusResponse { + id: id.to_string(), + status: RunStatus::Queued, + error: None, + queue_position: None, + pending_control: None, + created_at: id.created_at(), + }), + ) + .into_response() +} + +fn schedule_worker_kill(state: Arc, run_id: RunId, worker_pid: u32) { + tokio::spawn(async move { + sleep(WORKER_CANCEL_GRACE).await; + let current_pid = { + let runs = state.runs.lock().expect("runs lock poisoned"); + runs.get(&run_id).and_then(|run| run.worker_pid) + }; + if current_pid == Some(worker_pid) && fabro_proc::process_group_alive(worker_pid) { + #[cfg(unix)] + fabro_proc::sigkill_process_group(worker_pid); + } + }); +} + +async fn cancel_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let ( + created_at, + response_status, + persist_cancelled_status, + answer_transport, + cancel_token, + cancel_tx, + worker_pid, + ) = { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get_mut(&id) { + Some(managed_run) => match managed_run.status { + RunStatus::Submitted + | RunStatus::Queued + | RunStatus::Starting + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } => { + let use_cancel_signal = !matches!( + managed_run.answer_transport, + Some(RunAnswerTransport::InProcess { .. }) + ); + let persist_cancelled_status = + matches!(managed_run.status, RunStatus::Submitted | RunStatus::Queued); + let response_status = if persist_cancelled_status { + let cancelled = RunStatus::Failed { + reason: FailureReason::Cancelled, + }; + managed_run.status = cancelled; + cancelled + } else { + managed_run.status + }; + ( + managed_run.created_at, + response_status, + persist_cancelled_status, + managed_run.answer_transport.clone(), + managed_run.cancel_token.clone(), + use_cancel_signal + .then(|| managed_run.cancel_tx.take()) + .flatten(), + managed_run.worker_pid, + ) + } + _ => { + return ApiError::new(StatusCode::CONFLICT, "Run is not cancellable.") + .into_response(); + } + }, + None => return ApiError::not_found("Run not found.").into_response(), + } + }; + + if pending_control != Some(RunControlAction::Cancel) { + if let Err(err) = append_control_request( + state.as_ref(), + id, + RunControlAction::Cancel, + Some(Principal::User(subject.0.clone())), + ) + .await + { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + + if let Some(token) = &cancel_token { + token.store(true, Ordering::SeqCst); + } + let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx { + let _ = cancel_tx.send(()); + true + } else { + false + }; + if let Some(answer_transport) = answer_transport { + if !(sent_cancel_signal && matches!(answer_transport, RunAnswerTransport::InProcess { .. })) + { + let _ = answer_transport.cancel_run().await; + } + } + if let Some(worker_pid) = worker_pid { + #[cfg(unix)] + fabro_proc::sigterm(worker_pid); + schedule_worker_kill(Arc::clone(&state), id, worker_pid); + } + + if persist_cancelled_status { + if let Err(err) = persist_cancelled_run_status(state.as_ref(), id).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + ( + StatusCode::OK, + Json(RunStatusResponse { + id: id.to_string(), + status: response_status, + error: None, + queue_position: None, + pending_control, + created_at, + }), + ) + .into_response() +} + +/// How `pause_run` should enact the transition, chosen from the current run +/// status. +enum PauseMode { + /// Worker is running; ask it to pause via SIGUSR1. Status flips to + /// `Paused` once the worker acknowledges. + Signal { worker_pid: u32 }, + /// Worker is blocked on a human gate; flip to `Paused` directly by + /// appending `RunPaused` ourselves. + AppendEvent, +} + +/// How `unpause_run` should enact the transition. +enum UnpauseMode { + /// No outstanding block; ask the worker to resume via SIGUSR2. + Signal { worker_pid: u32 }, + /// Was paused while blocked; append `RunUnpaused` and let the reducer + /// restore the underlying blocked state from `Paused { prior_block }`. + AppendEvent, +} + +async fn pause_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let (created_at, mode) = { + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) if managed_run.status == RunStatus::Running => { + let Some(worker_pid) = managed_run.worker_pid else { + return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") + .into_response(); + }; + (managed_run.created_at, PauseMode::Signal { worker_pid }) + } + Some(managed_run) if matches!(managed_run.status, RunStatus::Blocked { .. }) => { + (managed_run.created_at, PauseMode::AppendEvent) + } + Some(_) => { + return ApiError::new(StatusCode::CONFLICT, "Run is not pausable.").into_response(); + } + None => return ApiError::not_found("Run not found.").into_response(), + } + }; + + if pending_control.is_some() { + return ApiError::new( + StatusCode::CONFLICT, + "Run control request is already pending.", + ) + .into_response(); + } + if let Err(err) = append_control_request( + state.as_ref(), + id, + RunControlAction::Pause, + Some(Principal::User(subject.0.clone())), + ) + .await + { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + let response_status = match mode { + PauseMode::Signal { worker_pid } => { + #[cfg(unix)] + fabro_proc::sigusr1(worker_pid); + #[cfg(not(unix))] + let _ = worker_pid; + RunStatus::Running + } + PauseMode::AppendEvent => { + if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { + events.push(workflow_event::Event::RunPaused); + }) + .await + { + return response; + } + state + .runs + .lock() + .expect("runs lock poisoned") + .get(&id) + .map_or(RunStatus::Paused { prior_block: None }, |run| run.status) + } + }; + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + ( + StatusCode::OK, + Json(RunStatusResponse { + id: id.to_string(), + status: response_status, + error: None, + queue_position: None, + pending_control, + created_at, + }), + ) + .into_response() +} + +async fn unpause_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let (created_at, mode) = { + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) => match managed_run.status { + RunStatus::Paused { + prior_block: Some(_), + } => (managed_run.created_at, UnpauseMode::AppendEvent), + RunStatus::Paused { prior_block: None } => { + let Some(worker_pid) = managed_run.worker_pid else { + return ApiError::new(StatusCode::CONFLICT, "Run worker is not available.") + .into_response(); + }; + (managed_run.created_at, UnpauseMode::Signal { worker_pid }) + } + _ => { + return ApiError::new(StatusCode::CONFLICT, "Run is not paused.") + .into_response(); + } + }, + None => return ApiError::not_found("Run not found.").into_response(), + } + }; + + if pending_control.is_some() { + return ApiError::new( + StatusCode::CONFLICT, + "Run control request is already pending.", + ) + .into_response(); + } + if let Err(err) = append_control_request( + state.as_ref(), + id, + RunControlAction::Unpause, + Some(Principal::User(subject.0.clone())), + ) + .await + { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + let response_status = match mode { + UnpauseMode::Signal { worker_pid } => { + #[cfg(unix)] + fabro_proc::sigusr2(worker_pid); + #[cfg(not(unix))] + let _ = worker_pid; + RunStatus::Paused { prior_block: None } + } + UnpauseMode::AppendEvent => { + if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { + events.push(workflow_event::Event::RunUnpaused); + }) + .await + { + return response; + } + state + .runs + .lock() + .expect("runs lock poisoned") + .get(&id) + .map_or(RunStatus::Running, |run| run.status) + } + }; + let pending_control = match load_pending_control(state.as_ref(), id).await { + Ok(pending_control) => pending_control, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + ( + StatusCode::OK, + Json(RunStatusResponse { + id: id.to_string(), + status: response_status, + error: None, + queue_position: None, + pending_control, + created_at, + }), + ) + .into_response() +} + +async fn archive_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + run_archive_action(state, subject, id, ArchiveAction::Archive).await +} + +async fn unarchive_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + run_archive_action(state, subject, id, ArchiveAction::Unarchive).await +} + +async fn rewind_run( + subject: RequiredUser, + State(state): State>, + Path(id): Path, + body: Option>, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let request = body.map(|Json(body)| body).unwrap_or_default(); + let target = match parse_fork_target(request.target) { + Ok(target) => target, + Err(err) => return err.into_response(), + }; + let input = operations::RewindInput { run_id: id, target }; + match Box::pin(operations::rewind( + &state.store, + &input, + Some(Principal::User(subject.0.clone())), + )) + .await + { + Ok(operations::RewindOutcome::Full { + source_run_id, + new_run_id, + target, + }) => ( + StatusCode::OK, + Json(RewindResponse { + source_run_id: source_run_id.to_string(), + new_run_id: new_run_id.to_string(), + target: target.response_target(), + archived: true, + archive_error: None, + }), + ) + .into_response(), + Ok(operations::RewindOutcome::Partial { + source_run_id, + new_run_id, + target, + archive_error, + }) => ( + StatusCode::MULTI_STATUS, + Json(RewindResponse { + source_run_id: source_run_id.to_string(), + new_run_id: new_run_id.to_string(), + target: target.response_target(), + archived: false, + archive_error: Some(archive_error), + }), + ) + .into_response(), + Err(err) => workflow_operation_error_response(err), + } +} + +async fn fork_run( + _subject: RequiredUser, + State(state): State>, + Path(id): Path, + body: Option>, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let request = body.map(|Json(body)| body).unwrap_or_default(); + let target = match parse_fork_target(request.target) { + Ok(target) => target, + Err(err) => return err.into_response(), + }; + let input = operations::ForkRunInput { + source_run_id: id, + target, + }; + match Box::pin(operations::fork_run(&state.store, &input)).await { + Ok(outcome) => ( + StatusCode::OK, + Json(ForkResponse { + source_run_id: outcome.source_run_id.to_string(), + new_run_id: outcome.new_run_id.to_string(), + target: outcome.target.response_target(), + }), + ) + .into_response(), + Err(err) => workflow_operation_error_response(err), + } +} + +async fn run_timeline( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + match operations::timeline(&state.store, &id).await { + Ok(entries) => Json( + entries + .into_iter() + .map(|entry| TimelineEntryResponse { + ordinal: std::num::NonZeroU64::new(entry.ordinal as u64) + .expect("timeline ordinals start at 1"), + node_name: entry.node_name, + visit: std::num::NonZeroU64::new(entry.visit as u64) + .expect("timeline visits start at 1"), + checkpoint_seq: std::num::NonZeroU64::new(u64::from(entry.checkpoint_seq)) + .expect("checkpoint event sequence starts at 1"), + run_commit_sha: entry.run_commit_sha, + }) + .collect::>(), + ) + .into_response(), + Err(err) => workflow_operation_error_response(err), + } +} + +fn parse_fork_target(target: Option) -> Result, ApiError> { + target + .map(|target| { + target + .parse::() + .map_err(|err| ApiError::bad_request(err.to_string())) + }) + .transpose() +} + +fn workflow_operation_error_response(err: WorkflowError) -> Response { + match err { + WorkflowError::Parse(message) | WorkflowError::Validation(message) => { + ApiError::bad_request(message).into_response() + } + WorkflowError::ValidationFailed { .. } => { + ApiError::bad_request("Validation failed").into_response() + } + WorkflowError::Precondition(message) => { + ApiError::new(StatusCode::CONFLICT, message).into_response() + } + WorkflowError::RunNotFound(_) => ApiError::not_found("Run not found.").into_response(), + WorkflowError::Unsupported(message) => { + ApiError::new(StatusCode::NOT_IMPLEMENTED, message).into_response() + } + err => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + } +} + +#[derive(Clone, Copy)] +enum ArchiveAction { + Archive, + Unarchive, +} + +async fn run_archive_action( + state: Arc, + subject: RequiredUser, + id: String, + action: ArchiveAction, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let actor = Some(Principal::User(subject.0.clone())); + let result = match action { + ArchiveAction::Archive => operations::archive(&state.store, &id, actor) + .await + .map(|_| ()), + ArchiveAction::Unarchive => operations::unarchive(&state.store, &id, actor) + .await + .map(|_| ()), + }; + match result { + Ok(()) => archive_status_response(state.as_ref(), id).await, + Err(WorkflowError::Precondition(message)) => { + ApiError::new(StatusCode::CONFLICT, message).into_response() + } + Err(WorkflowError::RunNotFound(_)) => ApiError::not_found("Run not found.").into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +/// Build a `RunStatusResponse` reflecting the durable projection after an +/// archive/unarchive transition. The run is terminal in both directions, so no +/// live queue position or worker-only fields apply. +async fn archive_status_response(state: &AppState, id: RunId) -> Response { + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let projection = match run_store.state().await { + Ok(projection) => projection, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let Some(status) = projection.status else { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "run has no status after archive/unarchive", + ) + .into_response(); + }; + ( + StatusCode::OK, + Json(RunStatusResponse { + id: id.to_string(), + status, + error: None, + queue_position: None, + pending_control: None, + created_at: id.created_at(), + }), + ) + .into_response() +} + +/// Persist a synchronous pause/unpause transition: append the caller-supplied +/// events to the run store and mirror the new status in the in-memory run map. +/// Returns `Some(Response)` on error, `None` on success. +async fn synchronous_transition( + state: &AppState, + id: RunId, + append_events: impl FnOnce(&mut Vec), +) -> Option { + let run_store = match state.store.open_run(&id).await { + Ok(run_store) => run_store, + Err(err) => { + return Some( + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + ); + } + }; + let mut events = Vec::new(); + append_events(&mut events); + for event in events { + if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { + return Some( + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + ); + } + let stored = workflow_event::to_run_event(&id, &event); + update_live_run_from_event(state, id, &stored); + } + None +} diff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs new file mode 100644 index 000000000..d2625395b --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/mod.rs @@ -0,0 +1,146 @@ +use std::sync::Arc; + +use axum::Router; +use axum::routing::{get, post}; + +use super::{ApiError, AppState, IntoResponse, Response, StatusCode, demo}; + +mod artifacts; +mod billing; +mod completions; +pub(in crate::server) mod events; +pub(in crate::server) mod graph; +mod lifecycle; +mod models; +mod pull_requests; +mod runs; +mod sandbox; +mod secrets; +pub(in crate::server) mod system; + +pub(super) use system::{health, openapi_spec}; + +async fn not_implemented() -> Response { + ApiError::new(StatusCode::NOT_IMPLEMENTED, "Not implemented.").into_response() +} + +pub(super) fn demo_routes() -> Router> { + Router::new() + .route("/runs", get(demo::list_runs).post(demo::create_run_stub)) + .route("/runs/resolve", get(demo::resolve_run)) + .route("/boards/runs", get(demo::list_board_runs)) + .route("/attach", get(demo::attach_events_stub)) + .route("/runs/{id}", get(demo::get_run_status)) + .route("/runs/{id}/questions", get(demo::get_questions_stub)) + .route("/runs/{id}/questions/{qid}/answer", post(demo::answer_stub)) + .route("/runs/{id}/state", get(not_implemented)) + .route("/runs/{id}/logs", get(not_implemented)) + .route( + "/runs/{id}/events", + get(not_implemented).post(not_implemented), + ) + .route("/runs/{id}/attach", get(demo::run_events_stub)) + .route("/runs/{id}/blobs", post(not_implemented)) + .route("/runs/{id}/blobs/{blobId}", get(not_implemented)) + .route( + "/runs/{id}/stages/{stageId}/logs/{stream}", + get(not_implemented), + ) + .route("/runs/{id}/checkpoint", get(demo::checkpoint_stub)) + .route("/runs/{id}/cancel", post(demo::cancel_stub)) + .route("/runs/{id}/start", post(demo::start_run_stub)) + .route("/runs/{id}/pause", post(demo::pause_stub)) + .route("/runs/{id}/unpause", post(demo::unpause_stub)) + .route("/runs/{id}/graph", get(demo::get_run_graph)) + .route("/runs/{id}/graph/source", get(demo::get_run_graph_source)) + .route("/runs/{id}/stages", get(demo::get_run_stages)) + .route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub)) + .route("/runs/{id}/files", get(demo::list_run_files_stub)) + .route( + "/runs/{id}/stages/{stageId}/turns", + get(demo::get_stage_turns), + ) + .route( + "/runs/{id}/stages/{stageId}/artifacts", + get(not_implemented).post(not_implemented), + ) + .route( + "/runs/{id}/stages/{stageId}/artifacts/download", + get(not_implemented), + ) + .route("/runs/{id}/billing", get(demo::get_run_billing)) + .route("/runs/{id}/settings", get(demo::get_run_settings)) + .route("/runs/{id}/preview", post(demo::generate_preview_url_stub)) + .route("/runs/{id}/ssh", post(demo::create_ssh_access_stub)) + .route( + "/runs/{id}/sandbox/files", + get(demo::list_sandbox_files_stub), + ) + .route( + "/runs/{id}/sandbox/file", + get(demo::get_sandbox_file_stub).put(demo::put_sandbox_file_stub), + ) + .route( + "/insights/queries", + get(demo::list_saved_queries).post(demo::save_query_stub), + ) + .route( + "/insights/queries/{id}", + get(demo::get_saved_query) + .put(demo::update_query_stub) + .delete(demo::delete_query_stub), + ) + .route("/insights/execute", post(demo::execute_query_stub)) + .route("/insights/history", get(demo::list_query_history)) + .route( + "/secrets", + get(demo::list_secrets) + .post(demo::create_secret) + .delete(demo::delete_secret_by_name), + ) + .route("/repos/github/{owner}/{name}", get(demo::get_github_repo)) + .route("/health/diagnostics", post(demo::run_diagnostics)) + .route("/settings", get(demo::get_server_settings)) + .route("/system/info", get(demo::get_system_info)) + .route("/system/df", get(demo::get_system_disk_usage)) + .route("/system/prune/runs", post(demo::prune_runs)) + .route("/billing", get(demo::get_aggregate_billing)) + .merge(runs::manifest_routes()) + .merge(graph::manifest_routes()) + .merge(models::routes()) + .merge(completions::routes()) +} + +pub(super) fn real_routes() -> Router> { + Router::new() + .route("/runs/{id}/stages/{stageId}/turns", get(not_implemented)) + .route("/runs/{id}/steer", post(not_implemented)) + .route("/workflows", get(not_implemented)) + .route("/workflows/{name}", get(not_implemented)) + .route("/workflows/{name}/runs", get(not_implemented)) + .route( + "/insights/queries", + get(not_implemented).post(not_implemented), + ) + .route( + "/insights/queries/{id}", + get(not_implemented) + .put(not_implemented) + .delete(not_implemented), + ) + .route("/insights/execute", post(not_implemented)) + .route("/insights/history", get(not_implemented)) + .merge(runs::routes()) + .merge(events::routes()) + .merge(billing::routes()) + .merge(pull_requests::routes()) + .merge(artifacts::routes()) + .merge(sandbox::routes()) + .merge(lifecycle::routes()) + .merge(graph::manifest_routes()) + .merge(graph::run_routes()) + .merge(models::routes()) + .merge(secrets::routes()) + .merge(system::routes()) + .merge(completions::routes()) +} diff --git a/lib/crates/fabro-server/src/server/handler/models.rs b/lib/crates/fabro-server/src/server/handler/models.rs new file mode 100644 index 000000000..00ad1449e --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/models.rs @@ -0,0 +1,154 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, FromStr, HashSet, IntoResponse, Json, MAX_PAGE_OFFSET, ModelTestMode, Path, + Provider, Query, RequiredUser, Response, Router, State, StatusCode, auth_issue_message, + default_page_limit, error, get, post, run_model_test, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/models", get(list_models)) + .route("/models/{id}/test", post(test_model)) +} + +#[derive(serde::Deserialize)] +struct ModelListParams { + #[serde(rename = "page[limit]", default = "default_page_limit")] + limit: u32, + #[serde(rename = "page[offset]", default)] + offset: u32, + #[serde(default)] + provider: Option, + #[serde(default)] + query: Option, +} + +#[derive(serde::Deserialize)] +struct ModelTestParams { + #[serde(default)] + mode: Option, +} + +async fn list_models( + _auth: RequiredUser, + State(state): State>, + Query(params): Query, +) -> Response { + let provider = match params.provider.as_deref() { + Some(value) => match Provider::from_str(value) { + Ok(provider) => Some(provider), + Err(_) => { + return ApiError::new( + StatusCode::BAD_REQUEST, + format!("unknown provider: {value}"), + ) + .into_response(); + } + }, + None => None, + }; + + let query = params.query.as_ref().map(|value| value.to_lowercase()); + let limit = params.limit.clamp(1, 100) as usize; + let offset = params.offset.min(MAX_PAGE_OFFSET) as usize; + let configured: HashSet = state + .llm_source + .configured_providers() + .await + .into_iter() + .collect(); + + let mut models = fabro_model::Catalog::builtin() + .list(provider) + .into_iter() + .filter(|model| match &query { + Some(query) => { + model.id.to_lowercase().contains(query) + || model.display_name.to_lowercase().contains(query) + || model + .aliases + .iter() + .any(|alias| alias.to_lowercase().contains(query)) + } + None => true, + }) + .cloned() + .map(|mut model| { + model.configured = configured.contains(&model.provider); + model + }) + .collect::>(); + + let has_more = models.len() > offset.saturating_add(limit); + let data = models.drain(offset..models.len().min(offset.saturating_add(limit))); + + ( + StatusCode::OK, + Json(serde_json::json!({ + "data": data.collect::>(), + "meta": { "has_more": has_more } + })), + ) + .into_response() +} + +async fn test_model( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let mode = match params.mode.as_deref() { + Some(value) => match ModelTestMode::from_str(value) { + Ok(mode) => mode, + Err(_) => { + return ApiError::new( + StatusCode::BAD_REQUEST, + format!("invalid model test mode: {value}"), + ) + .into_response(); + } + }, + None => ModelTestMode::Basic, + }; + let Some(info) = fabro_model::Catalog::builtin().get(&id) else { + return ApiError::not_found(format!("Model not found: {id}")).into_response(); + }; + + let llm_result = match state.resolve_llm_client().await { + Ok(result) => result, + Err(err) => { + error!(error = ?err, "Failed to resolve LLM client"); + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to resolve LLM client: {err}"), + ) + .into_response(); + } + }; + if let Some((_, issue)) = llm_result + .auth_issues + .iter() + .find(|(provider, _)| *provider == info.provider) + { + return ApiError::bad_request(auth_issue_message(info.provider, issue)).into_response(); + } + let provider_name = <&'static str>::from(info.provider); + if !llm_result.client.provider_names().contains(&provider_name) { + return Json(serde_json::json!({ + "model_id": info.id, + "status": "skip", + })) + .into_response(); + } + let client = Arc::new(llm_result.client); + + let outcome = run_model_test(info, mode, client).await; + Json(serde_json::json!({ + "model_id": info.id, + "status": <&'static str>::from(outcome.status), + "error_message": outcome.error_message, + })) + .into_response() +} diff --git a/lib/crates/fabro-server/src/server/handler/pull_requests.rs b/lib/crates/fabro-server/src/server/handler/pull_requests.rs new file mode 100644 index 000000000..99ad4c4a6 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/pull_requests.rs @@ -0,0 +1,399 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, Catalog, CloseRunPullRequestResponse, CreateRunPullRequestRequest, + IntoResponse, Json, MergeRunPullRequestRequest, MergeRunPullRequestResponse, PullRequestRecord, + RequireRunScoped, Response, Router, RunId, State, StatusCode, get, lock_pull_request_create, + post, pull_request, warn, workflow_event, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route( + "/runs/{id}/pull_request", + get(get_run_pull_request).post(create_run_pull_request), + ) + .route( + "/runs/{id}/pull_request/merge", + post(merge_run_pull_request), + ) + .route( + "/runs/{id}/pull_request/close", + post(close_run_pull_request), + ) +} + +#[expect( + clippy::disallowed_types, + reason = "Pull-request API validates public github.com URLs; these raw URLs are not credential-bearing log output." +)] +fn parse_github_owner_repo_from_url(url: &str, kind: &str) -> Result<(String, String), ApiError> { + let parsed = fabro_http::Url::parse(url) + .map_err(|err| ApiError::bad_request(format!("Invalid {kind}: {err}")))?; + match parsed.host_str() { + Some("github.com") => {} + Some(host) => { + return Err(ApiError::with_code( + StatusCode::BAD_REQUEST, + format!("Pull request operations support github.com only (got {host})."), + "unsupported_host", + )); + } + None => { + return Err(ApiError::bad_request(format!( + "Invalid {kind}: missing host" + ))); + } + } + + fabro_github::parse_github_owner_repo(url).map_err(|err| ApiError::bad_request(err.to_string())) +} + +fn load_server_github_credentials( + state: &AppState, +) -> Result { + let settings = state.server_settings(); + match state.github_credentials(&settings.server.integrations.github) { + Ok(Some(creds)) => Ok(creds), + Ok(None) => { + warn!("GitHub integration unavailable on server: credentials not configured"); + Err(ApiError::with_code( + StatusCode::SERVICE_UNAVAILABLE, + "GitHub integration unavailable on server.", + "integration_unavailable", + )) + } + Err(err) => { + warn!(error = %err, "GitHub integration unavailable on server"); + Err(ApiError::with_code( + StatusCode::SERVICE_UNAVAILABLE, + "GitHub integration unavailable on server.", + "integration_unavailable", + )) + } + } +} + +fn server_github_context<'a>( + state: &'a AppState, + creds: &'a fabro_github::GitHubCredentials, +) -> Result, ApiError> { + let http_client = state.http_client().map_err(|err| { + ApiError::with_code( + StatusCode::SERVICE_UNAVAILABLE, + format!("GitHub integration unavailable on server: {err}"), + "integration_unavailable", + ) + })?; + Ok(fabro_github::GitHubContext::with_http_client( + creds, + state.github_api_base_url.as_str(), + http_client, + )) +} + +fn github_pull_request_not_found_error(record: &PullRequestRecord) -> ApiError { + ApiError::with_code( + StatusCode::BAD_GATEWAY, + format!("Pull request #{} was deleted on GitHub.", record.number), + "github_not_found", + ) +} + +struct PullRequestGithubContext { + record: PullRequestRecord, + creds: fabro_github::GitHubCredentials, +} + +async fn load_pull_request_github_context( + state: &Arc, + id: &RunId, +) -> Result { + let run_store = state + .store + .open_run_reader(id) + .await + .map_err(|_| ApiError::not_found("Run not found."))?; + let run_state = run_store + .state() + .await + .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + let record = run_state.pull_request.ok_or_else(|| { + ApiError::with_code( + StatusCode::NOT_FOUND, + format!("No pull request found in store. Create one first with: fabro pr create {id}"), + "no_stored_record", + ) + })?; + parse_github_owner_repo_from_url(&record.html_url, "pull request URL")?; + let creds = load_server_github_credentials(state.as_ref())?; + Ok(PullRequestGithubContext { record, creds }) +} + +struct RunPrInputs<'a> { + goal: &'a str, + base_branch: &'a str, + run_branch: &'a str, + diff: &'a str, + conclusion: &'a fabro_types::Conclusion, + normalized_origin: String, +} + +impl<'a> RunPrInputs<'a> { + fn extract(run_state: &'a fabro_store::RunProjection, force: bool) -> Result { + if let Some(record) = run_state.pull_request.as_ref() { + return Err(ApiError::with_code( + StatusCode::CONFLICT, + format!("Pull request already exists at {}", record.html_url), + "pull_request_exists", + )); + } + let run_spec = run_state.spec.as_ref().ok_or_else(|| { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "Run spec missing from store.", + ) + })?; + let origin_url = run_spec.repo_origin_url().ok_or_else(|| { + ApiError::with_code( + StatusCode::BAD_REQUEST, + "Run has no repo origin URL — pull request creation requires git metadata.", + "missing_repo_origin", + ) + })?; + let base_branch = run_spec.base_branch().ok_or_else(|| { + ApiError::with_code( + StatusCode::BAD_REQUEST, + "Run has no base branch — pull request creation requires git metadata.", + "missing_base_branch", + ) + })?; + let run_branch = run_state + .start + .as_ref() + .and_then(|start| start.run_branch.as_deref()) + .ok_or_else(|| { + ApiError::with_code( + StatusCode::BAD_REQUEST, + "Run has no run_branch — was it run with git push enabled?", + "missing_run_branch", + ) + })?; + let diff = run_state + .final_patch + .as_deref() + .filter(|d| !d.trim().is_empty()) + .ok_or_else(|| { + ApiError::with_code( + StatusCode::BAD_REQUEST, + "Stored diff is empty — nothing to create a PR for", + "empty_diff", + ) + })?; + let conclusion = run_state.conclusion.as_ref().ok_or_else(|| { + ApiError::with_code( + StatusCode::BAD_REQUEST, + "Run is not finished yet.", + "run_not_finished", + ) + })?; + if !force && !conclusion.status.is_successful() { + return Err(ApiError::with_code( + StatusCode::BAD_REQUEST, + format!( + "Run status is '{}', expected succeeded or partially_succeeded", + conclusion.status + ), + "run_not_successful", + )); + } + let normalized_origin = fabro_github::normalize_repo_origin_url(origin_url); + parse_github_owner_repo_from_url(&normalized_origin, "repo origin URL")?; + Ok(Self { + goal: run_spec.graph.goal(), + base_branch, + run_branch, + diff, + conclusion, + normalized_origin, + }) + } +} + +async fn create_run_pull_request( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, + Json(body): Json, +) -> Response { + let _create_guard = lock_pull_request_create(&state.pull_request_create_locks, &id).await; + let Ok(run_store) = state.store.open_run(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let run_state = match run_store.state().await { + Ok(run_state) => run_state, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let inputs = match RunPrInputs::extract(&run_state, body.force) { + Ok(inputs) => inputs, + Err(err) => return err.into_response(), + }; + let creds = match load_server_github_credentials(state.as_ref()) { + Ok(creds) => creds, + Err(err) => return err.into_response(), + }; + let github = match server_github_context(state.as_ref(), &creds) { + Ok(ctx) => ctx, + Err(err) => return err.into_response(), + }; + let model = if let Some(model) = body.model { + model + } else { + let configured = state.llm_source.configured_providers().await; + Catalog::builtin() + .default_for_configured(&configured) + .id + .clone() + }; + + let run_store_handle = run_store.clone().into(); + let request = pull_request::OpenPullRequestRequest { + github, + origin_url: &inputs.normalized_origin, + base_branch: inputs.base_branch, + head_branch: inputs.run_branch, + goal: inputs.goal, + diff: inputs.diff, + model: &model, + draft: true, + auto_merge: None, + run_store: &run_store_handle, + llm_source: state.llm_source.as_ref(), + conclusion: Some(inputs.conclusion), + run_state: Some(&run_state), + }; + let pull_request = match pull_request::maybe_open_pull_request(request).await { + Ok(Some(record)) => record, + Ok(None) => { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "Pull request creation returned no record unexpectedly.", + ) + .into_response(); + } + Err(err) => return ApiError::new(StatusCode::BAD_GATEWAY, err).into_response(), + }; + + let event = workflow_event::Event::pull_request_created(&pull_request, true); + if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + + Json(pull_request).into_response() +} + +async fn get_run_pull_request( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, +) -> Response { + let ctx = match load_pull_request_github_context(&state, &id).await { + Ok(ctx) => ctx, + Err(err) => return err.into_response(), + }; + let github = match server_github_context(state.as_ref(), &ctx.creds) { + Ok(github) => github, + Err(err) => return err.into_response(), + }; + + match fabro_github::get_pull_request( + &github, + &ctx.record.owner, + &ctx.record.repo, + ctx.record.number, + ) + .await + { + Ok(github) => Json(fabro_types::PullRequestDetail { + record: ctx.record, + github, + }) + .into_response(), + Err(fabro_github::PullRequestApiError::NotFound { .. }) => { + github_pull_request_not_found_error(&ctx.record).into_response() + } + Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), + } +} + +async fn merge_run_pull_request( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, + Json(body): Json, +) -> Response { + let ctx = match load_pull_request_github_context(&state, &id).await { + Ok(ctx) => ctx, + Err(err) => return err.into_response(), + }; + let github = match server_github_context(state.as_ref(), &ctx.creds) { + Ok(github) => github, + Err(err) => return err.into_response(), + }; + + match fabro_github::merge_pull_request( + &github, + &ctx.record.owner, + &ctx.record.repo, + ctx.record.number, + body.method, + ) + .await + { + Ok(()) => Json(MergeRunPullRequestResponse { + number: i64::try_from(ctx.record.number) + .expect("stored pull request number should fit in i64"), + html_url: ctx.record.html_url, + method: body.method, + }) + .into_response(), + Err(fabro_github::PullRequestApiError::NotFound { .. }) => { + github_pull_request_not_found_error(&ctx.record).into_response() + } + Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), + } +} + +async fn close_run_pull_request( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, +) -> Response { + let ctx = match load_pull_request_github_context(&state, &id).await { + Ok(ctx) => ctx, + Err(err) => return err.into_response(), + }; + let github = match server_github_context(state.as_ref(), &ctx.creds) { + Ok(github) => github, + Err(err) => return err.into_response(), + }; + + match fabro_github::close_pull_request( + &github, + &ctx.record.owner, + &ctx.record.repo, + ctx.record.number, + ) + .await + { + Ok(()) => Json(CloseRunPullRequestResponse { + number: i64::try_from(ctx.record.number) + .expect("stored pull request number should fit in i64"), + html_url: ctx.record.html_url, + }) + .into_response(), + Err(fabro_github::PullRequestApiError::NotFound { .. }) => { + github_pull_request_not_found_error(&ctx.record).into_response() + } + Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), + } +} diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs new file mode 100644 index 000000000..6979b21cc --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -0,0 +1,780 @@ +use super::super::*; + +pub(super) fn manifest_routes() -> Router> { + Router::new() + .route("/preflight", post(run_preflight)) + .route("/validate", post(validate_run_manifest)) +} + +pub(super) fn routes() -> Router> { + Router::new() + .route("/runs", get(list_runs).post(create_run)) + .route("/runs/resolve", get(resolve_run)) + .route("/boards/runs", get(list_board_runs)) + .route("/runs/{id}", get(get_run_status).delete(delete_run)) + .route("/runs/{id}/questions", get(get_questions)) + .route("/runs/{id}/questions/{qid}/answer", post(submit_answer)) + .route("/runs/{id}/state", get(get_run_state)) + .route("/runs/{id}/logs", get(get_run_logs)) + .route( + "/runs/{id}/stages/{stageId}/logs/{stream}", + get(get_run_stage_command_log), + ) + .route("/runs/{id}/settings", get(get_run_settings)) + .route("/runs/{id}/files", get(list_run_files)) + .merge(manifest_routes()) +} + +#[derive(serde::Deserialize)] +struct ListRunsParams { + #[serde(rename = "page[limit]", default = "default_page_limit")] + limit: u32, + #[serde(rename = "page[offset]", default)] + offset: u32, + #[serde(default)] + include_archived: bool, +} + +impl ListRunsParams { + fn pagination(&self) -> PaginationParams { + PaginationParams { + limit: self.limit, + offset: self.offset, + } + } +} + +fn board_column(status: RunStatus) -> Option<&'static str> { + match status { + RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"), + RunStatus::Running | RunStatus::Paused { .. } => Some("running"), + RunStatus::Blocked { .. } => Some("blocked"), + RunStatus::Succeeded { .. } => Some("succeeded"), + RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"), + RunStatus::Removing | RunStatus::Archived { .. } => None, + } +} + +pub(crate) fn board_columns() -> serde_json::Value { + serde_json::json!([ + {"id": "initializing", "name": "Initializing"}, + {"id": "running", "name": "Running"}, + {"id": "blocked", "name": "Blocked"}, + {"id": "succeeded", "name": "Succeeded"}, + {"id": "failed", "name": "Failed"}, + ]) +} + +async fn board_run_metadata( + state: &AppState, + run_id: RunId, +) -> serde_json::Map { + let mut metadata = serde_json::Map::new(); + let Ok(run_store) = state.store.open_run_reader(&run_id).await else { + return metadata; + }; + let Ok(run_state) = run_store.state().await else { + return metadata; + }; + + if let Some(pull_request) = run_state.pull_request { + metadata.insert( + "pull_request".to_string(), + serde_json::json!({ + "number": pull_request.number, + }), + ); + } + + if let Some(sandbox) = run_state.sandbox { + let mut sandbox_metadata = serde_json::Map::new(); + sandbox_metadata.insert( + "working_directory".to_string(), + serde_json::json!(sandbox.working_directory), + ); + if let Some(identifier) = sandbox.identifier { + sandbox_metadata.insert("id".to_string(), serde_json::json!(identifier)); + } + metadata.insert( + "sandbox".to_string(), + serde_json::Value::Object(sandbox_metadata), + ); + } + + if let Some((_, record)) = + run_state + .pending_interviews + .iter() + .min_by(|(left_id, left), (right_id, right)| { + left.started_at + .cmp(&right.started_at) + .then_with(|| left_id.cmp(right_id)) + }) + { + metadata.insert( + "question".to_string(), + serde_json::json!({ + "text": record.question.text, + }), + ); + } + + metadata +} + +fn paginate_items(items: Vec, pagination: &PaginationParams) -> (Vec, bool) { + let limit = pagination.limit.clamp(1, 100) as usize; + let offset = pagination.offset.min(MAX_PAGE_OFFSET) as usize; + let mut data: Vec<_> = items.into_iter().skip(offset).take(limit + 1).collect(); + let has_more = data.len() > limit; + data.truncate(limit); + (data, has_more) +} + +async fn list_board_runs( + _auth: RequiredUser, + State(state): State>, + Query(pagination): Query, +) -> Response { + let summaries = match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(runs) => runs, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let board_summaries: Vec<_> = summaries + .into_iter() + .filter_map(|summary| { + let column = board_column(summary.status)?; + Some((summary, column)) + }) + .collect(); + let (page_summaries, has_more) = paginate_items(board_summaries, &pagination); + + let mut data = Vec::with_capacity(page_summaries.len()); + for (summary, column) in page_summaries { + let run_id = summary.run_id; + let mut item = + serde_json::to_value(&summary).expect("RunSummary serialization is infallible"); + item["column"] = serde_json::json!(column); + if let Some(object) = item.as_object_mut() { + object.extend(board_run_metadata(state.as_ref(), run_id).await); + } + data.push(item); + } + ( + StatusCode::OK, + Json(serde_json::json!({ + "columns": board_columns(), + "data": data, + "meta": { "has_more": has_more } + })), + ) + .into_response() +} + +async fn list_runs( + _auth: RequiredUser, + State(state): State>, + Query(params): Query, +) -> Response { + match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(runs) => { + let include_archived = params.include_archived; + let items = runs + .into_iter() + .filter(|summary| { + include_archived || !matches!(summary.status, RunStatus::Archived { .. }) + }) + .collect::>(); + let (data, has_more) = paginate_items(items, ¶ms.pagination()); + ( + StatusCode::OK, + Json(serde_json::json!({ + "data": data, + "meta": { "has_more": has_more } + })), + ) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +#[derive(Debug, serde::Deserialize)] +struct ResolveRunQuery { + selector: String, +} + +#[derive(Debug, Default, serde::Deserialize)] +struct DeleteRunQuery { + #[serde(default)] + force: bool, +} + +fn default_command_log_limit() -> u64 { + 65_536 +} + +#[derive(Debug, serde::Deserialize)] +struct CommandLogQuery { + #[serde(default)] + offset: u64, + #[serde(default = "default_command_log_limit")] + limit: u64, +} + +#[derive(Debug, serde::Serialize)] +struct CommandLogResponseBody { + stream: CommandOutputStream, + offset: u64, + next_offset: u64, + total_bytes: u64, + bytes_base64: String, + eof: bool, + cas_ref: Option, + live_streaming: bool, +} + +async fn resolve_run( + _auth: RequiredUser, + State(state): State>, + Query(query): Query, +) -> Response { + let runs = match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(runs) => runs, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + match resolve_run_by_selector( + &runs, + &query.selector, + |run| run.run_id.to_string(), + |run| run.workflow_slug.clone(), + |run| run.workflow_name.clone(), + |run| run.run_id.created_at(), + |run| run.run_id.created_at().to_rfc3339(), + |run| run.repo_origin_url.clone(), + ) { + Ok(run) => (StatusCode::OK, Json(run.clone())).into_response(), + Err(err @ (ResolveRunError::InvalidSelector | ResolveRunError::AmbiguousPrefix { .. })) => { + ApiError::bad_request(err.to_string()).into_response() + } + Err(err @ ResolveRunError::NotFound { .. }) => { + ApiError::not_found(err.to_string()).into_response() + } + } +} + +async fn delete_run( + _auth: RequiredUser, + State(state): State>, + Query(query): Query, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + + match delete_run_internal(&state, id, query.force).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(response) => response, + } +} + +async fn create_run( + RequestAuth(auth_slot): RequestAuth, + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let subject = match require_user(&auth_slot) { + Ok(subject) => subject, + Err(err) => return err.into_response(), + }; + let req = match serde_json::from_slice::(&body) { + Ok(req) => req, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let manifest_run_defaults = state.manifest_run_defaults(); + let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { + Ok(prepared) => prepared, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let run_id = prepared.run_id.unwrap_or_else(RunId::new); + info!(run_id = %run_id, "Run created"); + + let configured_providers = state.llm_source.configured_providers().await; + let mut create_input = run_manifest::create_run_input(prepared.clone(), configured_providers); + create_input.run_id = Some(run_id); + create_input.provenance = Some(run_provenance(&headers, &subject)); + create_input.submitted_manifest_bytes = Some(body.to_vec()); + + let storage_root = match resolve_interp_string(&state.server_settings().server.storage.root) { + Ok(path) => PathBuf::from(path), + Err(err) => { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to resolve server storage root: {err}"), + ) + .into_response(); + } + }; + let created = match Box::pin(operations::create( + state.store.as_ref(), + create_input, + storage_root, + )) + .await + { + Ok(created) => created, + Err(WorkflowError::ValidationFailed { .. } | WorkflowError::Parse(_)) => { + return ApiError::bad_request("Validation failed").into_response(); + } + Err(err) => { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to persist run state: {err}"), + ) + .into_response(); + } + }; + let created_at = created.run_id.created_at(); + + { + let mut runs = state.runs.lock().expect("runs lock poisoned"); + runs.insert( + created.run_id, + managed_run( + created.persisted.source().to_string(), + RunStatus::Submitted, + created_at, + created.run_dir, + RunExecutionMode::Start, + ), + ); + } + + ( + StatusCode::CREATED, + Json(RunStatusResponse { + id: run_id.to_string(), + status: RunStatus::Submitted, + error: None, + queue_position: None, + pending_control: None, + created_at, + }), + ) + .into_response() +} + +fn run_provenance(headers: &HeaderMap, subject: &UserPrincipal) -> RunProvenance { + RunProvenance { + server: Some(RunServerProvenance { + version: FABRO_VERSION.to_string(), + }), + client: run_client_provenance(headers), + subject: Some(Principal::User(subject.clone())), + } +} + +fn run_client_provenance(headers: &HeaderMap) -> Option { + let user_agent = headers + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(str::to_string)?; + let (name, version) = parse_known_fabro_user_agent(&user_agent) + .map_or((None, None), |(name, version)| { + (Some(name.to_string()), Some(version.to_string())) + }); + Some(RunClientProvenance { + user_agent: Some(user_agent), + name, + version, + }) +} + +fn parse_known_fabro_user_agent(user_agent: &str) -> Option<(&str, &str)> { + let token = user_agent.split_whitespace().next()?; + let (name, version) = token.split_once('/')?; + if version.is_empty() { + return None; + } + match name { + "fabro-cli" | "fabro-web" => Some((name, version)), + _ => None, + } +} + +async fn run_preflight( + _auth: RequiredUser, + State(state): State>, + Json(req): Json, +) -> Response { + let manifest_run_defaults = state.manifest_run_defaults(); + let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { + Ok(prepared) => prepared, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let validated = match run_manifest::validate_prepared_manifest(&prepared) { + Ok(validated) => validated, + Err(WorkflowError::Parse(_)) => { + return ApiError::bad_request("Validation failed").into_response(); + } + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let response = match run_manifest::run_preflight(&state, &prepared, &validated).await { + Ok((response, _ok)) => response, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + (StatusCode::OK, Json(response)).into_response() +} + +async fn validate_run_manifest( + _auth: RequiredUser, + State(state): State>, + Json(req): Json, +) -> Response { + let manifest_run_defaults = state.manifest_run_defaults(); + let prepared = match run_manifest::prepare_manifest(manifest_run_defaults.as_ref(), &req) { + Ok(prepared) => prepared, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + let validated = match run_manifest::validate_prepared_manifest(&prepared) { + Ok(validated) => validated, + Err(WorkflowError::Parse(_)) => { + return ApiError::bad_request("Validation failed").into_response(); + } + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; + ( + StatusCode::OK, + Json(run_manifest::validate_response(&prepared, &validated)), + ) + .into_response() +} + +async fn get_run_status( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(runs) => match runs.into_iter().find(|run| run.run_id == id) { + Some(run) => (StatusCode::OK, Json(run)).into_response(), + None => ApiError::not_found("Run not found.").into_response(), + }, + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +async fn get_run_settings( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let run_store = match state.store.open_run_reader(&id).await { + Ok(store) => store, + Err(fabro_store::Error::RunNotFound(_)) => { + return ApiError::not_found("Run not found.").into_response(); + } + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let run_state = match run_store.state().await { + Ok(state) => state, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let Some(run_spec) = run_state.spec else { + return ApiError::not_found("Run not found.").into_response(); + }; + (StatusCode::OK, Json(run_spec.settings)).into_response() +} + +async fn get_questions( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => { + let questions = run_state + .pending_interviews + .values() + .map(api_question_from_pending_interview) + .collect::>(); + (StatusCode::OK, Json(ListResponse::new(questions))).into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(fabro_store::Error::RunNotFound(_)) => { + ApiError::not_found("Run not found.").into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +async fn submit_answer( + auth: RequiredUser, + State(state): State>, + Path((id, qid)): Path<(String, String)>, + Json(req): Json, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let pending = match load_pending_interview(state.as_ref(), id, &qid).await { + Ok(pending) => pending, + Err(response) => return response, + }; + let answer = match answer_from_request(req, &pending.question) { + Ok(answer) => answer, + Err(response) => return response, + }; + let submission = AnswerSubmission::new(answer, Principal::User(auth.0)); + match submit_pending_interview_answer(state.as_ref(), &pending, submission).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(response) => response, + } +} + +async fn get_run_state( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, +) -> Response { + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => Json(run_state).into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + +async fn get_run_logs( + RequireRunScoped(id): RequireRunScoped, + State(state): State>, +) -> Response { + if state.store.open_run_reader(&id).await.is_err() { + return ApiError::not_found("Run not found.").into_response(); + } + + let path = Storage::new(state.server_storage_dir()) + .run_scratch(&id) + .runtime_dir() + .join("server.log"); + match fs::read(&path).await { + Ok(bytes) => ([(header::CONTENT_TYPE, "text/plain; charset=utf-8")], bytes).into_response(), + Err(err) if err.kind() == ErrorKind::NotFound => { + ApiError::not_found("Run log not available.").into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +async fn get_run_stage_command_log( + RequireCommandLog(id, stage_id, stream): RequireCommandLog, + State(state): State>, + Query(query): Query, +) -> Response { + const MAX_COMMAND_LOG_LIMIT: u64 = 1_048_576; + + if query.limit == 0 { + return ApiError::bad_request("limit must be greater than 0").into_response(); + } + let limit = query.limit.min(MAX_COMMAND_LOG_LIMIT); + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let run_state = match run_store.state().await { + Ok(run_state) => run_state, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + let Some(node) = run_state.stage(&stage_id) else { + return ApiError::not_found("Stage not found.").into_response(); + }; + + let stream_value = match stream { + CommandOutputStream::Stdout => node.stdout.as_deref(), + CommandOutputStream::Stderr => node.stderr.as_deref(), + }; + let cas_ref = stream_value + .filter(|value| parse_blob_ref(value).is_some()) + .map(str::to_string); + let live_streaming = node + .live_streaming + .unwrap_or_else(|| cas_ref.is_none() && node.completion.is_none()); + let run_dir = Storage::new(state.server_storage_dir()) + .run_scratch(&id) + .root() + .to_path_buf(); + let scratch_path = command_log_path(&run_dir, &stage_id, stream); + + match read_log_slice(&scratch_path, query.offset, limit).await { + Ok((bytes, total_bytes)) => { + return build_command_log_response( + stream, + query.offset, + limit, + LogSource::Sliced { bytes, total_bytes }, + cas_ref.is_some(), + cas_ref, + live_streaming, + ); + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + } + + if let Some(cas_ref) = cas_ref { + let text = match read_json_string_blob(&run_store.clone().into(), &cas_ref).await { + Ok(Some(text)) => text, + Ok(None) => String::new(), + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + return build_command_log_response( + stream, + query.offset, + limit, + LogSource::Full(text.as_bytes()), + true, + Some(cas_ref), + live_streaming, + ); + } + + if let Some(inline_text) = stream_value { + return build_command_log_response( + stream, + query.offset, + limit, + LogSource::Full(inline_text.as_bytes()), + true, + None, + live_streaming, + ); + } + + build_command_log_response( + stream, + query.offset, + limit, + LogSource::Full(&[]), + node.completion.is_some(), + None, + live_streaming, + ) +} + +enum LogSource<'a> { + Sliced { + bytes: Vec, + total_bytes: u64, + }, + Full(&'a [u8]), +} + +fn build_command_log_response( + stream: CommandOutputStream, + requested_offset: u64, + limit: u64, + source: LogSource<'_>, + eof: bool, + cas_ref: Option, + live_streaming: bool, +) -> Response { + let (body_bytes, total_bytes, offset) = match source { + LogSource::Sliced { bytes, total_bytes } => { + let offset = requested_offset.min(total_bytes); + (bytes, total_bytes, offset) + } + LogSource::Full(bytes) => { + let total_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + let offset = requested_offset.min(total_bytes); + let start = usize::try_from(offset).unwrap_or(bytes.len()); + let end = start + .saturating_add(usize::try_from(limit).unwrap_or(usize::MAX)) + .min(bytes.len()); + (bytes[start..end].to_vec(), total_bytes, offset) + } + }; + Json(CommandLogResponseBody { + stream, + offset, + next_offset: offset + u64::try_from(body_bytes.len()).unwrap_or(u64::MAX), + total_bytes, + bytes_base64: BASE64_STANDARD.encode(body_bytes), + eof, + cas_ref, + live_streaming, + }) + .into_response() +} diff --git a/lib/crates/fabro-server/src/server/handler/sandbox.rs b/lib/crates/fabro-server/src/server/handler/sandbox.rs new file mode 100644 index 000000000..2761139d6 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/sandbox.rs @@ -0,0 +1,274 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, Bytes, DaytonaSandbox, EnvVars, IntoResponse, Json, NamedTempFile, Path, + PreviewUrlRequest, PreviewUrlResponse, Query, RequiredUser, Response, Router, RunId, Sandbox, + SandboxFileEntry, SandboxFileListResponse, SandboxProvider, SshAccessRequest, + SshAccessResponse, State, StatusCode, collect_causes, fs, get, octet_stream_response, + parse_run_id_path, post, reconnect, reject_if_archived, render_with_causes, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/runs/{id}/preview", post(generate_preview_url)) + .route("/runs/{id}/ssh", post(create_ssh_access)) + .route("/runs/{id}/sandbox/files", get(list_sandbox_files)) + .route( + "/runs/{id}/sandbox/file", + get(get_sandbox_file).put(put_sandbox_file), + ) +} + +#[derive(serde::Deserialize)] +struct SandboxFilesParams { + path: String, + #[serde(default)] + depth: Option, +} + +#[derive(serde::Deserialize)] +struct SandboxFileParams { + path: String, +} + +async fn generate_preview_url( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Json(request): Json, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let Ok(port) = u16::try_from(request.port) else { + return ApiError::bad_request("Port must fit in a u16.").into_response(); + }; + let Ok(expires_in_secs) = i32::try_from(request.expires_in_secs.get()) else { + return ApiError::bad_request("Preview expiry exceeds supported range.").into_response(); + }; + + let sandbox = match reconnect_daytona_sandbox(&state, &id).await { + Ok(sandbox) => sandbox, + Err(response) => return response, + }; + + let response = if request.signed { + match sandbox + .get_signed_preview_url(port, Some(expires_in_secs)) + .await + { + Ok(preview) => PreviewUrlResponse { + token: None, + url: preview.url, + }, + Err(err) => { + return ApiError::new(StatusCode::CONFLICT, err.display_with_causes()) + .into_response(); + } + } + } else { + match sandbox.get_preview_link(port).await { + Ok(preview) => PreviewUrlResponse { + token: Some(preview.token), + url: preview.url, + }, + Err(err) => { + return ApiError::new(StatusCode::CONFLICT, err.display_with_causes()) + .into_response(); + } + } + }; + + (StatusCode::CREATED, Json(response)).into_response() +} + +async fn create_ssh_access( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Json(request): Json, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let sandbox = match reconnect_daytona_sandbox(&state, &id).await { + Ok(sandbox) => sandbox, + Err(response) => return response, + }; + match sandbox.create_ssh_access(Some(request.ttl_minutes)).await { + Ok(command) => (StatusCode::CREATED, Json(SshAccessResponse { command })).into_response(), + Err(err) => ApiError::new(StatusCode::CONFLICT, err.display_with_causes()).into_response(), + } +} + +async fn list_sandbox_files( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let sandbox = match reconnect_run_sandbox(&state, &id).await { + Ok(sandbox) => sandbox, + Err(response) => return response, + }; + match sandbox.list_directory(¶ms.path, params.depth).await { + Ok(entries) => Json(SandboxFileListResponse { + data: entries + .into_iter() + .map(|entry| SandboxFileEntry { + is_dir: entry.is_dir, + name: entry.name, + size: entry.size.map(u64::cast_signed), + }) + .collect(), + }) + .into_response(), + Err(err) => ApiError::new(StatusCode::NOT_FOUND, err.display_with_causes()).into_response(), + } +} + +async fn get_sandbox_file( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + let sandbox = match reconnect_run_sandbox(&state, &id).await { + Ok(sandbox) => sandbox, + Err(response) => return response, + }; + let temp = match NamedTempFile::new() { + Ok(temp) => temp, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + if let Err(err) = sandbox + .download_file_to_local(¶ms.path, temp.path()) + .await + { + return ApiError::new(StatusCode::NOT_FOUND, err.display_with_causes()).into_response(); + } + match fs::read(temp.path()).await { + Ok(bytes) => octet_stream_response(bytes.into()), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } +} + +async fn put_sandbox_file( + _auth: RequiredUser, + State(state): State>, + Path(id): Path, + Query(params): Query, + body: Bytes, +) -> Response { + let id = match parse_run_id_path(&id) { + Ok(id) => id, + Err(response) => return response, + }; + if let Some(response) = reject_if_archived(state.as_ref(), &id).await { + return response; + } + let sandbox = match reconnect_run_sandbox(&state, &id).await { + Ok(sandbox) => sandbox, + Err(response) => return response, + }; + let temp = match NamedTempFile::new() { + Ok(temp) => temp, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + if let Err(err) = fs::write(temp.path(), &body).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + match sandbox + .upload_file_from_local(temp.path(), ¶ms.path) + .await + { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(err) => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.display_with_causes()) + .into_response(), + } +} + +async fn reconnect_run_sandbox( + state: &Arc, + run_id: &RunId, +) -> Result, Response> { + let record = load_run_sandbox_record(state, run_id).await?; + let daytona_api_key = state.vault_or_env(EnvVars::DAYTONA_API_KEY); + reconnect(&record, daytona_api_key).await.map_err(|err| { + let detail = render_with_causes(&err.to_string(), &collect_causes(err.as_ref())); + ApiError::new(StatusCode::CONFLICT, detail).into_response() + }) +} + +async fn reconnect_daytona_sandbox( + state: &Arc, + run_id: &RunId, +) -> Result { + let record = load_run_sandbox_record(state, run_id).await?; + if record.provider != SandboxProvider::Daytona.to_string() { + return Err(ApiError::new( + StatusCode::CONFLICT, + "Sandbox provider does not support this capability.", + ) + .into_response()); + } + let Some(name) = record.identifier.as_deref() else { + return Err(ApiError::new( + StatusCode::CONFLICT, + "Sandbox record is missing the Daytona identifier.", + ) + .into_response()); + }; + let Some(repo_cloned) = record.repo_cloned else { + return Err(ApiError::new( + StatusCode::CONFLICT, + "Sandbox record is missing clone metadata.", + ) + .into_response()); + }; + let daytona_api_key = state.vault_or_env(EnvVars::DAYTONA_API_KEY); + DaytonaSandbox::reconnect( + name, + daytona_api_key, + repo_cloned, + record.clone_origin_url.clone(), + record.clone_branch.clone(), + ) + .await + .map_err(|err| ApiError::new(StatusCode::CONFLICT, err.display_with_causes()).into_response()) +} + +async fn load_run_sandbox_record( + state: &Arc, + run_id: &RunId, +) -> Result { + match state.store.open_run_reader(run_id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => run_state.sandbox.ok_or_else(|| { + ApiError::new(StatusCode::CONFLICT, "Run has no active sandbox.").into_response() + }), + Err(err) => Err( + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + ), + }, + Err(_) => Err(ApiError::not_found("Run not found.").into_response()), + } +} diff --git a/lib/crates/fabro-server/src/server/handler/secrets.rs b/lib/crates/fabro-server/src/server/handler/secrets.rs new file mode 100644 index 000000000..cbbeecc29 --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/secrets.rs @@ -0,0 +1,102 @@ +use std::sync::Arc; + +use super::super::{ + ApiError, AppState, CreateSecretRequest, DeleteSecretRequest, IntoResponse, Json, RequiredUser, + Response, Router, SecretType, State, StatusCode, VaultError, get, parse_credential_secret, + spawn_blocking, +}; + +pub(super) fn routes() -> Router> { + Router::new().route( + "/secrets", + get(list_secrets) + .post(create_secret) + .delete(delete_secret_by_name), + ) +} + +async fn list_secrets(_auth: RequiredUser, State(state): State>) -> Response { + let data = state.vault.read().await.list(); + (StatusCode::OK, Json(serde_json::json!({ "data": data }))).into_response() +} + +async fn create_secret( + _auth: RequiredUser, + State(state): State>, + Json(body): Json, +) -> Response { + let secret_type = body.type_; + let name = body.name; + let value = body.value; + let description = body.description; + if secret_type == SecretType::Credential { + if let Err(err) = parse_credential_secret(&name, &value) { + return ApiError::bad_request(err).into_response(); + } + } + let state_for_write = Arc::clone(&state); + let result = spawn_blocking(move || { + let mut vault = state_for_write.vault.blocking_write(); + vault.set(&name, &value, secret_type, description.as_deref()) + }) + .await; + + match result { + Ok(Ok(meta)) => (StatusCode::OK, Json(meta)).into_response(), + Ok(Err(VaultError::InvalidName(_))) => { + ApiError::bad_request("invalid secret name").into_response() + } + Ok(Err(VaultError::Io(err))) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + Ok(Err(VaultError::Serde(err))) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + Ok(Err(VaultError::NotFound(_))) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "secret unexpectedly missing", + ) + .into_response(), + Err(err) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("secret write task failed: {err}"), + ) + .into_response(), + } +} + +async fn delete_secret_by_name( + _auth: RequiredUser, + State(state): State>, + Json(body): Json, +) -> Response { + let name = body.name; + let state_for_write = Arc::clone(&state); + let result = spawn_blocking(move || { + let mut vault = state_for_write.vault.blocking_write(); + vault.remove(&name) + }) + .await; + + match result { + Ok(Ok(())) => StatusCode::NO_CONTENT.into_response(), + Ok(Err(VaultError::InvalidName(_))) => { + ApiError::bad_request("invalid secret name").into_response() + } + Ok(Err(VaultError::NotFound(name))) => { + ApiError::new(StatusCode::NOT_FOUND, format!("secret not found: {name}")) + .into_response() + } + Ok(Err(VaultError::Io(err))) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + Ok(Err(VaultError::Serde(err))) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + Err(err) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("secret delete task failed: {err}"), + ) + .into_response(), + } +} diff --git a/lib/crates/fabro-server/src/server/handler/system.rs b/lib/crates/fabro-server/src/server/handler/system.rs new file mode 100644 index 000000000..63aa8546a --- /dev/null +++ b/lib/crates/fabro-server/src/server/handler/system.rs @@ -0,0 +1,506 @@ +use std::sync::Arc; + +use super::super::{ + AggregateBilling, AggregateBillingTotals, ApiError, AppState, BilledTokenCounts, + BillingByModel, DfParams, FABRO_VERSION, GithubIntegrationStrategy, IntoResponse, Json, + ModelReference, Path, PruneRunsRequest, PruneRunsResponse, Query, RequiredUser, Response, + Router, RunStatus, State, StatusCode, SystemInfoResponse, SystemRunCounts, + build_disk_usage_response, build_prune_plan, delete_run_internal, diagnostics, get, post, + resolve_interp_string, spawn_blocking, system_features, system_sandbox_provider, to_i64, +}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/repos/github/{owner}/{name}", get(get_github_repo)) + .route("/health/diagnostics", post(run_diagnostics)) + .route("/settings", get(get_server_settings)) + .route("/system/info", get(get_system_info)) + .route("/system/df", get(get_system_df)) + .route("/system/prune/runs", post(prune_runs)) + .route("/billing", get(get_aggregate_billing)) +} + +pub(in crate::server) async fn health() -> Response { + Json(serde_json::json!({ + "status": "ok", + })) + .into_response() +} + +async fn get_server_settings(_auth: RequiredUser, State(state): State>) -> Response { + ( + StatusCode::OK, + Json(state.server_settings().as_ref().clone()), + ) + .into_response() +} + +async fn get_system_info(_auth: RequiredUser, State(state): State>) -> Response { + let manifest_run_settings = state.manifest_run_settings(); + let server_settings = state.server_settings(); + let (total_runs, active_runs) = { + let runs = state.runs.lock().expect("runs lock poisoned"); + let active = runs + .values() + .filter(|run| { + matches!( + run.status, + RunStatus::Queued + | RunStatus::Starting + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } + ) + }) + .count(); + (runs.len(), active) + }; + + let response = SystemInfoResponse { + version: Some(FABRO_VERSION.to_string()), + server_url: Some(server_settings.server.web.url.as_source()), + git_sha: option_env!("FABRO_GIT_SHA").map(str::to_string), + build_date: option_env!("FABRO_BUILD_DATE").map(str::to_string), + profile: option_env!("FABRO_BUILD_PROFILE").map(str::to_string), + os: Some(std::env::consts::OS.to_string()), + arch: Some(std::env::consts::ARCH.to_string()), + storage_engine: Some("slatedb".to_string()), + storage_dir: Some(state.server_storage_dir().display().to_string()), + uptime_secs: Some(to_i64(state.started_at.elapsed().as_secs())), + runs: Some(SystemRunCounts { + total: Some(to_i64(total_runs)), + active: Some(to_i64(active_runs)), + }), + sandbox_provider: Some(system_sandbox_provider(&manifest_run_settings)), + features: Some(system_features( + server_settings.as_ref(), + &manifest_run_settings, + )), + }; + (StatusCode::OK, Json(response)).into_response() +} + +async fn get_system_df( + _auth: RequiredUser, + State(state): State>, + Query(params): Query, +) -> Response { + let storage_dir = state.server_storage_dir(); + let summaries = match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(summaries) => summaries, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + let response = match spawn_blocking(move || { + build_disk_usage_response(&summaries, &storage_dir, params.verbose) + }) + .await + { + Ok(Ok(response)) => response, + Ok(Err(err)) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + (StatusCode::OK, Json(response)).into_response() +} + +async fn prune_runs( + _auth: RequiredUser, + State(state): State>, + Json(body): Json, +) -> Response { + let storage_dir = state.server_storage_dir(); + let summaries = match state + .store + .list_runs(&fabro_store::ListRunsQuery::default()) + .await + { + Ok(summaries) => summaries, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + let dry_run = body.dry_run; + let body_for_plan = body.clone(); + let prune_plan = + match spawn_blocking(move || build_prune_plan(&body_for_plan, &summaries, &storage_dir)) + .await + { + Ok(Ok(plan)) => plan, + Ok(Err(err)) => { + return ApiError::new(StatusCode::BAD_REQUEST, err.to_string()).into_response(); + } + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; + + if dry_run { + return ( + StatusCode::OK, + Json(PruneRunsResponse { + dry_run: Some(true), + runs: Some(prune_plan.rows), + total_count: Some(to_i64(prune_plan.run_ids.len())), + total_size_bytes: Some(to_i64(prune_plan.total_size_bytes)), + deleted_count: Some(0), + freed_bytes: Some(0), + }), + ) + .into_response(); + } + + for run_id in &prune_plan.run_ids { + if let Err(response) = delete_run_internal(&state, *run_id, true).await { + return response; + } + } + + ( + StatusCode::OK, + Json(PruneRunsResponse { + dry_run: Some(false), + runs: None, + total_count: Some(to_i64(prune_plan.run_ids.len())), + total_size_bytes: Some(to_i64(prune_plan.total_size_bytes)), + deleted_count: Some(to_i64(prune_plan.run_ids.len())), + freed_bytes: Some(to_i64(prune_plan.total_size_bytes)), + }), + ) + .into_response() +} + +#[derive(serde::Deserialize)] +struct GitHubRepoResponse { + default_branch: String, + private: bool, + permissions: Option, +} + +/// Reject owner/repo path segments that could rewrite the GitHub API endpoint +/// via `..` traversal after URL normalization. Conservative compared to +/// GitHub's real rules, which is fine for server-side input validation. +#[allow( + clippy::result_large_err, + reason = "GitHub slug validation returns HTTP 400 responses directly." +)] +pub(in crate::server) fn validate_github_slug( + kind: &str, + value: &str, + max_len: usize, +) -> Result<(), Response> { + if value.is_empty() || value.len() > max_len || matches!(value, "." | "..") { + return Err(ApiError::bad_request(format!("invalid github {kind}")).into_response()); + } + if !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + { + return Err(ApiError::bad_request(format!("invalid github {kind}")).into_response()); + } + Ok(()) +} + +async fn get_github_repo( + _auth: RequiredUser, + State(state): State>, + Path((owner, name)): Path<(String, String)>, +) -> Response { + if let Err(response) = validate_github_slug("owner", &owner, 39) { + return response; + } + if let Err(response) = validate_github_slug("repo", &name, 100) { + return response; + } + let settings = state.server_settings(); + let github_settings = &settings.server.integrations.github; + let base_url = fabro_github::github_api_base_url(); + let mut client: Option = None; + let token = match github_settings.strategy { + GithubIntegrationStrategy::App => { + let Some(app_id) = github_settings.app_id.as_ref() else { + return ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "server.integrations.github.app_id is not configured", + ) + .into_response(); + }; + if let Err(err) = resolve_interp_string(app_id) { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) + .into_response(); + } + let creds = match state.github_credentials(github_settings) { + Ok(Some(fabro_github::GitHubCredentials::App(creds))) => creds, + Ok(Some(_)) => unreachable!("app strategy should not return token credentials"), + Ok(None) => { + return ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "GITHUB_APP_PRIVATE_KEY is not configured", + ) + .into_response(); + } + Err(err) => { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err).into_response(); + } + }; + + let jwt = match fabro_github::sign_app_jwt(&creds.app_id, &creds.private_key_pem) { + Ok(jwt) => jwt, + Err(err) => { + tracing::error!(error = ?err, "failed to sign GitHub App JWT"); + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) + .into_response(); + } + }; + let install_url = match github_settings.slug.as_ref() { + Some(slug) => match resolve_interp_string(slug) { + Ok(slug) => format!("https://github.com/apps/{slug}/installations/new"), + Err(err) => { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) + .into_response(); + } + }, + None => format!("https://github.com/organizations/{owner}/settings/installations"), + }; + + if client.is_none() { + client = Some(match state.http_client() { + Ok(http) => http, + Err(err) => { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) + .into_response(); + } + }); + } + let client_ref = client.as_ref().expect("client initialized above"); + let installed = + match fabro_github::check_app_installed(client_ref, &jwt, &owner, &name, &base_url) + .await + { + Ok(installed) => installed, + Err(err) => { + tracing::error!(error = ?err, "failed to check GitHub App installation"); + return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()) + .into_response(); + } + }; + + if !installed { + return ( + StatusCode::OK, + Json(serde_json::json!({ + "owner": owner, + "name": name, + "accessible": false, + "default_branch": null, + "private": null, + "permissions": null, + "install_url": install_url, + })), + ) + .into_response(); + } + + match fabro_github::create_installation_access_token_with_permissions_and_install_url( + client_ref, + &jwt, + &owner, + &name, + &base_url, + serde_json::json!({ "contents": "write", "pull_requests": "write" }), + Some(&install_url), + ) + .await + { + Ok(token) => token, + Err(err) => { + tracing::error!( + error = ?err, + "failed to create GitHub App installation token" + ); + return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(); + } + } + } + GithubIntegrationStrategy::Token => match state.github_credentials(github_settings) { + Ok(Some(fabro_github::GitHubCredentials::Token(token))) => token, + Ok(Some(_)) => unreachable!("token strategy should not return app credentials"), + Ok(None) => { + return ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "GITHUB_TOKEN is not configured", + ) + .into_response(); + } + Err(err) => { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err).into_response(); + } + }, + }; + + let client = match client { + Some(client) => client, + None => match state.http_client() { + Ok(http) => http, + Err(err) => { + return ApiError::new(StatusCode::SERVICE_UNAVAILABLE, err.to_string()) + .into_response(); + } + }, + }; + let repo_response = match client + .get(format!("{base_url}/repos/{owner}/{name}")) + .header("Authorization", format!("Bearer {token}")) + .header("Accept", "application/vnd.github+json") + .header("User-Agent", "fabro-server") + .send() + .await + { + Ok(response) if response.status().is_success() => response, + Ok(response) + if github_settings.strategy == GithubIntegrationStrategy::Token + && matches!( + response.status(), + fabro_http::StatusCode::FORBIDDEN | fabro_http::StatusCode::NOT_FOUND + ) => + { + return ( + StatusCode::OK, + Json(serde_json::json!({ + "owner": owner, + "name": name, + "accessible": false, + "default_branch": null, + "private": null, + "permissions": null, + "install_url": serde_json::Value::Null, + })), + ) + .into_response(); + } + Ok(response) + if github_settings.strategy == GithubIntegrationStrategy::Token + && response.status() == fabro_http::StatusCode::UNAUTHORIZED => + { + return ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "Stored GitHub token is invalid — run fabro install or update GITHUB_TOKEN", + ) + .into_response(); + } + Ok(response) => { + return ApiError::new( + StatusCode::BAD_GATEWAY, + format!("GitHub repo lookup failed: {}", response.status()), + ) + .into_response(); + } + Err(err) => return ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), + }; + + let repo = match repo_response.json::().await { + Ok(repo) => repo, + Err(err) => { + return ApiError::new( + StatusCode::BAD_GATEWAY, + format!("Failed to parse GitHub repo response: {err}"), + ) + .into_response(); + } + }; + + ( + StatusCode::OK, + Json(serde_json::json!({ + "owner": owner, + "name": name, + "accessible": true, + "default_branch": repo.default_branch, + "private": repo.private, + "permissions": repo.permissions, + "install_url": serde_json::Value::Null, + })), + ) + .into_response() +} + +async fn run_diagnostics(_auth: RequiredUser, State(state): State>) -> Response { + ( + StatusCode::OK, + Json(diagnostics::run_all(state.as_ref()).await), + ) + .into_response() +} + +pub(in crate::server) async fn openapi_spec() -> Response { + let yaml = include_str!("../../../../../../docs/public/api-reference/fabro-api.yaml"); + let value: serde_json::Value = + serde_yaml::from_str(yaml).expect("embedded OpenAPI YAML is invalid"); + Json(value).into_response() +} + +async fn get_aggregate_billing( + _auth: RequiredUser, + State(state): State>, +) -> Response { + let agg = state + .aggregate_billing + .lock() + .expect("aggregate_billing lock poisoned"); + let by_model: Vec = agg + .by_model + .iter() + .map(|(model, totals)| BillingByModel { + billing: totals.billing.clone(), + model: ModelReference { id: model.clone() }, + stages: totals.stages, + }) + .collect(); + let total_billing = + agg.by_model + .values() + .fold(BilledTokenCounts::default(), |mut acc, totals| { + let billing = &totals.billing; + acc.input_tokens += billing.input_tokens; + acc.output_tokens += billing.output_tokens; + acc.reasoning_tokens += billing.reasoning_tokens; + acc.cache_read_tokens += billing.cache_read_tokens; + acc.cache_write_tokens += billing.cache_write_tokens; + acc.total_tokens += billing.total_tokens; + if let Some(value) = billing.total_usd_micros { + *acc.total_usd_micros.get_or_insert(0) += value; + } + acc + }); + let response = AggregateBilling { + totals: AggregateBillingTotals { + cache_read_tokens: total_billing.cache_read_tokens, + cache_write_tokens: total_billing.cache_write_tokens, + input_tokens: total_billing.input_tokens, + output_tokens: total_billing.output_tokens, + reasoning_tokens: total_billing.reasoning_tokens, + runs: agg.total_runs, + runtime_secs: agg.total_runtime_secs, + total_tokens: total_billing.total_tokens, + total_usd_micros: total_billing.total_usd_micros, + }, + by_model, + }; + (StatusCode::OK, Json(response)).into_response() +}