diff --git a/apps/arc-web/app/data/retros.ts b/apps/arc-web/app/data/retros.ts new file mode 100644 index 000000000..954291f91 --- /dev/null +++ b/apps/arc-web/app/data/retros.ts @@ -0,0 +1,437 @@ +export type SmoothnessRating = "effortless" | "smooth" | "bumpy" | "struggled" | "failed"; + +export type LearningCategory = "repo" | "code" | "workflow" | "tool"; + +export interface Learning { + category: LearningCategory; + text: string; +} + +export type FrictionKind = "retry" | "timeout" | "wrong_approach" | "tool_failure" | "ambiguity"; + +export interface FrictionPoint { + kind: FrictionKind; + description: string; + stage_id?: string; +} + +export type OpenItemKind = "tech_debt" | "follow_up" | "investigation" | "test_gap"; + +export interface OpenItem { + kind: OpenItemKind; + description: string; +} + +export interface StageRetro { + stage_id: string; + stage_label: string; + status: string; + duration_ms: number; + retries: number; + cost?: number; + notes?: string; + failure_reason?: string; + files_touched: string[]; +} + +export interface AggregateStats { + total_duration_ms: number; + total_cost?: number; + total_retries: number; + files_touched: string[]; + stages_completed: number; + stages_failed: number; +} + +export interface Retro { + run_id: string; + pipeline_name: string; + goal: string; + timestamp: string; + smoothness?: SmoothnessRating; + stages: StageRetro[]; + stats: AggregateStats; + intent?: string; + outcome?: string; + learnings?: Learning[]; + friction_points?: FrictionPoint[]; + open_items?: OpenItem[]; +} + +export const smoothnessConfig: Record = { + effortless: { label: "Effortless", bg: "bg-emerald-500/15", text: "text-emerald-400", dot: "bg-emerald-400" }, + smooth: { label: "Smooth", bg: "bg-mint/15", text: "text-mint", dot: "bg-mint" }, + bumpy: { label: "Bumpy", bg: "bg-amber/15", text: "text-amber", dot: "bg-amber" }, + struggled: { label: "Struggled", bg: "bg-orange-500/15", text: "text-orange-400", dot: "bg-orange-400" }, + failed: { label: "Failed", bg: "bg-coral/15", text: "text-coral", dot: "bg-coral" }, +}; + +export const learningCategoryConfig: Record = { + repo: { label: "Repo", text: "text-teal-400" }, + code: { label: "Code", text: "text-sky-400" }, + workflow: { label: "Workflow", text: "text-violet-400" }, + tool: { label: "Tool", text: "text-amber" }, +}; + +export const frictionKindConfig: Record = { + retry: { label: "Retry", text: "text-amber" }, + timeout: { label: "Timeout", text: "text-coral" }, + wrong_approach: { label: "Wrong Approach", text: "text-orange-400" }, + tool_failure: { label: "Tool Failure", text: "text-coral" }, + ambiguity: { label: "Ambiguity", text: "text-violet-400" }, +}; + +export const openItemKindConfig: Record = { + tech_debt: { label: "Tech Debt", text: "text-orange-400" }, + follow_up: { label: "Follow-up", text: "text-teal-400" }, + investigation: { label: "Investigation", text: "text-sky-400" }, + test_gap: { label: "Test Gap", text: "text-coral" }, +}; + +const mockRetros: Retro[] = [ + { + run_id: "run-1", + pipeline_name: "implement", + goal: "Add rate limiting to auth endpoints", + timestamp: "2026-02-28T14:32:00Z", + smoothness: "smooth", + intent: "Implement token-bucket rate limiting on /auth/login and /auth/register to prevent brute-force attacks.", + outcome: "Rate limiter deployed with configurable per-IP limits. Integration tests added. Redis-backed counter with sliding window.", + stages: [ + { + stage_id: "detect-drift", + stage_label: "Detect Drift", + status: "completed", + duration_ms: 72_000, + retries: 0, + cost: 0.48, + files_touched: ["src/middleware/rate-limit.ts"], + }, + { + stage_id: "propose-changes", + stage_label: "Propose Changes", + status: "completed", + duration_ms: 154_000, + retries: 0, + cost: 1.12, + files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts"], + }, + { + stage_id: "review-changes", + stage_label: "Review Changes", + status: "completed", + duration_ms: 45_000, + retries: 0, + cost: 0.31, + files_touched: [], + }, + { + stage_id: "apply-changes", + stage_label: "Apply Changes", + status: "completed", + duration_ms: 118_000, + retries: 0, + cost: 0.87, + files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts", "tests/rate-limit.test.ts"], + }, + ], + stats: { + total_duration_ms: 389_000, + total_cost: 2.78, + total_retries: 0, + files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts", "tests/rate-limit.test.ts"], + stages_completed: 4, + stages_failed: 0, + }, + learnings: [ + { category: "repo", text: "Redis client is initialized lazily in src/infra/redis.ts -- reuse existing connection pool." }, + { category: "code", text: "Auth middleware chain order matters: rate-limit must run before JWT validation." }, + ], + friction_points: [], + open_items: [ + { kind: "follow_up", description: "Add rate-limit headers (X-RateLimit-Remaining) to response." }, + ], + }, + { + run_id: "run-2", + pipeline_name: "implement", + goal: "Migrate to React Router v7", + timestamp: "2026-02-28T10:15:00Z", + smoothness: "bumpy", + intent: "Upgrade react-router from v6 to v7, updating all route definitions and loader/action patterns to the new API.", + outcome: "Migration completed but required 3 retries in the apply stage due to breaking changes in nested route handling. All routes now use the v7 data API.", + stages: [ + { + stage_id: "detect-drift", + stage_label: "Detect Drift", + status: "completed", + duration_ms: 95_000, + retries: 0, + cost: 0.62, + files_touched: ["package.json"], + }, + { + stage_id: "propose-changes", + stage_label: "Propose Changes", + status: "completed", + duration_ms: 312_000, + retries: 1, + cost: 2.45, + notes: "First proposal missed nested outlet patterns. Retry produced correct migration.", + files_touched: ["src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx"], + }, + { + stage_id: "review-changes", + stage_label: "Review Changes", + status: "completed", + duration_ms: 88_000, + retries: 0, + cost: 0.54, + files_touched: [], + }, + { + stage_id: "apply-changes", + stage_label: "Apply Changes", + status: "completed", + duration_ms: 480_000, + retries: 3, + cost: 3.21, + notes: "Type errors in nested layouts required multiple correction passes.", + files_touched: ["src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx", "src/routes/profile.tsx", "tests/routes.test.tsx"], + }, + ], + stats: { + total_duration_ms: 975_000, + total_cost: 6.82, + total_retries: 4, + files_touched: ["package.json", "src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx", "src/routes/profile.tsx", "tests/routes.test.tsx"], + stages_completed: 4, + stages_failed: 0, + }, + learnings: [ + { category: "workflow", text: "Framework migration tasks benefit from running type-check after each stage, not just at the end." }, + { category: "code", text: "React Router v7 outlets require explicit type annotations for loader data in nested routes." }, + { category: "tool", text: "The codemod tool missed JSX spread patterns -- manual fixup was needed." }, + ], + friction_points: [ + { kind: "retry", description: "Nested route outlet types were incorrect on first 3 attempts.", stage_id: "apply-changes" }, + { kind: "wrong_approach", description: "Initially tried to keep v6 compat layer, which created more issues than a clean migration.", stage_id: "propose-changes" }, + ], + open_items: [ + { kind: "tech_debt", description: "Leftover v6 compat shims in src/utils/router-compat.ts should be deleted." }, + { kind: "test_gap", description: "No E2E coverage for the new nested layout error boundaries." }, + ], + }, + { + run_id: "run-6", + pipeline_name: "implement", + goal: "Add dark mode toggle", + timestamp: "2026-02-27T16:45:00Z", + smoothness: "effortless", + intent: "Add a theme toggle component to the dashboard header with system/light/dark options, persisting preference to localStorage.", + outcome: "Dark mode toggle shipped with smooth CSS transitions. All existing components already used CSS variables, so no style refactoring was needed.", + stages: [ + { + stage_id: "detect-drift", + stage_label: "Detect Drift", + status: "completed", + duration_ms: 42_000, + retries: 0, + cost: 0.28, + files_touched: [], + }, + { + stage_id: "propose-changes", + stage_label: "Propose Changes", + status: "completed", + duration_ms: 98_000, + retries: 0, + cost: 0.71, + files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts"], + }, + { + stage_id: "apply-changes", + stage_label: "Apply Changes", + status: "completed", + duration_ms: 76_000, + retries: 0, + cost: 0.52, + files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts", "src/layouts/Header.tsx"], + }, + ], + stats: { + total_duration_ms: 216_000, + total_cost: 1.51, + total_retries: 0, + files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts", "src/layouts/Header.tsx"], + stages_completed: 3, + stages_failed: 0, + }, + learnings: [ + { category: "repo", text: "CSS variables are defined in src/styles/tokens.css and already support dark values." }, + ], + friction_points: [], + open_items: [], + }, + { + run_id: "run-3", + pipeline_name: "fix_build", + goal: "Fix config parsing for nested values", + timestamp: "2026-02-27T09:20:00Z", + smoothness: "struggled", + intent: "Fix TOML config parser to handle deeply nested table arrays, which was causing silent data loss on certain pipeline configs.", + outcome: "Root cause identified as incorrect recursion depth limit in the TOML walker. Fix applied but exposed a second bug in default value merging that required additional changes.", + stages: [ + { + stage_id: "investigate", + stage_label: "Investigate", + status: "completed", + duration_ms: 340_000, + retries: 2, + cost: 1.85, + notes: "First investigation looked at wrong parser path. Second attempt found the actual recursion limit.", + files_touched: ["src/config/parser.ts", "src/config/defaults.ts"], + }, + { + stage_id: "propose-fix", + stage_label: "Propose Fix", + status: "completed", + duration_ms: 210_000, + retries: 1, + cost: 1.42, + files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts"], + }, + { + stage_id: "apply-fix", + stage_label: "Apply Fix", + status: "completed", + duration_ms: 185_000, + retries: 1, + cost: 1.15, + failure_reason: "Initial fix broke the default value merging path. Required a second pass.", + files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts", "tests/config-parser.test.ts"], + }, + { + stage_id: "verify", + stage_label: "Verify", + status: "completed", + duration_ms: 95_000, + retries: 0, + cost: 0.55, + files_touched: [], + }, + ], + stats: { + total_duration_ms: 830_000, + total_cost: 4.97, + total_retries: 4, + files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts", "tests/config-parser.test.ts"], + stages_completed: 4, + stages_failed: 0, + }, + learnings: [ + { category: "code", text: "TOML walker in parser.ts has a hardcoded depth limit of 8 -- needs to be configurable." }, + { category: "code", text: "Default merging in merge.ts uses shallow spread, which silently drops nested keys." }, + { category: "workflow", text: "Bug fix pipelines should include a regression test stage before verification." }, + ], + friction_points: [ + { kind: "wrong_approach", description: "Initial investigation focused on the YAML compatibility layer instead of the TOML parser.", stage_id: "investigate" }, + { kind: "retry", description: "Fix introduced a regression in default value merging that required rework.", stage_id: "apply-fix" }, + { kind: "ambiguity", description: "Config schema docs were outdated, making it unclear which nesting depth was intended." }, + ], + open_items: [ + { kind: "tech_debt", description: "Remove the hardcoded depth limit in src/config/parser.ts and make it configurable." }, + { kind: "investigation", description: "Audit other parsers for similar shallow-spread bugs in merging logic." }, + { kind: "test_gap", description: "No tests for configs nested deeper than 4 levels." }, + ], + }, + { + run_id: "run-8", + pipeline_name: "implement", + goal: "Implement webhook retry logic", + timestamp: "2026-02-26T11:00:00Z", + smoothness: "smooth", + intent: "Add exponential backoff retry logic for failed webhook deliveries with configurable max attempts and dead-letter queue.", + outcome: "Webhook retry system implemented with exponential backoff (base 2s, max 5 retries). Failed deliveries route to SQS dead-letter queue. Dashboard shows retry status.", + stages: [ + { + stage_id: "detect-drift", + stage_label: "Detect Drift", + status: "completed", + duration_ms: 55_000, + retries: 0, + cost: 0.35, + files_touched: [], + }, + { + stage_id: "propose-changes", + stage_label: "Propose Changes", + status: "completed", + duration_ms: 178_000, + retries: 0, + cost: 1.28, + files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts"], + }, + { + stage_id: "review-changes", + stage_label: "Review Changes", + status: "completed", + duration_ms: 62_000, + retries: 0, + cost: 0.41, + files_touched: [], + }, + { + stage_id: "apply-changes", + stage_label: "Apply Changes", + status: "completed", + duration_ms: 145_000, + retries: 1, + cost: 1.05, + notes: "Minor type fix needed on retry delay calculation.", + files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts", "tests/webhook-retry.test.ts"], + }, + ], + stats: { + total_duration_ms: 440_000, + total_cost: 3.09, + total_retries: 1, + files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts", "tests/webhook-retry.test.ts"], + stages_completed: 4, + stages_failed: 0, + }, + learnings: [ + { category: "repo", text: "SQS client wrapper is in src/infra/sqs.ts with pre-configured DLQ ARNs per environment." }, + { category: "code", text: "Webhook dispatcher already had a hook point for retry logic via the onFailure callback." }, + ], + friction_points: [ + { kind: "retry", description: "Retry delay formula had an off-by-one in the exponent calculation.", stage_id: "apply-changes" }, + ], + open_items: [ + { kind: "follow_up", description: "Add webhook retry metrics to the Grafana dashboard." }, + { kind: "follow_up", description: "Document the DLQ reprocessing procedure in the runbook." }, + ], + }, +]; + +export function allRetros(): Retro[] { + return mockRetros; +} + +export function findRetro(runId: string): Retro | undefined { + return mockRetros.find((r) => r.run_id === runId); +} + +function formatDuration(ms: number): string { + const totalSeconds = Math.floor(ms / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + if (minutes >= 60) { + const hours = Math.floor(minutes / 60); + const remainMinutes = minutes % 60; + return `${hours}h ${remainMinutes}m`; + } + return `${minutes}m ${seconds}s`; +} + +export { formatDuration }; diff --git a/apps/arc-web/app/routes.ts b/apps/arc-web/app/routes.ts index db3d7241e..45756ddd6 100644 --- a/apps/arc-web/app/routes.ts +++ b/apps/arc-web/app/routes.ts @@ -24,6 +24,7 @@ export default [ route("graph", "routes/run-graph.tsx"), route("files", "routes/run-files-changed.tsx"), route("usage", "routes/run-usage.tsx"), + route("retro", "routes/run-retro.tsx"), ]), route("verifications", "routes/verifications.tsx"), route("retros", "routes/retros.tsx"), diff --git a/crates/arc-api/src/server.rs b/crates/arc-api/src/server.rs index c6c9af052..5670ebc6d 100644 --- a/crates/arc-api/src/server.rs +++ b/crates/arc-api/src/server.rs @@ -62,6 +62,7 @@ struct ManagedPipeline { checkpoint: Option, cancel_tx: Option>, cancel_token: Arc, + logs_root: Option, } /// Shared application state for the server. @@ -120,6 +121,7 @@ pub fn build_router(state: Arc, auth_mode: AuthMode) -> Router { .route("/pipelines/{id}/context", get(get_context)) .route("/pipelines/{id}/cancel", post(cancel_pipeline)) .route("/pipelines/{id}/graph", get(get_graph)) + .route("/pipelines/{id}/retro", get(get_retro)) .layer(axum::Extension(auth_mode)) .with_state(state) } @@ -219,6 +221,7 @@ async fn start_pipeline( checkpoint: None, cancel_tx: Some(cancel_tx), cancel_token: Arc::clone(&cancel_token), + logs_root: None, }, ); } @@ -255,6 +258,27 @@ async fn start_pipeline( // Save final checkpoint let checkpoint = Checkpoint::load(&config.logs_root.join("checkpoint.json")).ok(); + // Auto-derive retro + if let Some(ref cp) = checkpoint { + let (failed, failure_reason) = match &result { + Ok(_) => (false, None), + Err(e) => (true, Some(e.to_string())), + }; + let stage_durations = + arc_workflows::retro::extract_stage_durations(&config.logs_root); + let retro = arc_workflows::retro::derive_retro( + &run_id_clone, + "pipeline", + "", + cp, + failed, + failure_reason.as_deref(), + 0, + &stage_durations, + ); + let _ = retro.save(&config.logs_root); + } + let mut pipelines = state_clone .pipelines .lock() @@ -273,6 +297,7 @@ async fn start_pipeline( } } pipeline.checkpoint = checkpoint; + pipeline.logs_root = Some(config.logs_root.clone()); pipeline.event_tx = None; } }); @@ -461,6 +486,29 @@ async fn cancel_pipeline( } } +async fn get_retro( + _auth: AuthenticatedService, + State(state): State>, + Path(id): Path, +) -> Response { + let logs_root = { + let pipelines = state.pipelines.lock().expect("pipelines lock poisoned"); + match pipelines.get(&id) { + Some(pipeline) => pipeline.logs_root.clone(), + None => return StatusCode::NOT_FOUND.into_response(), + } + }; + + let Some(logs_root) = logs_root else { + return (StatusCode::OK, Json(serde_json::json!(null))).into_response(); + }; + + match arc_workflows::retro::Retro::load(&logs_root) { + Ok(retro) => (StatusCode::OK, Json(retro)).into_response(), + Err(_) => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + } +} + async fn get_graph( _auth: AuthenticatedService, State(state): State>, diff --git a/crates/arc-workflows/src/cli/run.rs b/crates/arc-workflows/src/cli/run.rs index aa1abc100..a18895b62 100644 --- a/crates/arc-workflows/src/cli/run.rs +++ b/crates/arc-workflows/src/cli/run.rs @@ -598,6 +598,64 @@ pub async fn run_command(args: RunArgs, styles: &'static Styles) -> anyhow::Resu } } + // Auto-derive retro (always, cheap) and optionally run retro agent + { + let (failed, failure_reason) = match &engine_result { + Ok(o) => ( + o.status == StageStatus::Fail, + o.failure_reason.clone(), + ), + Err(e) => (true, Some(e.to_string())), + }; + if let Ok(cp) = Checkpoint::load(&logs_dir.join("checkpoint.json")) { + let stage_durations = crate::retro::extract_stage_durations(&logs_dir); + let mut retro = crate::retro::derive_retro( + &config.run_id, + &graph.name, + graph.goal(), + &cp, + failed, + failure_reason.as_deref(), + run_duration_ms, + &stage_durations, + ); + let _ = retro.save(&logs_dir); + + // Run retro agent session (execution_env still alive via _cleanup_guard) + if !dry_run_mode { + if let Ok(client) = arc_llm::client::Client::from_env().await { + match crate::retro_agent::run_retro_agent( + &execution_env, + &logs_dir, + &client, + provider_enum, + &model, + ) + .await + { + Ok(narrative) => { + retro.apply_narrative(narrative); + let _ = retro.save(&logs_dir); + eprintln!( + "{dim}Retro saved to {}/retro.json{reset}", + logs_dir.display(), + dim = styles.dim, + reset = styles.reset, + ); + } + Err(e) => { + eprintln!( + "{dim}Retro agent skipped: {e}{reset}", + dim = styles.dim, + reset = styles.reset, + ); + } + } + } + } + } + } + let outcome = engine_result?; // 8. Print result diff --git a/crates/arc-workflows/src/lib.rs b/crates/arc-workflows/src/lib.rs index 74444bbcf..73b17d9ad 100644 --- a/crates/arc-workflows/src/lib.rs +++ b/crates/arc-workflows/src/lib.rs @@ -15,6 +15,8 @@ pub mod outcome; pub mod parser; pub mod pipeline; pub mod preamble; +pub mod retro; +pub mod retro_agent; pub mod stylesheet; pub mod transform; pub mod validation; diff --git a/crates/arc-workflows/src/retro.rs b/crates/arc-workflows/src/retro.rs new file mode 100644 index 000000000..797ce3b9c --- /dev/null +++ b/crates/arc-workflows/src/retro.rs @@ -0,0 +1,553 @@ +use std::collections::HashMap; +use std::path::Path; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::checkpoint::Checkpoint; +use crate::error::{ArcError, Result}; +use crate::event::PipelineEvent; +use crate::outcome::StageStatus; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SmoothnessRating { + Effortless, + Smooth, + Bumpy, + Struggled, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LearningCategory { + Repo, + Code, + Workflow, + Tool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Learning { + pub category: LearningCategory, + pub text: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FrictionKind { + Retry, + Timeout, + WrongApproach, + ToolFailure, + Ambiguity, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FrictionPoint { + pub kind: FrictionKind, + pub description: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stage_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpenItemKind { + TechDebt, + FollowUp, + Investigation, + TestGap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenItem { + pub kind: OpenItemKind, + pub description: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StageRetro { + pub stage_id: String, + pub stage_label: String, + pub status: String, + pub duration_ms: u64, + pub retries: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub notes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_reason: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub files_touched: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AggregateStats { + pub total_duration_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_cost: Option, + pub total_retries: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub files_touched: Vec, + pub stages_completed: usize, + pub stages_failed: usize, +} + +/// Agent-generated qualitative narrative fields. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetroNarrative { + pub smoothness: SmoothnessRating, + pub intent: String, + pub outcome: String, + #[serde(default)] + pub learnings: Vec, + #[serde(default)] + pub friction_points: Vec, + #[serde(default)] + pub open_items: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Retro { + pub run_id: String, + pub pipeline_name: String, + pub goal: String, + pub timestamp: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub smoothness: Option, + pub stages: Vec, + pub stats: AggregateStats, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub intent: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub outcome: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub learnings: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub friction_points: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub open_items: Option>, +} + +impl Retro { + /// Merge agent-generated narrative into this retro. + pub fn apply_narrative(&mut self, narrative: RetroNarrative) { + self.smoothness = Some(narrative.smoothness); + self.intent = Some(narrative.intent); + self.outcome = Some(narrative.outcome); + self.learnings = if narrative.learnings.is_empty() { + None + } else { + Some(narrative.learnings) + }; + self.friction_points = if narrative.friction_points.is_empty() { + None + } else { + Some(narrative.friction_points) + }; + self.open_items = if narrative.open_items.is_empty() { + None + } else { + Some(narrative.open_items) + }; + } + + /// Save the retro as JSON to `logs_root/retro.json`. + pub fn save(&self, logs_root: &Path) -> Result<()> { + let json = serde_json::to_string_pretty(self) + .map_err(|e| ArcError::Checkpoint(format!("retro serialize failed: {e}")))?; + std::fs::write(logs_root.join("retro.json"), json)?; + Ok(()) + } + + /// Load a retro from `logs_root/retro.json`. + pub fn load(logs_root: &Path) -> Result { + let data = std::fs::read_to_string(logs_root.join("retro.json"))?; + let retro: Self = serde_json::from_str(&data) + .map_err(|e| ArcError::Checkpoint(format!("retro deserialize failed: {e}")))?; + Ok(retro) + } +} + +/// Extract stage durations from `progress.ndjson` by reading `StageCompleted` events. +pub fn extract_stage_durations(logs_root: &Path) -> HashMap { + let mut durations = HashMap::new(); + let ndjson_path = logs_root.join("progress.ndjson"); + let Ok(data) = std::fs::read_to_string(&ndjson_path) else { + return durations; + }; + for line in data.lines() { + let Ok(envelope) = serde_json::from_str::(line) else { + continue; + }; + let Some(event_value) = envelope.get("event") else { + continue; + }; + let Ok(event) = serde_json::from_value::(event_value.clone()) else { + continue; + }; + if let PipelineEvent::StageCompleted { + name, duration_ms, .. + } = event + { + durations.insert(name, duration_ms); + } + } + durations +} + +/// Build a `Retro` from checkpoint data and run metadata. All qualitative +/// fields (`smoothness`, `intent`, `outcome`, etc.) are left as `None` for +/// the retro agent to fill in. +pub fn derive_retro( + run_id: &str, + pipeline_name: &str, + goal: &str, + checkpoint: &Checkpoint, + pipeline_failed: bool, + _pipeline_error: Option<&str>, + duration_ms: u64, + stage_durations: &HashMap, +) -> Retro { + let mut stages = Vec::new(); + let mut all_files: Vec = Vec::new(); + let mut total_cost: Option = None; + let mut total_retries: u32 = 0; + let mut stages_completed: usize = 0; + let mut stages_failed: usize = 0; + + for node_id in &checkpoint.completed_nodes { + let outcome = checkpoint.node_outcomes.get(node_id); + // node_retries stores attempts_used (1-indexed), convert to retry count + let retries = checkpoint + .node_retries + .get(node_id) + .copied() + .unwrap_or(1) + .saturating_sub(1); + total_retries += retries; + + let status = outcome + .map(|o| o.status.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + match outcome.map(|o| &o.status) { + Some(StageStatus::Success | StageStatus::PartialSuccess) => stages_completed += 1, + Some(StageStatus::Fail) => stages_failed += 1, + _ => {} + } + + let cost = outcome.and_then(|o| o.usage.as_ref()).and_then(|u| u.cost); + if let Some(c) = cost { + *total_cost.get_or_insert(0.0) += c; + } + + let files = outcome + .map(|o| o.files_touched.clone()) + .unwrap_or_default(); + all_files.extend(files.iter().cloned()); + + stages.push(StageRetro { + stage_id: node_id.clone(), + stage_label: node_id.clone(), + status, + duration_ms: stage_durations.get(node_id).copied().unwrap_or(0), + retries, + cost, + notes: outcome.and_then(|o| o.notes.clone()), + failure_reason: outcome.and_then(|o| o.failure_reason.clone()), + files_touched: files, + }); + } + + // If pipeline failed with an error not captured in stages, record it + if pipeline_failed && stages_failed == 0 { + stages_failed = 1; + } + + all_files.sort(); + all_files.dedup(); + + let stats = AggregateStats { + total_duration_ms: duration_ms, + total_cost, + total_retries, + files_touched: all_files, + stages_completed, + stages_failed, + }; + + Retro { + run_id: run_id.to_string(), + pipeline_name: pipeline_name.to_string(), + goal: goal.to_string(), + timestamp: Utc::now(), + smoothness: None, + stages, + stats, + intent: None, + outcome: None, + learnings: None, + friction_points: None, + open_items: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::outcome::Outcome; + + fn make_checkpoint_with_stages() -> Checkpoint { + let mut node_outcomes = HashMap::new(); + let mut outcome_a = Outcome::success(); + outcome_a.notes = Some("Planned the approach".to_string()); + outcome_a.files_touched = vec!["src/main.rs".to_string()]; + outcome_a.usage = Some(crate::outcome::StageUsage { + model: "claude-opus-4-6".to_string(), + input_tokens: 1000, + output_tokens: 500, + cache_read_tokens: None, + cache_write_tokens: None, + reasoning_tokens: None, + cost: Some(0.05), + }); + node_outcomes.insert("plan".to_string(), outcome_a); + + let mut outcome_b = Outcome::success(); + outcome_b.files_touched = vec!["src/main.rs".to_string(), "src/lib.rs".to_string()]; + outcome_b.usage = Some(crate::outcome::StageUsage { + model: "claude-opus-4-6".to_string(), + input_tokens: 2000, + output_tokens: 1000, + cache_read_tokens: None, + cache_write_tokens: None, + reasoning_tokens: None, + cost: Some(0.10), + }); + node_outcomes.insert("code".to_string(), outcome_b); + + let mut node_retries = HashMap::new(); + // 2 attempts_used = 1 actual retry + node_retries.insert("code".to_string(), 2u32); + + Checkpoint { + timestamp: Utc::now(), + current_node: "code".to_string(), + completed_nodes: vec!["plan".to_string(), "code".to_string()], + node_retries, + context_values: HashMap::new(), + logs: Vec::new(), + node_outcomes, + next_node_id: None, + git_commit_sha: None, + } + } + + #[test] + fn derive_retro_builds_stages_from_checkpoint() { + let cp = make_checkpoint_with_stages(); + let durations: HashMap = + [("plan".to_string(), 5000), ("code".to_string(), 15000)] + .into_iter() + .collect(); + + let retro = derive_retro("run-1", "my_pipeline", "Fix the bug", &cp, false, None, 20000, &durations); + + assert_eq!(retro.run_id, "run-1"); + assert_eq!(retro.pipeline_name, "my_pipeline"); + assert_eq!(retro.goal, "Fix the bug"); + assert_eq!(retro.stages.len(), 2); + assert_eq!(retro.stages[0].stage_id, "plan"); + assert_eq!(retro.stages[0].duration_ms, 5000); + assert_eq!(retro.stages[0].retries, 0); + assert_eq!(retro.stages[1].stage_id, "code"); + assert_eq!(retro.stages[1].duration_ms, 15000); + assert_eq!(retro.stages[1].retries, 1); + assert_eq!(retro.stats.total_duration_ms, 20000); + assert_eq!(retro.stats.total_retries, 1); + assert_eq!(retro.stats.stages_completed, 2); + assert_eq!(retro.stats.stages_failed, 0); + assert!((retro.stats.total_cost.unwrap() - 0.15).abs() < f64::EPSILON); + assert_eq!(retro.stats.files_touched, vec!["src/lib.rs", "src/main.rs"]); + assert!(retro.smoothness.is_none()); + assert!(retro.intent.is_none()); + } + + #[test] + fn derive_retro_handles_failed_pipeline() { + let cp = Checkpoint { + timestamp: Utc::now(), + current_node: "start".to_string(), + completed_nodes: vec!["start".to_string()], + node_retries: HashMap::new(), + context_values: HashMap::new(), + logs: Vec::new(), + node_outcomes: { + let mut m = HashMap::new(); + m.insert("start".to_string(), Outcome::success()); + m + }, + next_node_id: None, + git_commit_sha: None, + }; + + let retro = derive_retro( + "run-2", "pipe", "goal", &cp, true, + Some("boom"), 5000, &HashMap::new(), + ); + + assert_eq!(retro.stats.stages_failed, 1); + assert_eq!(retro.stats.stages_completed, 1); + } + + #[test] + fn apply_narrative_merges_fields() { + let cp = make_checkpoint_with_stages(); + let mut retro = derive_retro("r1", "p", "g", &cp, false, None, 1000, &HashMap::new()); + + let narrative = RetroNarrative { + smoothness: SmoothnessRating::Smooth, + intent: "Fix authentication bug".to_string(), + outcome: "Successfully fixed the login flow".to_string(), + learnings: vec![Learning { + category: LearningCategory::Code, + text: "Token refresh logic was in the wrong module".to_string(), + }], + friction_points: vec![], + open_items: vec![OpenItem { + kind: OpenItemKind::TestGap, + description: "No integration test for token refresh".to_string(), + }], + }; + + retro.apply_narrative(narrative); + + assert_eq!(retro.smoothness, Some(SmoothnessRating::Smooth)); + assert_eq!(retro.intent.as_deref(), Some("Fix authentication bug")); + assert_eq!(retro.outcome.as_deref(), Some("Successfully fixed the login flow")); + assert_eq!(retro.learnings.as_ref().unwrap().len(), 1); + assert!(retro.friction_points.is_none()); // empty vec -> None + assert_eq!(retro.open_items.as_ref().unwrap().len(), 1); + } + + #[test] + fn save_and_load_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let cp = make_checkpoint_with_stages(); + let mut retro = derive_retro("r1", "pipe", "goal", &cp, false, None, 1000, &HashMap::new()); + retro.smoothness = Some(SmoothnessRating::Bumpy); + retro.intent = Some("Test intent".to_string()); + + retro.save(dir.path()).unwrap(); + let loaded = Retro::load(dir.path()).unwrap(); + + assert_eq!(loaded.run_id, "r1"); + assert_eq!(loaded.smoothness, Some(SmoothnessRating::Bumpy)); + assert_eq!(loaded.intent.as_deref(), Some("Test intent")); + assert_eq!(loaded.stages.len(), 2); + } + + #[test] + fn load_nonexistent_retro() { + let result = Retro::load(Path::new("/nonexistent")); + assert!(result.is_err()); + } + + #[test] + fn smoothness_rating_serde_roundtrip() { + let json = serde_json::to_string(&SmoothnessRating::Effortless).unwrap(); + assert_eq!(json, "\"effortless\""); + let parsed: SmoothnessRating = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, SmoothnessRating::Effortless); + } + + #[test] + fn retro_narrative_serde() { + let narrative = RetroNarrative { + smoothness: SmoothnessRating::Failed, + intent: "Deploy feature".to_string(), + outcome: "Build failed".to_string(), + learnings: vec![], + friction_points: vec![FrictionPoint { + kind: FrictionKind::ToolFailure, + description: "Compiler error".to_string(), + stage_id: Some("build".to_string()), + }], + open_items: vec![], + }; + let json = serde_json::to_string(&narrative).unwrap(); + let parsed: RetroNarrative = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.smoothness, SmoothnessRating::Failed); + assert_eq!(parsed.friction_points.len(), 1); + assert_eq!(parsed.friction_points[0].kind, FrictionKind::ToolFailure); + } + + #[test] + fn extract_stage_durations_from_ndjson() { + let dir = tempfile::tempdir().unwrap(); + let ndjson = dir.path().join("progress.ndjson"); + + let event1 = serde_json::json!({ + "timestamp": "2025-01-01T00:00:00.000Z", + "run_id": "r1", + "event": { + "StageCompleted": { + "name": "plan", + "index": 0, + "duration_ms": 5000, + "status": "success", + "preferred_label": null, + "suggested_next_ids": [], + "usage": null, + "failure_reason": null, + "notes": null, + "files_touched": [], + "attempt": 1, + "max_attempts": 1, + "failure_class": null + } + } + }); + let event2 = serde_json::json!({ + "timestamp": "2025-01-01T00:00:05.000Z", + "run_id": "r1", + "event": { + "StageCompleted": { + "name": "code", + "index": 1, + "duration_ms": 15000, + "status": "success", + "preferred_label": null, + "suggested_next_ids": [], + "usage": null, + "failure_reason": null, + "notes": null, + "files_touched": [], + "attempt": 1, + "max_attempts": 1, + "failure_class": null + } + } + }); + let content = format!( + "{}\n{}\n", + serde_json::to_string(&event1).unwrap(), + serde_json::to_string(&event2).unwrap() + ); + std::fs::write(&ndjson, content).unwrap(); + + let durations = extract_stage_durations(dir.path()); + assert_eq!(durations.get("plan"), Some(&5000)); + assert_eq!(durations.get("code"), Some(&15000)); + } + + #[test] + fn extract_stage_durations_missing_file() { + let durations = extract_stage_durations(Path::new("/nonexistent")); + assert!(durations.is_empty()); + } +} diff --git a/crates/arc-workflows/src/retro_agent.rs b/crates/arc-workflows/src/retro_agent.rs new file mode 100644 index 000000000..37d0fd3f7 --- /dev/null +++ b/crates/arc-workflows/src/retro_agent.rs @@ -0,0 +1,288 @@ +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use arc_agent::{ + AnthropicProfile, ExecutionEnvironment, GeminiProfile, OpenAiProfile, ProviderProfile, + Session, SessionConfig, +}; +use arc_llm::client::Client; +use arc_llm::provider::Provider; +use arc_llm::types::ToolDefinition; + +use crate::retro::RetroNarrative; + +const RETRO_SYSTEM_PROMPT: &str = r#"You are a pipeline retrospective analyst. Your job is to analyze a completed pipeline run and generate a structured retrospective. + +You have access to the pipeline's data files: +- `progress.ndjson` — the full event stream (stage starts/completions, agent tool calls, errors, retries) +- `checkpoint.json` — final execution state with node outcomes +- `manifest.json` — run metadata (if available) + +## Your task + +1. **Explore the data** using grep and read tools to understand what happened: + - Look for failures, retries, and error messages + - Check agent tool call patterns for wrong approaches or pivots + - Note which stages took longest or had issues + - Look for patterns indicating friction (repeated similar tool calls, error recovery) + +2. **Call the `submit_retro` tool** with your structured analysis. + +## Smoothness grading guidelines + +Grade the run on a 5-point scale: + +- **effortless** — Pipeline achieved its goal on the first try with no retries, no wrong approaches. Agent moved efficiently from start to finish. +- **smooth** — Goal achieved with minor hiccups (1-2 retries or a brief wrong approach quickly corrected). No human intervention needed. Overall clean execution. +- **bumpy** — Goal achieved but with notable friction: multiple retries, at least one significant wrong approach, or substantial time spent on dead ends. +- **struggled** — Goal achieved only with difficulty: many retries, major approach changes, human intervention, or partial failures requiring recovery. +- **failed** — Pipeline did not achieve its stated goal. May have completed some stages but the overall intent was not fulfilled. + +Consider the full context: not just stage pass/fail, but the quality of the journey visible in the agent events (tool call patterns, error recovery, approach pivots). + +## Guidelines for qualitative fields + +- **intent**: What was the pipeline trying to accomplish? Summarize the goal in a sentence. +- **outcome**: What actually happened? Did it succeed? What was produced? +- **learnings**: What was discovered about the repo, code, workflow, or tools? +- **friction_points**: Where did things get stuck? What caused slowdowns? +- **open_items**: What follow-up work, tech debt, or gaps were identified? + +Be specific and concise. Reference actual stage names, file paths, and error messages where relevant."#; + +const SUBMIT_RETRO_SCHEMA: &str = r#"{ + "type": "object", + "properties": { + "smoothness": { + "type": "string", + "enum": ["effortless", "smooth", "bumpy", "struggled", "failed"], + "description": "Overall smoothness rating for the pipeline run" + }, + "intent": { + "type": "string", + "description": "What was the pipeline trying to accomplish?" + }, + "outcome": { + "type": "string", + "description": "What actually happened? Did it succeed?" + }, + "learnings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { "type": "string", "enum": ["repo", "code", "workflow", "tool"] }, + "text": { "type": "string" } + }, + "required": ["category", "text"] + }, + "description": "What was discovered during the run?" + }, + "friction_points": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { "type": "string", "enum": ["retry", "timeout", "wrong_approach", "tool_failure", "ambiguity"] }, + "description": { "type": "string" }, + "stage_id": { "type": "string" } + }, + "required": ["kind", "description"] + }, + "description": "Where did things get stuck?" + }, + "open_items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { "type": "string", "enum": ["tech_debt", "follow_up", "investigation", "test_gap"] }, + "description": { "type": "string" } + }, + "required": ["kind", "description"] + }, + "description": "Follow-up work or gaps identified" + } + }, + "required": ["smoothness", "intent", "outcome"] +}"#; + +/// Run a retro agent session that analyzes pipeline run data and produces +/// a structured narrative. The agent explores `progress.ndjson` and other +/// files via tool access, then calls `submit_retro` with its analysis. +pub async fn run_retro_agent( + execution_env: &Arc, + logs_root: &Path, + llm_client: &Client, + provider: Provider, + model: &str, +) -> anyhow::Result { + // Upload data files into execution env (needed for Daytona; no-op effect for local + // since the agent can also read from the original paths via tools). + let retro_data_dir = "/tmp/retro_data"; + upload_data_files(execution_env, logs_root, retro_data_dir).await?; + + // Build provider profile with the submit_retro tool + let captured: Arc>> = Arc::new(Mutex::new(None)); + let captured_clone = Arc::clone(&captured); + + let mut profile = build_profile(provider, model); + + // Register submit_retro tool + let submit_tool = arc_agent::tool_registry::RegisteredTool { + definition: ToolDefinition { + name: "submit_retro".to_string(), + description: "Submit the structured retrospective analysis. Call this once you have analyzed the pipeline run data.".to_string(), + parameters: serde_json::from_str(SUBMIT_RETRO_SCHEMA) + .expect("submit_retro schema should be valid JSON"), + }, + executor: Arc::new(move |args, _ctx| { + let captured = Arc::clone(&captured_clone); + Box::pin(async move { + let narrative: RetroNarrative = serde_json::from_value(args) + .map_err(|e| format!("Invalid retro submission: {e}"))?; + *captured.lock().unwrap() = Some(narrative); + Ok("Retrospective submitted successfully.".to_string()) + }) + }), + }; + profile.tool_registry_mut().register(submit_tool); + + let profile: Arc = Arc::from(profile); + + let config = SessionConfig { + max_tool_rounds_per_input: 10, + // Disable features not needed for retro analysis + enable_context_compaction: false, + skill_dirs: Some(vec![]), + user_instructions: Some(RETRO_SYSTEM_PROMPT.to_string()), + ..SessionConfig::default() + }; + + let mut session = Session::new( + llm_client.clone(), + profile, + Arc::clone(execution_env), + config, + ); + + session.initialize().await; + + let prompt = format!( + "Analyze the pipeline run data at `{retro_data_dir}/` and generate a retrospective. \ + The key file is `{retro_data_dir}/progress.ndjson` which contains the full event stream. \ + Also check `{retro_data_dir}/checkpoint.json` for stage outcomes. \ + Use grep to search for interesting signals (failures, retries, errors, approach changes) \ + rather than reading the entire file. When done, call the `submit_retro` tool with your analysis." + ); + + session.process_input(&prompt).await.map_err(|e| { + anyhow::anyhow!("Retro agent session failed: {e}") + })?; + + // Extract the captured narrative + let narrative = captured + .lock() + .unwrap() + .take() + .ok_or_else(|| anyhow::anyhow!("Retro agent did not call submit_retro"))?; + + Ok(narrative) +} + +fn build_profile(provider: Provider, model: &str) -> Box { + match provider { + Provider::OpenAi => Box::new(OpenAiProfile::new(model)), + Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception => { + Box::new(OpenAiProfile::new(model).with_provider(provider)) + } + Provider::Gemini => Box::new(GeminiProfile::new(model)), + Provider::Anthropic => Box::new(AnthropicProfile::new(model)), + } +} + +async fn upload_data_files( + execution_env: &Arc, + logs_root: &Path, + target_dir: &str, +) -> anyhow::Result<()> { + // Create target directory + execution_env + .exec_command(&format!("mkdir -p {target_dir}"), 10_000, None, None, None) + .await + .map_err(|e| anyhow::anyhow!("Failed to create retro data dir: {e}"))?; + + let files = ["progress.ndjson", "checkpoint.json", "manifest.json"]; + for filename in &files { + let source = logs_root.join(filename); + if source.exists() { + let content = std::fs::read_to_string(&source)?; + execution_env + .write_file(&format!("{target_dir}/{filename}"), &content) + .await + .map_err(|e| anyhow::anyhow!("Failed to upload {filename}: {e}"))?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn submit_retro_schema_is_valid_json() { + let schema: serde_json::Value = serde_json::from_str(SUBMIT_RETRO_SCHEMA).unwrap(); + assert_eq!(schema["type"], "object"); + assert!(schema["properties"]["smoothness"].is_object()); + assert!(schema["properties"]["intent"].is_object()); + assert!(schema["properties"]["outcome"].is_object()); + } + + #[test] + fn retro_narrative_parses_from_submit_retro_args() { + let args = serde_json::json!({ + "smoothness": "smooth", + "intent": "Fix the login bug", + "outcome": "Successfully fixed the authentication flow", + "learnings": [ + { "category": "code", "text": "Token refresh was in wrong module" } + ], + "friction_points": [ + { "kind": "retry", "description": "First attempt had wrong import", "stage_id": "code" } + ], + "open_items": [ + { "kind": "test_gap", "description": "No integration test for token refresh" } + ] + }); + + let narrative: RetroNarrative = serde_json::from_value(args).unwrap(); + assert_eq!( + narrative.smoothness, + crate::retro::SmoothnessRating::Smooth + ); + assert_eq!(narrative.intent, "Fix the login bug"); + assert_eq!(narrative.learnings.len(), 1); + assert_eq!(narrative.friction_points.len(), 1); + assert_eq!(narrative.open_items.len(), 1); + } + + #[test] + fn retro_narrative_parses_minimal_args() { + let args = serde_json::json!({ + "smoothness": "effortless", + "intent": "Deploy feature", + "outcome": "Deployed successfully" + }); + + let narrative: RetroNarrative = serde_json::from_value(args).unwrap(); + assert_eq!( + narrative.smoothness, + crate::retro::SmoothnessRating::Effortless + ); + assert!(narrative.learnings.is_empty()); + assert!(narrative.friction_points.is_empty()); + assert!(narrative.open_items.is_empty()); + } +} diff --git a/test/retro-e2e.dot b/test/retro-e2e.dot new file mode 100644 index 000000000..86b94cac3 --- /dev/null +++ b/test/retro-e2e.dot @@ -0,0 +1,12 @@ +digraph RetroE2E { + graph [goal="Verify retro generation works end-to-end"] + rankdir=LR + + start [shape=Mdiamond, label="Start"] + exit [shape=Msquare, label="Exit"] + + check [label="Check", shape=parallelogram, script="echo 'Hello from Daytona sandbox' && ls -la"] + summarize [label="Summarize", prompt="Briefly summarize what the check stage did. Keep it under 2 sentences."] + + start -> check -> summarize -> exit +}