mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-07 08:27:12 +00:00
Add per-run retrospectives with LLM-powered retro agent
After each pipeline run, auto-derive stats from the checkpoint (stages,
retries, cost, files touched) then run an Opus agent session that
explores progress.ndjson to produce qualitative analysis: smoothness
rating, intent, outcome, learnings, friction points, and open items.
Backend:
- retro.rs: data model, save/load, derive_retro(), extract_stage_durations()
- retro_agent.rs: post-pipeline agent session with submit_retro tool
- cli/run.rs: hook retro generation after final.json, before engine_result?
- server.rs: GET /pipelines/{id}/retro endpoint, auto-derive on completion
Frontend:
- data/retros.ts: TS types + mock data + smoothness color config
- routes/retros.tsx: list page with smoothness badges
- routes/run-retro.tsx: detail view (stats, intent, stages, learnings)
- routes.ts + run-detail.tsx: wire up retro route and tab
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
4f085b1f12
commit
73b00f047b
8 changed files with 1399 additions and 0 deletions
437
apps/arc-web/app/data/retros.ts
Normal file
437
apps/arc-web/app/data/retros.ts
Normal file
|
|
@ -0,0 +1,437 @@
|
|||
export type SmoothnessRating = "effortless" | "smooth" | "bumpy" | "struggled" | "failed";
|
||||
|
||||
export type LearningCategory = "repo" | "code" | "workflow" | "tool";
|
||||
|
||||
export interface Learning {
|
||||
category: LearningCategory;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export type FrictionKind = "retry" | "timeout" | "wrong_approach" | "tool_failure" | "ambiguity";
|
||||
|
||||
export interface FrictionPoint {
|
||||
kind: FrictionKind;
|
||||
description: string;
|
||||
stage_id?: string;
|
||||
}
|
||||
|
||||
export type OpenItemKind = "tech_debt" | "follow_up" | "investigation" | "test_gap";
|
||||
|
||||
export interface OpenItem {
|
||||
kind: OpenItemKind;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface StageRetro {
|
||||
stage_id: string;
|
||||
stage_label: string;
|
||||
status: string;
|
||||
duration_ms: number;
|
||||
retries: number;
|
||||
cost?: number;
|
||||
notes?: string;
|
||||
failure_reason?: string;
|
||||
files_touched: string[];
|
||||
}
|
||||
|
||||
export interface AggregateStats {
|
||||
total_duration_ms: number;
|
||||
total_cost?: number;
|
||||
total_retries: number;
|
||||
files_touched: string[];
|
||||
stages_completed: number;
|
||||
stages_failed: number;
|
||||
}
|
||||
|
||||
export interface Retro {
|
||||
run_id: string;
|
||||
pipeline_name: string;
|
||||
goal: string;
|
||||
timestamp: string;
|
||||
smoothness?: SmoothnessRating;
|
||||
stages: StageRetro[];
|
||||
stats: AggregateStats;
|
||||
intent?: string;
|
||||
outcome?: string;
|
||||
learnings?: Learning[];
|
||||
friction_points?: FrictionPoint[];
|
||||
open_items?: OpenItem[];
|
||||
}
|
||||
|
||||
export const smoothnessConfig: Record<SmoothnessRating, { label: string; bg: string; text: string; dot: string }> = {
|
||||
effortless: { label: "Effortless", bg: "bg-emerald-500/15", text: "text-emerald-400", dot: "bg-emerald-400" },
|
||||
smooth: { label: "Smooth", bg: "bg-mint/15", text: "text-mint", dot: "bg-mint" },
|
||||
bumpy: { label: "Bumpy", bg: "bg-amber/15", text: "text-amber", dot: "bg-amber" },
|
||||
struggled: { label: "Struggled", bg: "bg-orange-500/15", text: "text-orange-400", dot: "bg-orange-400" },
|
||||
failed: { label: "Failed", bg: "bg-coral/15", text: "text-coral", dot: "bg-coral" },
|
||||
};
|
||||
|
||||
export const learningCategoryConfig: Record<LearningCategory, { label: string; text: string }> = {
|
||||
repo: { label: "Repo", text: "text-teal-400" },
|
||||
code: { label: "Code", text: "text-sky-400" },
|
||||
workflow: { label: "Workflow", text: "text-violet-400" },
|
||||
tool: { label: "Tool", text: "text-amber" },
|
||||
};
|
||||
|
||||
export const frictionKindConfig: Record<FrictionKind, { label: string; text: string }> = {
|
||||
retry: { label: "Retry", text: "text-amber" },
|
||||
timeout: { label: "Timeout", text: "text-coral" },
|
||||
wrong_approach: { label: "Wrong Approach", text: "text-orange-400" },
|
||||
tool_failure: { label: "Tool Failure", text: "text-coral" },
|
||||
ambiguity: { label: "Ambiguity", text: "text-violet-400" },
|
||||
};
|
||||
|
||||
export const openItemKindConfig: Record<OpenItemKind, { label: string; text: string }> = {
|
||||
tech_debt: { label: "Tech Debt", text: "text-orange-400" },
|
||||
follow_up: { label: "Follow-up", text: "text-teal-400" },
|
||||
investigation: { label: "Investigation", text: "text-sky-400" },
|
||||
test_gap: { label: "Test Gap", text: "text-coral" },
|
||||
};
|
||||
|
||||
const mockRetros: Retro[] = [
|
||||
{
|
||||
run_id: "run-1",
|
||||
pipeline_name: "implement",
|
||||
goal: "Add rate limiting to auth endpoints",
|
||||
timestamp: "2026-02-28T14:32:00Z",
|
||||
smoothness: "smooth",
|
||||
intent: "Implement token-bucket rate limiting on /auth/login and /auth/register to prevent brute-force attacks.",
|
||||
outcome: "Rate limiter deployed with configurable per-IP limits. Integration tests added. Redis-backed counter with sliding window.",
|
||||
stages: [
|
||||
{
|
||||
stage_id: "detect-drift",
|
||||
stage_label: "Detect Drift",
|
||||
status: "completed",
|
||||
duration_ms: 72_000,
|
||||
retries: 0,
|
||||
cost: 0.48,
|
||||
files_touched: ["src/middleware/rate-limit.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "propose-changes",
|
||||
stage_label: "Propose Changes",
|
||||
status: "completed",
|
||||
duration_ms: 154_000,
|
||||
retries: 0,
|
||||
cost: 1.12,
|
||||
files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "review-changes",
|
||||
stage_label: "Review Changes",
|
||||
status: "completed",
|
||||
duration_ms: 45_000,
|
||||
retries: 0,
|
||||
cost: 0.31,
|
||||
files_touched: [],
|
||||
},
|
||||
{
|
||||
stage_id: "apply-changes",
|
||||
stage_label: "Apply Changes",
|
||||
status: "completed",
|
||||
duration_ms: 118_000,
|
||||
retries: 0,
|
||||
cost: 0.87,
|
||||
files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts", "tests/rate-limit.test.ts"],
|
||||
},
|
||||
],
|
||||
stats: {
|
||||
total_duration_ms: 389_000,
|
||||
total_cost: 2.78,
|
||||
total_retries: 0,
|
||||
files_touched: ["src/middleware/rate-limit.ts", "src/routes/auth.ts", "src/config.ts", "tests/rate-limit.test.ts"],
|
||||
stages_completed: 4,
|
||||
stages_failed: 0,
|
||||
},
|
||||
learnings: [
|
||||
{ category: "repo", text: "Redis client is initialized lazily in src/infra/redis.ts -- reuse existing connection pool." },
|
||||
{ category: "code", text: "Auth middleware chain order matters: rate-limit must run before JWT validation." },
|
||||
],
|
||||
friction_points: [],
|
||||
open_items: [
|
||||
{ kind: "follow_up", description: "Add rate-limit headers (X-RateLimit-Remaining) to response." },
|
||||
],
|
||||
},
|
||||
{
|
||||
run_id: "run-2",
|
||||
pipeline_name: "implement",
|
||||
goal: "Migrate to React Router v7",
|
||||
timestamp: "2026-02-28T10:15:00Z",
|
||||
smoothness: "bumpy",
|
||||
intent: "Upgrade react-router from v6 to v7, updating all route definitions and loader/action patterns to the new API.",
|
||||
outcome: "Migration completed but required 3 retries in the apply stage due to breaking changes in nested route handling. All routes now use the v7 data API.",
|
||||
stages: [
|
||||
{
|
||||
stage_id: "detect-drift",
|
||||
stage_label: "Detect Drift",
|
||||
status: "completed",
|
||||
duration_ms: 95_000,
|
||||
retries: 0,
|
||||
cost: 0.62,
|
||||
files_touched: ["package.json"],
|
||||
},
|
||||
{
|
||||
stage_id: "propose-changes",
|
||||
stage_label: "Propose Changes",
|
||||
status: "completed",
|
||||
duration_ms: 312_000,
|
||||
retries: 1,
|
||||
cost: 2.45,
|
||||
notes: "First proposal missed nested outlet patterns. Retry produced correct migration.",
|
||||
files_touched: ["src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx"],
|
||||
},
|
||||
{
|
||||
stage_id: "review-changes",
|
||||
stage_label: "Review Changes",
|
||||
status: "completed",
|
||||
duration_ms: 88_000,
|
||||
retries: 0,
|
||||
cost: 0.54,
|
||||
files_touched: [],
|
||||
},
|
||||
{
|
||||
stage_id: "apply-changes",
|
||||
stage_label: "Apply Changes",
|
||||
status: "completed",
|
||||
duration_ms: 480_000,
|
||||
retries: 3,
|
||||
cost: 3.21,
|
||||
notes: "Type errors in nested layouts required multiple correction passes.",
|
||||
files_touched: ["src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx", "src/routes/profile.tsx", "tests/routes.test.tsx"],
|
||||
},
|
||||
],
|
||||
stats: {
|
||||
total_duration_ms: 975_000,
|
||||
total_cost: 6.82,
|
||||
total_retries: 4,
|
||||
files_touched: ["package.json", "src/routes.ts", "src/app.tsx", "src/routes/dashboard.tsx", "src/routes/settings.tsx", "src/routes/profile.tsx", "tests/routes.test.tsx"],
|
||||
stages_completed: 4,
|
||||
stages_failed: 0,
|
||||
},
|
||||
learnings: [
|
||||
{ category: "workflow", text: "Framework migration tasks benefit from running type-check after each stage, not just at the end." },
|
||||
{ category: "code", text: "React Router v7 outlets require explicit type annotations for loader data in nested routes." },
|
||||
{ category: "tool", text: "The codemod tool missed JSX spread patterns -- manual fixup was needed." },
|
||||
],
|
||||
friction_points: [
|
||||
{ kind: "retry", description: "Nested route outlet types were incorrect on first 3 attempts.", stage_id: "apply-changes" },
|
||||
{ kind: "wrong_approach", description: "Initially tried to keep v6 compat layer, which created more issues than a clean migration.", stage_id: "propose-changes" },
|
||||
],
|
||||
open_items: [
|
||||
{ kind: "tech_debt", description: "Leftover v6 compat shims in src/utils/router-compat.ts should be deleted." },
|
||||
{ kind: "test_gap", description: "No E2E coverage for the new nested layout error boundaries." },
|
||||
],
|
||||
},
|
||||
{
|
||||
run_id: "run-6",
|
||||
pipeline_name: "implement",
|
||||
goal: "Add dark mode toggle",
|
||||
timestamp: "2026-02-27T16:45:00Z",
|
||||
smoothness: "effortless",
|
||||
intent: "Add a theme toggle component to the dashboard header with system/light/dark options, persisting preference to localStorage.",
|
||||
outcome: "Dark mode toggle shipped with smooth CSS transitions. All existing components already used CSS variables, so no style refactoring was needed.",
|
||||
stages: [
|
||||
{
|
||||
stage_id: "detect-drift",
|
||||
stage_label: "Detect Drift",
|
||||
status: "completed",
|
||||
duration_ms: 42_000,
|
||||
retries: 0,
|
||||
cost: 0.28,
|
||||
files_touched: [],
|
||||
},
|
||||
{
|
||||
stage_id: "propose-changes",
|
||||
stage_label: "Propose Changes",
|
||||
status: "completed",
|
||||
duration_ms: 98_000,
|
||||
retries: 0,
|
||||
cost: 0.71,
|
||||
files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "apply-changes",
|
||||
stage_label: "Apply Changes",
|
||||
status: "completed",
|
||||
duration_ms: 76_000,
|
||||
retries: 0,
|
||||
cost: 0.52,
|
||||
files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts", "src/layouts/Header.tsx"],
|
||||
},
|
||||
],
|
||||
stats: {
|
||||
total_duration_ms: 216_000,
|
||||
total_cost: 1.51,
|
||||
total_retries: 0,
|
||||
files_touched: ["src/components/ThemeToggle.tsx", "src/hooks/useTheme.ts", "src/layouts/Header.tsx"],
|
||||
stages_completed: 3,
|
||||
stages_failed: 0,
|
||||
},
|
||||
learnings: [
|
||||
{ category: "repo", text: "CSS variables are defined in src/styles/tokens.css and already support dark values." },
|
||||
],
|
||||
friction_points: [],
|
||||
open_items: [],
|
||||
},
|
||||
{
|
||||
run_id: "run-3",
|
||||
pipeline_name: "fix_build",
|
||||
goal: "Fix config parsing for nested values",
|
||||
timestamp: "2026-02-27T09:20:00Z",
|
||||
smoothness: "struggled",
|
||||
intent: "Fix TOML config parser to handle deeply nested table arrays, which was causing silent data loss on certain pipeline configs.",
|
||||
outcome: "Root cause identified as incorrect recursion depth limit in the TOML walker. Fix applied but exposed a second bug in default value merging that required additional changes.",
|
||||
stages: [
|
||||
{
|
||||
stage_id: "investigate",
|
||||
stage_label: "Investigate",
|
||||
status: "completed",
|
||||
duration_ms: 340_000,
|
||||
retries: 2,
|
||||
cost: 1.85,
|
||||
notes: "First investigation looked at wrong parser path. Second attempt found the actual recursion limit.",
|
||||
files_touched: ["src/config/parser.ts", "src/config/defaults.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "propose-fix",
|
||||
stage_label: "Propose Fix",
|
||||
status: "completed",
|
||||
duration_ms: 210_000,
|
||||
retries: 1,
|
||||
cost: 1.42,
|
||||
files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "apply-fix",
|
||||
stage_label: "Apply Fix",
|
||||
status: "completed",
|
||||
duration_ms: 185_000,
|
||||
retries: 1,
|
||||
cost: 1.15,
|
||||
failure_reason: "Initial fix broke the default value merging path. Required a second pass.",
|
||||
files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts", "tests/config-parser.test.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "verify",
|
||||
stage_label: "Verify",
|
||||
status: "completed",
|
||||
duration_ms: 95_000,
|
||||
retries: 0,
|
||||
cost: 0.55,
|
||||
files_touched: [],
|
||||
},
|
||||
],
|
||||
stats: {
|
||||
total_duration_ms: 830_000,
|
||||
total_cost: 4.97,
|
||||
total_retries: 4,
|
||||
files_touched: ["src/config/parser.ts", "src/config/defaults.ts", "src/config/merge.ts", "tests/config-parser.test.ts"],
|
||||
stages_completed: 4,
|
||||
stages_failed: 0,
|
||||
},
|
||||
learnings: [
|
||||
{ category: "code", text: "TOML walker in parser.ts has a hardcoded depth limit of 8 -- needs to be configurable." },
|
||||
{ category: "code", text: "Default merging in merge.ts uses shallow spread, which silently drops nested keys." },
|
||||
{ category: "workflow", text: "Bug fix pipelines should include a regression test stage before verification." },
|
||||
],
|
||||
friction_points: [
|
||||
{ kind: "wrong_approach", description: "Initial investigation focused on the YAML compatibility layer instead of the TOML parser.", stage_id: "investigate" },
|
||||
{ kind: "retry", description: "Fix introduced a regression in default value merging that required rework.", stage_id: "apply-fix" },
|
||||
{ kind: "ambiguity", description: "Config schema docs were outdated, making it unclear which nesting depth was intended." },
|
||||
],
|
||||
open_items: [
|
||||
{ kind: "tech_debt", description: "Remove the hardcoded depth limit in src/config/parser.ts and make it configurable." },
|
||||
{ kind: "investigation", description: "Audit other parsers for similar shallow-spread bugs in merging logic." },
|
||||
{ kind: "test_gap", description: "No tests for configs nested deeper than 4 levels." },
|
||||
],
|
||||
},
|
||||
{
|
||||
run_id: "run-8",
|
||||
pipeline_name: "implement",
|
||||
goal: "Implement webhook retry logic",
|
||||
timestamp: "2026-02-26T11:00:00Z",
|
||||
smoothness: "smooth",
|
||||
intent: "Add exponential backoff retry logic for failed webhook deliveries with configurable max attempts and dead-letter queue.",
|
||||
outcome: "Webhook retry system implemented with exponential backoff (base 2s, max 5 retries). Failed deliveries route to SQS dead-letter queue. Dashboard shows retry status.",
|
||||
stages: [
|
||||
{
|
||||
stage_id: "detect-drift",
|
||||
stage_label: "Detect Drift",
|
||||
status: "completed",
|
||||
duration_ms: 55_000,
|
||||
retries: 0,
|
||||
cost: 0.35,
|
||||
files_touched: [],
|
||||
},
|
||||
{
|
||||
stage_id: "propose-changes",
|
||||
stage_label: "Propose Changes",
|
||||
status: "completed",
|
||||
duration_ms: 178_000,
|
||||
retries: 0,
|
||||
cost: 1.28,
|
||||
files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts"],
|
||||
},
|
||||
{
|
||||
stage_id: "review-changes",
|
||||
stage_label: "Review Changes",
|
||||
status: "completed",
|
||||
duration_ms: 62_000,
|
||||
retries: 0,
|
||||
cost: 0.41,
|
||||
files_touched: [],
|
||||
},
|
||||
{
|
||||
stage_id: "apply-changes",
|
||||
stage_label: "Apply Changes",
|
||||
status: "completed",
|
||||
duration_ms: 145_000,
|
||||
retries: 1,
|
||||
cost: 1.05,
|
||||
notes: "Minor type fix needed on retry delay calculation.",
|
||||
files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts", "tests/webhook-retry.test.ts"],
|
||||
},
|
||||
],
|
||||
stats: {
|
||||
total_duration_ms: 440_000,
|
||||
total_cost: 3.09,
|
||||
total_retries: 1,
|
||||
files_touched: ["src/webhooks/retry.ts", "src/webhooks/dlq.ts", "src/webhooks/dispatcher.ts", "tests/webhook-retry.test.ts"],
|
||||
stages_completed: 4,
|
||||
stages_failed: 0,
|
||||
},
|
||||
learnings: [
|
||||
{ category: "repo", text: "SQS client wrapper is in src/infra/sqs.ts with pre-configured DLQ ARNs per environment." },
|
||||
{ category: "code", text: "Webhook dispatcher already had a hook point for retry logic via the onFailure callback." },
|
||||
],
|
||||
friction_points: [
|
||||
{ kind: "retry", description: "Retry delay formula had an off-by-one in the exponent calculation.", stage_id: "apply-changes" },
|
||||
],
|
||||
open_items: [
|
||||
{ kind: "follow_up", description: "Add webhook retry metrics to the Grafana dashboard." },
|
||||
{ kind: "follow_up", description: "Document the DLQ reprocessing procedure in the runbook." },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export function allRetros(): Retro[] {
|
||||
return mockRetros;
|
||||
}
|
||||
|
||||
export function findRetro(runId: string): Retro | undefined {
|
||||
return mockRetros.find((r) => r.run_id === runId);
|
||||
}
|
||||
|
||||
function formatDuration(ms: number): string {
|
||||
const totalSeconds = Math.floor(ms / 1000);
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
if (minutes >= 60) {
|
||||
const hours = Math.floor(minutes / 60);
|
||||
const remainMinutes = minutes % 60;
|
||||
return `${hours}h ${remainMinutes}m`;
|
||||
}
|
||||
return `${minutes}m ${seconds}s`;
|
||||
}
|
||||
|
||||
export { formatDuration };
|
||||
|
|
@ -24,6 +24,7 @@ export default [
|
|||
route("graph", "routes/run-graph.tsx"),
|
||||
route("files", "routes/run-files-changed.tsx"),
|
||||
route("usage", "routes/run-usage.tsx"),
|
||||
route("retro", "routes/run-retro.tsx"),
|
||||
]),
|
||||
route("verifications", "routes/verifications.tsx"),
|
||||
route("retros", "routes/retros.tsx"),
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ struct ManagedPipeline {
|
|||
checkpoint: Option<Checkpoint>,
|
||||
cancel_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
cancel_token: Arc<AtomicBool>,
|
||||
logs_root: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
/// Shared application state for the server.
|
||||
|
|
@ -120,6 +121,7 @@ pub fn build_router(state: Arc<AppState>, auth_mode: AuthMode) -> Router {
|
|||
.route("/pipelines/{id}/context", get(get_context))
|
||||
.route("/pipelines/{id}/cancel", post(cancel_pipeline))
|
||||
.route("/pipelines/{id}/graph", get(get_graph))
|
||||
.route("/pipelines/{id}/retro", get(get_retro))
|
||||
.layer(axum::Extension(auth_mode))
|
||||
.with_state(state)
|
||||
}
|
||||
|
|
@ -219,6 +221,7 @@ async fn start_pipeline(
|
|||
checkpoint: None,
|
||||
cancel_tx: Some(cancel_tx),
|
||||
cancel_token: Arc::clone(&cancel_token),
|
||||
logs_root: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -255,6 +258,27 @@ async fn start_pipeline(
|
|||
// Save final checkpoint
|
||||
let checkpoint = Checkpoint::load(&config.logs_root.join("checkpoint.json")).ok();
|
||||
|
||||
// Auto-derive retro
|
||||
if let Some(ref cp) = checkpoint {
|
||||
let (failed, failure_reason) = match &result {
|
||||
Ok(_) => (false, None),
|
||||
Err(e) => (true, Some(e.to_string())),
|
||||
};
|
||||
let stage_durations =
|
||||
arc_workflows::retro::extract_stage_durations(&config.logs_root);
|
||||
let retro = arc_workflows::retro::derive_retro(
|
||||
&run_id_clone,
|
||||
"pipeline",
|
||||
"",
|
||||
cp,
|
||||
failed,
|
||||
failure_reason.as_deref(),
|
||||
0,
|
||||
&stage_durations,
|
||||
);
|
||||
let _ = retro.save(&config.logs_root);
|
||||
}
|
||||
|
||||
let mut pipelines = state_clone
|
||||
.pipelines
|
||||
.lock()
|
||||
|
|
@ -273,6 +297,7 @@ async fn start_pipeline(
|
|||
}
|
||||
}
|
||||
pipeline.checkpoint = checkpoint;
|
||||
pipeline.logs_root = Some(config.logs_root.clone());
|
||||
pipeline.event_tx = None;
|
||||
}
|
||||
});
|
||||
|
|
@ -461,6 +486,29 @@ async fn cancel_pipeline(
|
|||
}
|
||||
}
|
||||
|
||||
async fn get_retro(
|
||||
_auth: AuthenticatedService,
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(id): Path<String>,
|
||||
) -> Response {
|
||||
let logs_root = {
|
||||
let pipelines = state.pipelines.lock().expect("pipelines lock poisoned");
|
||||
match pipelines.get(&id) {
|
||||
Some(pipeline) => pipeline.logs_root.clone(),
|
||||
None => return StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
};
|
||||
|
||||
let Some(logs_root) = logs_root else {
|
||||
return (StatusCode::OK, Json(serde_json::json!(null))).into_response();
|
||||
};
|
||||
|
||||
match arc_workflows::retro::Retro::load(&logs_root) {
|
||||
Ok(retro) => (StatusCode::OK, Json(retro)).into_response(),
|
||||
Err(_) => (StatusCode::OK, Json(serde_json::json!(null))).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_graph(
|
||||
_auth: AuthenticatedService,
|
||||
State(state): State<Arc<AppState>>,
|
||||
|
|
|
|||
|
|
@ -598,6 +598,64 @@ pub async fn run_command(args: RunArgs, styles: &'static Styles) -> anyhow::Resu
|
|||
}
|
||||
}
|
||||
|
||||
// Auto-derive retro (always, cheap) and optionally run retro agent
|
||||
{
|
||||
let (failed, failure_reason) = match &engine_result {
|
||||
Ok(o) => (
|
||||
o.status == StageStatus::Fail,
|
||||
o.failure_reason.clone(),
|
||||
),
|
||||
Err(e) => (true, Some(e.to_string())),
|
||||
};
|
||||
if let Ok(cp) = Checkpoint::load(&logs_dir.join("checkpoint.json")) {
|
||||
let stage_durations = crate::retro::extract_stage_durations(&logs_dir);
|
||||
let mut retro = crate::retro::derive_retro(
|
||||
&config.run_id,
|
||||
&graph.name,
|
||||
graph.goal(),
|
||||
&cp,
|
||||
failed,
|
||||
failure_reason.as_deref(),
|
||||
run_duration_ms,
|
||||
&stage_durations,
|
||||
);
|
||||
let _ = retro.save(&logs_dir);
|
||||
|
||||
// Run retro agent session (execution_env still alive via _cleanup_guard)
|
||||
if !dry_run_mode {
|
||||
if let Ok(client) = arc_llm::client::Client::from_env().await {
|
||||
match crate::retro_agent::run_retro_agent(
|
||||
&execution_env,
|
||||
&logs_dir,
|
||||
&client,
|
||||
provider_enum,
|
||||
&model,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(narrative) => {
|
||||
retro.apply_narrative(narrative);
|
||||
let _ = retro.save(&logs_dir);
|
||||
eprintln!(
|
||||
"{dim}Retro saved to {}/retro.json{reset}",
|
||||
logs_dir.display(),
|
||||
dim = styles.dim,
|
||||
reset = styles.reset,
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{dim}Retro agent skipped: {e}{reset}",
|
||||
dim = styles.dim,
|
||||
reset = styles.reset,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let outcome = engine_result?;
|
||||
|
||||
// 8. Print result
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ pub mod outcome;
|
|||
pub mod parser;
|
||||
pub mod pipeline;
|
||||
pub mod preamble;
|
||||
pub mod retro;
|
||||
pub mod retro_agent;
|
||||
pub mod stylesheet;
|
||||
pub mod transform;
|
||||
pub mod validation;
|
||||
|
|
|
|||
553
crates/arc-workflows/src/retro.rs
Normal file
553
crates/arc-workflows/src/retro.rs
Normal file
|
|
@ -0,0 +1,553 @@
|
|||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::checkpoint::Checkpoint;
|
||||
use crate::error::{ArcError, Result};
|
||||
use crate::event::PipelineEvent;
|
||||
use crate::outcome::StageStatus;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SmoothnessRating {
|
||||
Effortless,
|
||||
Smooth,
|
||||
Bumpy,
|
||||
Struggled,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum LearningCategory {
|
||||
Repo,
|
||||
Code,
|
||||
Workflow,
|
||||
Tool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Learning {
|
||||
pub category: LearningCategory,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrictionKind {
|
||||
Retry,
|
||||
Timeout,
|
||||
WrongApproach,
|
||||
ToolFailure,
|
||||
Ambiguity,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FrictionPoint {
|
||||
pub kind: FrictionKind,
|
||||
pub description: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub stage_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum OpenItemKind {
|
||||
TechDebt,
|
||||
FollowUp,
|
||||
Investigation,
|
||||
TestGap,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OpenItem {
|
||||
pub kind: OpenItemKind,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StageRetro {
|
||||
pub stage_id: String,
|
||||
pub stage_label: String,
|
||||
pub status: String,
|
||||
pub duration_ms: u64,
|
||||
pub retries: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notes: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failure_reason: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub files_touched: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AggregateStats {
|
||||
pub total_duration_ms: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub total_cost: Option<f64>,
|
||||
pub total_retries: u32,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub files_touched: Vec<String>,
|
||||
pub stages_completed: usize,
|
||||
pub stages_failed: usize,
|
||||
}
|
||||
|
||||
/// Agent-generated qualitative narrative fields.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RetroNarrative {
|
||||
pub smoothness: SmoothnessRating,
|
||||
pub intent: String,
|
||||
pub outcome: String,
|
||||
#[serde(default)]
|
||||
pub learnings: Vec<Learning>,
|
||||
#[serde(default)]
|
||||
pub friction_points: Vec<FrictionPoint>,
|
||||
#[serde(default)]
|
||||
pub open_items: Vec<OpenItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Retro {
|
||||
pub run_id: String,
|
||||
pub pipeline_name: String,
|
||||
pub goal: String,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub smoothness: Option<SmoothnessRating>,
|
||||
pub stages: Vec<StageRetro>,
|
||||
pub stats: AggregateStats,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub intent: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub outcome: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub learnings: Option<Vec<Learning>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub friction_points: Option<Vec<FrictionPoint>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub open_items: Option<Vec<OpenItem>>,
|
||||
}
|
||||
|
||||
impl Retro {
|
||||
/// Merge agent-generated narrative into this retro.
|
||||
pub fn apply_narrative(&mut self, narrative: RetroNarrative) {
|
||||
self.smoothness = Some(narrative.smoothness);
|
||||
self.intent = Some(narrative.intent);
|
||||
self.outcome = Some(narrative.outcome);
|
||||
self.learnings = if narrative.learnings.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(narrative.learnings)
|
||||
};
|
||||
self.friction_points = if narrative.friction_points.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(narrative.friction_points)
|
||||
};
|
||||
self.open_items = if narrative.open_items.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(narrative.open_items)
|
||||
};
|
||||
}
|
||||
|
||||
/// Save the retro as JSON to `logs_root/retro.json`.
|
||||
pub fn save(&self, logs_root: &Path) -> Result<()> {
|
||||
let json = serde_json::to_string_pretty(self)
|
||||
.map_err(|e| ArcError::Checkpoint(format!("retro serialize failed: {e}")))?;
|
||||
std::fs::write(logs_root.join("retro.json"), json)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load a retro from `logs_root/retro.json`.
|
||||
pub fn load(logs_root: &Path) -> Result<Self> {
|
||||
let data = std::fs::read_to_string(logs_root.join("retro.json"))?;
|
||||
let retro: Self = serde_json::from_str(&data)
|
||||
.map_err(|e| ArcError::Checkpoint(format!("retro deserialize failed: {e}")))?;
|
||||
Ok(retro)
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract stage durations from `progress.ndjson` by reading `StageCompleted` events.
|
||||
pub fn extract_stage_durations(logs_root: &Path) -> HashMap<String, u64> {
|
||||
let mut durations = HashMap::new();
|
||||
let ndjson_path = logs_root.join("progress.ndjson");
|
||||
let Ok(data) = std::fs::read_to_string(&ndjson_path) else {
|
||||
return durations;
|
||||
};
|
||||
for line in data.lines() {
|
||||
let Ok(envelope) = serde_json::from_str::<serde_json::Value>(line) else {
|
||||
continue;
|
||||
};
|
||||
let Some(event_value) = envelope.get("event") else {
|
||||
continue;
|
||||
};
|
||||
let Ok(event) = serde_json::from_value::<PipelineEvent>(event_value.clone()) else {
|
||||
continue;
|
||||
};
|
||||
if let PipelineEvent::StageCompleted {
|
||||
name, duration_ms, ..
|
||||
} = event
|
||||
{
|
||||
durations.insert(name, duration_ms);
|
||||
}
|
||||
}
|
||||
durations
|
||||
}
|
||||
|
||||
/// Build a `Retro` from checkpoint data and run metadata. All qualitative
|
||||
/// fields (`smoothness`, `intent`, `outcome`, etc.) are left as `None` for
|
||||
/// the retro agent to fill in.
|
||||
pub fn derive_retro(
|
||||
run_id: &str,
|
||||
pipeline_name: &str,
|
||||
goal: &str,
|
||||
checkpoint: &Checkpoint,
|
||||
pipeline_failed: bool,
|
||||
_pipeline_error: Option<&str>,
|
||||
duration_ms: u64,
|
||||
stage_durations: &HashMap<String, u64>,
|
||||
) -> Retro {
|
||||
let mut stages = Vec::new();
|
||||
let mut all_files: Vec<String> = Vec::new();
|
||||
let mut total_cost: Option<f64> = None;
|
||||
let mut total_retries: u32 = 0;
|
||||
let mut stages_completed: usize = 0;
|
||||
let mut stages_failed: usize = 0;
|
||||
|
||||
for node_id in &checkpoint.completed_nodes {
|
||||
let outcome = checkpoint.node_outcomes.get(node_id);
|
||||
// node_retries stores attempts_used (1-indexed), convert to retry count
|
||||
let retries = checkpoint
|
||||
.node_retries
|
||||
.get(node_id)
|
||||
.copied()
|
||||
.unwrap_or(1)
|
||||
.saturating_sub(1);
|
||||
total_retries += retries;
|
||||
|
||||
let status = outcome
|
||||
.map(|o| o.status.to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
match outcome.map(|o| &o.status) {
|
||||
Some(StageStatus::Success | StageStatus::PartialSuccess) => stages_completed += 1,
|
||||
Some(StageStatus::Fail) => stages_failed += 1,
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let cost = outcome.and_then(|o| o.usage.as_ref()).and_then(|u| u.cost);
|
||||
if let Some(c) = cost {
|
||||
*total_cost.get_or_insert(0.0) += c;
|
||||
}
|
||||
|
||||
let files = outcome
|
||||
.map(|o| o.files_touched.clone())
|
||||
.unwrap_or_default();
|
||||
all_files.extend(files.iter().cloned());
|
||||
|
||||
stages.push(StageRetro {
|
||||
stage_id: node_id.clone(),
|
||||
stage_label: node_id.clone(),
|
||||
status,
|
||||
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
|
||||
retries,
|
||||
cost,
|
||||
notes: outcome.and_then(|o| o.notes.clone()),
|
||||
failure_reason: outcome.and_then(|o| o.failure_reason.clone()),
|
||||
files_touched: files,
|
||||
});
|
||||
}
|
||||
|
||||
// If pipeline failed with an error not captured in stages, record it
|
||||
if pipeline_failed && stages_failed == 0 {
|
||||
stages_failed = 1;
|
||||
}
|
||||
|
||||
all_files.sort();
|
||||
all_files.dedup();
|
||||
|
||||
let stats = AggregateStats {
|
||||
total_duration_ms: duration_ms,
|
||||
total_cost,
|
||||
total_retries,
|
||||
files_touched: all_files,
|
||||
stages_completed,
|
||||
stages_failed,
|
||||
};
|
||||
|
||||
Retro {
|
||||
run_id: run_id.to_string(),
|
||||
pipeline_name: pipeline_name.to_string(),
|
||||
goal: goal.to_string(),
|
||||
timestamp: Utc::now(),
|
||||
smoothness: None,
|
||||
stages,
|
||||
stats,
|
||||
intent: None,
|
||||
outcome: None,
|
||||
learnings: None,
|
||||
friction_points: None,
|
||||
open_items: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::outcome::Outcome;
|
||||
|
||||
fn make_checkpoint_with_stages() -> Checkpoint {
|
||||
let mut node_outcomes = HashMap::new();
|
||||
let mut outcome_a = Outcome::success();
|
||||
outcome_a.notes = Some("Planned the approach".to_string());
|
||||
outcome_a.files_touched = vec!["src/main.rs".to_string()];
|
||||
outcome_a.usage = Some(crate::outcome::StageUsage {
|
||||
model: "claude-opus-4-6".to_string(),
|
||||
input_tokens: 1000,
|
||||
output_tokens: 500,
|
||||
cache_read_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cost: Some(0.05),
|
||||
});
|
||||
node_outcomes.insert("plan".to_string(), outcome_a);
|
||||
|
||||
let mut outcome_b = Outcome::success();
|
||||
outcome_b.files_touched = vec!["src/main.rs".to_string(), "src/lib.rs".to_string()];
|
||||
outcome_b.usage = Some(crate::outcome::StageUsage {
|
||||
model: "claude-opus-4-6".to_string(),
|
||||
input_tokens: 2000,
|
||||
output_tokens: 1000,
|
||||
cache_read_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cost: Some(0.10),
|
||||
});
|
||||
node_outcomes.insert("code".to_string(), outcome_b);
|
||||
|
||||
let mut node_retries = HashMap::new();
|
||||
// 2 attempts_used = 1 actual retry
|
||||
node_retries.insert("code".to_string(), 2u32);
|
||||
|
||||
Checkpoint {
|
||||
timestamp: Utc::now(),
|
||||
current_node: "code".to_string(),
|
||||
completed_nodes: vec!["plan".to_string(), "code".to_string()],
|
||||
node_retries,
|
||||
context_values: HashMap::new(),
|
||||
logs: Vec::new(),
|
||||
node_outcomes,
|
||||
next_node_id: None,
|
||||
git_commit_sha: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_retro_builds_stages_from_checkpoint() {
|
||||
let cp = make_checkpoint_with_stages();
|
||||
let durations: HashMap<String, u64> =
|
||||
[("plan".to_string(), 5000), ("code".to_string(), 15000)]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let retro = derive_retro("run-1", "my_pipeline", "Fix the bug", &cp, false, None, 20000, &durations);
|
||||
|
||||
assert_eq!(retro.run_id, "run-1");
|
||||
assert_eq!(retro.pipeline_name, "my_pipeline");
|
||||
assert_eq!(retro.goal, "Fix the bug");
|
||||
assert_eq!(retro.stages.len(), 2);
|
||||
assert_eq!(retro.stages[0].stage_id, "plan");
|
||||
assert_eq!(retro.stages[0].duration_ms, 5000);
|
||||
assert_eq!(retro.stages[0].retries, 0);
|
||||
assert_eq!(retro.stages[1].stage_id, "code");
|
||||
assert_eq!(retro.stages[1].duration_ms, 15000);
|
||||
assert_eq!(retro.stages[1].retries, 1);
|
||||
assert_eq!(retro.stats.total_duration_ms, 20000);
|
||||
assert_eq!(retro.stats.total_retries, 1);
|
||||
assert_eq!(retro.stats.stages_completed, 2);
|
||||
assert_eq!(retro.stats.stages_failed, 0);
|
||||
assert!((retro.stats.total_cost.unwrap() - 0.15).abs() < f64::EPSILON);
|
||||
assert_eq!(retro.stats.files_touched, vec!["src/lib.rs", "src/main.rs"]);
|
||||
assert!(retro.smoothness.is_none());
|
||||
assert!(retro.intent.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_retro_handles_failed_pipeline() {
|
||||
let cp = Checkpoint {
|
||||
timestamp: Utc::now(),
|
||||
current_node: "start".to_string(),
|
||||
completed_nodes: vec!["start".to_string()],
|
||||
node_retries: HashMap::new(),
|
||||
context_values: HashMap::new(),
|
||||
logs: Vec::new(),
|
||||
node_outcomes: {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("start".to_string(), Outcome::success());
|
||||
m
|
||||
},
|
||||
next_node_id: None,
|
||||
git_commit_sha: None,
|
||||
};
|
||||
|
||||
let retro = derive_retro(
|
||||
"run-2", "pipe", "goal", &cp, true,
|
||||
Some("boom"), 5000, &HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(retro.stats.stages_failed, 1);
|
||||
assert_eq!(retro.stats.stages_completed, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_narrative_merges_fields() {
|
||||
let cp = make_checkpoint_with_stages();
|
||||
let mut retro = derive_retro("r1", "p", "g", &cp, false, None, 1000, &HashMap::new());
|
||||
|
||||
let narrative = RetroNarrative {
|
||||
smoothness: SmoothnessRating::Smooth,
|
||||
intent: "Fix authentication bug".to_string(),
|
||||
outcome: "Successfully fixed the login flow".to_string(),
|
||||
learnings: vec![Learning {
|
||||
category: LearningCategory::Code,
|
||||
text: "Token refresh logic was in the wrong module".to_string(),
|
||||
}],
|
||||
friction_points: vec![],
|
||||
open_items: vec![OpenItem {
|
||||
kind: OpenItemKind::TestGap,
|
||||
description: "No integration test for token refresh".to_string(),
|
||||
}],
|
||||
};
|
||||
|
||||
retro.apply_narrative(narrative);
|
||||
|
||||
assert_eq!(retro.smoothness, Some(SmoothnessRating::Smooth));
|
||||
assert_eq!(retro.intent.as_deref(), Some("Fix authentication bug"));
|
||||
assert_eq!(retro.outcome.as_deref(), Some("Successfully fixed the login flow"));
|
||||
assert_eq!(retro.learnings.as_ref().unwrap().len(), 1);
|
||||
assert!(retro.friction_points.is_none()); // empty vec -> None
|
||||
assert_eq!(retro.open_items.as_ref().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_and_load_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cp = make_checkpoint_with_stages();
|
||||
let mut retro = derive_retro("r1", "pipe", "goal", &cp, false, None, 1000, &HashMap::new());
|
||||
retro.smoothness = Some(SmoothnessRating::Bumpy);
|
||||
retro.intent = Some("Test intent".to_string());
|
||||
|
||||
retro.save(dir.path()).unwrap();
|
||||
let loaded = Retro::load(dir.path()).unwrap();
|
||||
|
||||
assert_eq!(loaded.run_id, "r1");
|
||||
assert_eq!(loaded.smoothness, Some(SmoothnessRating::Bumpy));
|
||||
assert_eq!(loaded.intent.as_deref(), Some("Test intent"));
|
||||
assert_eq!(loaded.stages.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_nonexistent_retro() {
|
||||
let result = Retro::load(Path::new("/nonexistent"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smoothness_rating_serde_roundtrip() {
|
||||
let json = serde_json::to_string(&SmoothnessRating::Effortless).unwrap();
|
||||
assert_eq!(json, "\"effortless\"");
|
||||
let parsed: SmoothnessRating = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed, SmoothnessRating::Effortless);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retro_narrative_serde() {
|
||||
let narrative = RetroNarrative {
|
||||
smoothness: SmoothnessRating::Failed,
|
||||
intent: "Deploy feature".to_string(),
|
||||
outcome: "Build failed".to_string(),
|
||||
learnings: vec![],
|
||||
friction_points: vec![FrictionPoint {
|
||||
kind: FrictionKind::ToolFailure,
|
||||
description: "Compiler error".to_string(),
|
||||
stage_id: Some("build".to_string()),
|
||||
}],
|
||||
open_items: vec![],
|
||||
};
|
||||
let json = serde_json::to_string(&narrative).unwrap();
|
||||
let parsed: RetroNarrative = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.smoothness, SmoothnessRating::Failed);
|
||||
assert_eq!(parsed.friction_points.len(), 1);
|
||||
assert_eq!(parsed.friction_points[0].kind, FrictionKind::ToolFailure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_stage_durations_from_ndjson() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let ndjson = dir.path().join("progress.ndjson");
|
||||
|
||||
let event1 = serde_json::json!({
|
||||
"timestamp": "2025-01-01T00:00:00.000Z",
|
||||
"run_id": "r1",
|
||||
"event": {
|
||||
"StageCompleted": {
|
||||
"name": "plan",
|
||||
"index": 0,
|
||||
"duration_ms": 5000,
|
||||
"status": "success",
|
||||
"preferred_label": null,
|
||||
"suggested_next_ids": [],
|
||||
"usage": null,
|
||||
"failure_reason": null,
|
||||
"notes": null,
|
||||
"files_touched": [],
|
||||
"attempt": 1,
|
||||
"max_attempts": 1,
|
||||
"failure_class": null
|
||||
}
|
||||
}
|
||||
});
|
||||
let event2 = serde_json::json!({
|
||||
"timestamp": "2025-01-01T00:00:05.000Z",
|
||||
"run_id": "r1",
|
||||
"event": {
|
||||
"StageCompleted": {
|
||||
"name": "code",
|
||||
"index": 1,
|
||||
"duration_ms": 15000,
|
||||
"status": "success",
|
||||
"preferred_label": null,
|
||||
"suggested_next_ids": [],
|
||||
"usage": null,
|
||||
"failure_reason": null,
|
||||
"notes": null,
|
||||
"files_touched": [],
|
||||
"attempt": 1,
|
||||
"max_attempts": 1,
|
||||
"failure_class": null
|
||||
}
|
||||
}
|
||||
});
|
||||
let content = format!(
|
||||
"{}\n{}\n",
|
||||
serde_json::to_string(&event1).unwrap(),
|
||||
serde_json::to_string(&event2).unwrap()
|
||||
);
|
||||
std::fs::write(&ndjson, content).unwrap();
|
||||
|
||||
let durations = extract_stage_durations(dir.path());
|
||||
assert_eq!(durations.get("plan"), Some(&5000));
|
||||
assert_eq!(durations.get("code"), Some(&15000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_stage_durations_missing_file() {
|
||||
let durations = extract_stage_durations(Path::new("/nonexistent"));
|
||||
assert!(durations.is_empty());
|
||||
}
|
||||
}
|
||||
288
crates/arc-workflows/src/retro_agent.rs
Normal file
288
crates/arc-workflows/src/retro_agent.rs
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use arc_agent::{
|
||||
AnthropicProfile, ExecutionEnvironment, GeminiProfile, OpenAiProfile, ProviderProfile,
|
||||
Session, SessionConfig,
|
||||
};
|
||||
use arc_llm::client::Client;
|
||||
use arc_llm::provider::Provider;
|
||||
use arc_llm::types::ToolDefinition;
|
||||
|
||||
use crate::retro::RetroNarrative;
|
||||
|
||||
const RETRO_SYSTEM_PROMPT: &str = r#"You are a pipeline retrospective analyst. Your job is to analyze a completed pipeline run and generate a structured retrospective.
|
||||
|
||||
You have access to the pipeline's data files:
|
||||
- `progress.ndjson` — the full event stream (stage starts/completions, agent tool calls, errors, retries)
|
||||
- `checkpoint.json` — final execution state with node outcomes
|
||||
- `manifest.json` — run metadata (if available)
|
||||
|
||||
## Your task
|
||||
|
||||
1. **Explore the data** using grep and read tools to understand what happened:
|
||||
- Look for failures, retries, and error messages
|
||||
- Check agent tool call patterns for wrong approaches or pivots
|
||||
- Note which stages took longest or had issues
|
||||
- Look for patterns indicating friction (repeated similar tool calls, error recovery)
|
||||
|
||||
2. **Call the `submit_retro` tool** with your structured analysis.
|
||||
|
||||
## Smoothness grading guidelines
|
||||
|
||||
Grade the run on a 5-point scale:
|
||||
|
||||
- **effortless** — Pipeline achieved its goal on the first try with no retries, no wrong approaches. Agent moved efficiently from start to finish.
|
||||
- **smooth** — Goal achieved with minor hiccups (1-2 retries or a brief wrong approach quickly corrected). No human intervention needed. Overall clean execution.
|
||||
- **bumpy** — Goal achieved but with notable friction: multiple retries, at least one significant wrong approach, or substantial time spent on dead ends.
|
||||
- **struggled** — Goal achieved only with difficulty: many retries, major approach changes, human intervention, or partial failures requiring recovery.
|
||||
- **failed** — Pipeline did not achieve its stated goal. May have completed some stages but the overall intent was not fulfilled.
|
||||
|
||||
Consider the full context: not just stage pass/fail, but the quality of the journey visible in the agent events (tool call patterns, error recovery, approach pivots).
|
||||
|
||||
## Guidelines for qualitative fields
|
||||
|
||||
- **intent**: What was the pipeline trying to accomplish? Summarize the goal in a sentence.
|
||||
- **outcome**: What actually happened? Did it succeed? What was produced?
|
||||
- **learnings**: What was discovered about the repo, code, workflow, or tools?
|
||||
- **friction_points**: Where did things get stuck? What caused slowdowns?
|
||||
- **open_items**: What follow-up work, tech debt, or gaps were identified?
|
||||
|
||||
Be specific and concise. Reference actual stage names, file paths, and error messages where relevant."#;
|
||||
|
||||
const SUBMIT_RETRO_SCHEMA: &str = r#"{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"smoothness": {
|
||||
"type": "string",
|
||||
"enum": ["effortless", "smooth", "bumpy", "struggled", "failed"],
|
||||
"description": "Overall smoothness rating for the pipeline run"
|
||||
},
|
||||
"intent": {
|
||||
"type": "string",
|
||||
"description": "What was the pipeline trying to accomplish?"
|
||||
},
|
||||
"outcome": {
|
||||
"type": "string",
|
||||
"description": "What actually happened? Did it succeed?"
|
||||
},
|
||||
"learnings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"category": { "type": "string", "enum": ["repo", "code", "workflow", "tool"] },
|
||||
"text": { "type": "string" }
|
||||
},
|
||||
"required": ["category", "text"]
|
||||
},
|
||||
"description": "What was discovered during the run?"
|
||||
},
|
||||
"friction_points": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["retry", "timeout", "wrong_approach", "tool_failure", "ambiguity"] },
|
||||
"description": { "type": "string" },
|
||||
"stage_id": { "type": "string" }
|
||||
},
|
||||
"required": ["kind", "description"]
|
||||
},
|
||||
"description": "Where did things get stuck?"
|
||||
},
|
||||
"open_items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["tech_debt", "follow_up", "investigation", "test_gap"] },
|
||||
"description": { "type": "string" }
|
||||
},
|
||||
"required": ["kind", "description"]
|
||||
},
|
||||
"description": "Follow-up work or gaps identified"
|
||||
}
|
||||
},
|
||||
"required": ["smoothness", "intent", "outcome"]
|
||||
}"#;
|
||||
|
||||
/// Run a retro agent session that analyzes pipeline run data and produces
|
||||
/// a structured narrative. The agent explores `progress.ndjson` and other
|
||||
/// files via tool access, then calls `submit_retro` with its analysis.
|
||||
pub async fn run_retro_agent(
|
||||
execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
logs_root: &Path,
|
||||
llm_client: &Client,
|
||||
provider: Provider,
|
||||
model: &str,
|
||||
) -> anyhow::Result<RetroNarrative> {
|
||||
// Upload data files into execution env (needed for Daytona; no-op effect for local
|
||||
// since the agent can also read from the original paths via tools).
|
||||
let retro_data_dir = "/tmp/retro_data";
|
||||
upload_data_files(execution_env, logs_root, retro_data_dir).await?;
|
||||
|
||||
// Build provider profile with the submit_retro tool
|
||||
let captured: Arc<Mutex<Option<RetroNarrative>>> = Arc::new(Mutex::new(None));
|
||||
let captured_clone = Arc::clone(&captured);
|
||||
|
||||
let mut profile = build_profile(provider, model);
|
||||
|
||||
// Register submit_retro tool
|
||||
let submit_tool = arc_agent::tool_registry::RegisteredTool {
|
||||
definition: ToolDefinition {
|
||||
name: "submit_retro".to_string(),
|
||||
description: "Submit the structured retrospective analysis. Call this once you have analyzed the pipeline run data.".to_string(),
|
||||
parameters: serde_json::from_str(SUBMIT_RETRO_SCHEMA)
|
||||
.expect("submit_retro schema should be valid JSON"),
|
||||
},
|
||||
executor: Arc::new(move |args, _ctx| {
|
||||
let captured = Arc::clone(&captured_clone);
|
||||
Box::pin(async move {
|
||||
let narrative: RetroNarrative = serde_json::from_value(args)
|
||||
.map_err(|e| format!("Invalid retro submission: {e}"))?;
|
||||
*captured.lock().unwrap() = Some(narrative);
|
||||
Ok("Retrospective submitted successfully.".to_string())
|
||||
})
|
||||
}),
|
||||
};
|
||||
profile.tool_registry_mut().register(submit_tool);
|
||||
|
||||
let profile: Arc<dyn ProviderProfile> = Arc::from(profile);
|
||||
|
||||
let config = SessionConfig {
|
||||
max_tool_rounds_per_input: 10,
|
||||
// Disable features not needed for retro analysis
|
||||
enable_context_compaction: false,
|
||||
skill_dirs: Some(vec![]),
|
||||
user_instructions: Some(RETRO_SYSTEM_PROMPT.to_string()),
|
||||
..SessionConfig::default()
|
||||
};
|
||||
|
||||
let mut session = Session::new(
|
||||
llm_client.clone(),
|
||||
profile,
|
||||
Arc::clone(execution_env),
|
||||
config,
|
||||
);
|
||||
|
||||
session.initialize().await;
|
||||
|
||||
let prompt = format!(
|
||||
"Analyze the pipeline run data at `{retro_data_dir}/` and generate a retrospective. \
|
||||
The key file is `{retro_data_dir}/progress.ndjson` which contains the full event stream. \
|
||||
Also check `{retro_data_dir}/checkpoint.json` for stage outcomes. \
|
||||
Use grep to search for interesting signals (failures, retries, errors, approach changes) \
|
||||
rather than reading the entire file. When done, call the `submit_retro` tool with your analysis."
|
||||
);
|
||||
|
||||
session.process_input(&prompt).await.map_err(|e| {
|
||||
anyhow::anyhow!("Retro agent session failed: {e}")
|
||||
})?;
|
||||
|
||||
// Extract the captured narrative
|
||||
let narrative = captured
|
||||
.lock()
|
||||
.unwrap()
|
||||
.take()
|
||||
.ok_or_else(|| anyhow::anyhow!("Retro agent did not call submit_retro"))?;
|
||||
|
||||
Ok(narrative)
|
||||
}
|
||||
|
||||
fn build_profile(provider: Provider, model: &str) -> Box<dyn ProviderProfile> {
|
||||
match provider {
|
||||
Provider::OpenAi => Box::new(OpenAiProfile::new(model)),
|
||||
Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception => {
|
||||
Box::new(OpenAiProfile::new(model).with_provider(provider))
|
||||
}
|
||||
Provider::Gemini => Box::new(GeminiProfile::new(model)),
|
||||
Provider::Anthropic => Box::new(AnthropicProfile::new(model)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_data_files(
|
||||
execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
logs_root: &Path,
|
||||
target_dir: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
// Create target directory
|
||||
execution_env
|
||||
.exec_command(&format!("mkdir -p {target_dir}"), 10_000, None, None, None)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create retro data dir: {e}"))?;
|
||||
|
||||
let files = ["progress.ndjson", "checkpoint.json", "manifest.json"];
|
||||
for filename in &files {
|
||||
let source = logs_root.join(filename);
|
||||
if source.exists() {
|
||||
let content = std::fs::read_to_string(&source)?;
|
||||
execution_env
|
||||
.write_file(&format!("{target_dir}/{filename}"), &content)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to upload {filename}: {e}"))?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn submit_retro_schema_is_valid_json() {
|
||||
let schema: serde_json::Value = serde_json::from_str(SUBMIT_RETRO_SCHEMA).unwrap();
|
||||
assert_eq!(schema["type"], "object");
|
||||
assert!(schema["properties"]["smoothness"].is_object());
|
||||
assert!(schema["properties"]["intent"].is_object());
|
||||
assert!(schema["properties"]["outcome"].is_object());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retro_narrative_parses_from_submit_retro_args() {
|
||||
let args = serde_json::json!({
|
||||
"smoothness": "smooth",
|
||||
"intent": "Fix the login bug",
|
||||
"outcome": "Successfully fixed the authentication flow",
|
||||
"learnings": [
|
||||
{ "category": "code", "text": "Token refresh was in wrong module" }
|
||||
],
|
||||
"friction_points": [
|
||||
{ "kind": "retry", "description": "First attempt had wrong import", "stage_id": "code" }
|
||||
],
|
||||
"open_items": [
|
||||
{ "kind": "test_gap", "description": "No integration test for token refresh" }
|
||||
]
|
||||
});
|
||||
|
||||
let narrative: RetroNarrative = serde_json::from_value(args).unwrap();
|
||||
assert_eq!(
|
||||
narrative.smoothness,
|
||||
crate::retro::SmoothnessRating::Smooth
|
||||
);
|
||||
assert_eq!(narrative.intent, "Fix the login bug");
|
||||
assert_eq!(narrative.learnings.len(), 1);
|
||||
assert_eq!(narrative.friction_points.len(), 1);
|
||||
assert_eq!(narrative.open_items.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retro_narrative_parses_minimal_args() {
|
||||
let args = serde_json::json!({
|
||||
"smoothness": "effortless",
|
||||
"intent": "Deploy feature",
|
||||
"outcome": "Deployed successfully"
|
||||
});
|
||||
|
||||
let narrative: RetroNarrative = serde_json::from_value(args).unwrap();
|
||||
assert_eq!(
|
||||
narrative.smoothness,
|
||||
crate::retro::SmoothnessRating::Effortless
|
||||
);
|
||||
assert!(narrative.learnings.is_empty());
|
||||
assert!(narrative.friction_points.is_empty());
|
||||
assert!(narrative.open_items.is_empty());
|
||||
}
|
||||
}
|
||||
12
test/retro-e2e.dot
Normal file
12
test/retro-e2e.dot
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
digraph RetroE2E {
|
||||
graph [goal="Verify retro generation works end-to-end"]
|
||||
rankdir=LR
|
||||
|
||||
start [shape=Mdiamond, label="Start"]
|
||||
exit [shape=Msquare, label="Exit"]
|
||||
|
||||
check [label="Check", shape=parallelogram, script="echo 'Hello from Daytona sandbox' && ls -la"]
|
||||
summarize [label="Summarize", prompt="Briefly summarize what the check stage did. Keep it under 2 sentences."]
|
||||
|
||||
start -> check -> summarize -> exit
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue