diff --git a/apps/fabro-web/app/routes/run-detail.test.ts b/apps/fabro-web/app/routes/run-detail.test.ts index 5c54733d9..76ccc0a0c 100644 --- a/apps/fabro-web/app/routes/run-detail.test.ts +++ b/apps/fabro-web/app/routes/run-detail.test.ts @@ -86,6 +86,7 @@ mock.module("../lib/queries", () => ({ isValidating: false, mutate: mock(() => Promise.resolve(null)), }), + useProviders: () => ({ data: null }), })); mock.module("../lib/run-events", () => ({ diff --git a/apps/fabro-web/app/routes/run-detail.tsx b/apps/fabro-web/app/routes/run-detail.tsx index 4ebf6ed03..c43846da6 100644 --- a/apps/fabro-web/app/routes/run-detail.tsx +++ b/apps/fabro-web/app/routes/run-detail.tsx @@ -1,5 +1,6 @@ import { useCallback, + useMemo, useRef, useState, type CSSProperties, @@ -34,7 +35,7 @@ import { } from "../lib/mutations"; import { useRunEvents } from "../lib/run-events"; import { useRunToasts } from "../hooks/use-run-toasts"; -import { useRun, useRunQuestions, useRunState } from "../lib/queries"; +import { useProviders, useRun, useRunQuestions, useRunState } from "../lib/queries"; import { canApprove, deleteErrorMessage, @@ -50,6 +51,8 @@ import { RunDetailAskFabroShell, RunDetailDockedControls, } from "./run-detail/docked-controls"; +import { buildRunFailureView } from "./run-detail/failure"; +import { RunFailureBanner } from "./run-detail/failure-banner"; import { RunDetailHeader } from "./run-detail/header"; import { createLifecycleToastState, @@ -148,6 +151,12 @@ export default function RunDetail({ params }: { params: { id: string } }) { useRunEvents(params.id); useRunToasts(params.id); + const providersQuery = useProviders(); + const providerNames = useMemo( + () => new Map((providersQuery.data?.data ?? []).map((p) => [p.id, p.display_name || p.id])), + [providersQuery.data], + ); + const failure = buildRunFailureView(runStateQuery.data, providerNames); const handleLifecycleMutationResult = useCallback( ( @@ -382,6 +391,8 @@ export default function RunDetail({ params }: { params: { id: string } }) { }} /> + {failure ? : null} + {retryDialogOpen && ( +
+ + +
+
+
+ {failure.provider ? "Provider message" : "Message"} +
+
+ {failure.message} +
+
+ {failure.output ? ( +
+
+ Output +
+
+
+                  {failure.output}
+                
+
+
+ ) : null} +
+
+ + ); +} diff --git a/apps/fabro-web/app/routes/run-detail/failure.test.ts b/apps/fabro-web/app/routes/run-detail/failure.test.ts new file mode 100644 index 000000000..5ed2decb9 --- /dev/null +++ b/apps/fabro-web/app/routes/run-detail/failure.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, test } from "bun:test"; +import type { RunProjection } from "@qltysh/fabro-api-client"; + +import { buildRunFailureView } from "./failure"; + +type State = Pick; + +const NAMES = new Map([ + ["anthropic", "Anthropic"], + ["openai", "OpenAI"], +]); + +const CREDIT = + "model request failed (invalid_request): provider anthropic Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits. [provider anthropic, status 400, code invalid_request_error]"; + +function stages(...entries: Array<[string, string, string]>): State["stages"] { + return Object.fromEntries( + entries.map(([key, state, started_at]) => [key, { state, started_at } as State["stages"][string]]), + ); +} + +function failedRun( + detail: Record, + { reason = "workflow_error", kind = "failed", runStages = {} as State["stages"] } = {}, +): State { + return { + status: (kind === "failed" ? { kind, reason } : { kind }) as State["status"], + conclusion: { + failure: { reason, detail: { category: "deterministic", ...detail } }, + } as State["conclusion"], + stages: runStages, + }; +} + +const view = (state: State) => buildRunFailureView(state, NAMES); + +describe("buildRunFailureView", () => { + test("names the step the run stopped at and the provider's real message", () => { + const failure = view( + failedRun({ message: CREDIT }, { + runStages: stages(["pick@1", "succeeded", "2026-10-10T18:05:42.783Z"], ["spec@1", "failed", "2026-10-10T18:41:53.513Z"]), + }), + ); + expect(failure).toMatchObject({ + stage: "spec", + headline: "Anthropic declined the request for billing reasons", + provider: "anthropic", + message: CREDIT, + }); + expect(failure?.nextStep).toBe("Add credit or raise the spend limit with Anthropic, then choose Retry from Actions."); + }); + + test("names an unknown provider by its id", () => { + const failure = view( + failedRun({ message: "model request failed (authentication): provider acme bad key [provider acme, status 401, code unauthorized]" }), + ); + expect(failure?.headline).toBe("acme rejected the API key"); + }); + + test("reads auth, rate limits, timeouts and outages from the request's own status and kind", () => { + const headline = (message: string) => view(failedRun({ message }))?.headline; + expect(headline("model request failed (authentication): provider anthropic invalid x-api-key [provider anthropic, status 401, code authentication_error]")) + .toBe("Anthropic rejected the API key"); + expect(headline("model request failed (rate_limit): provider openai Too many requests [provider openai, status 429, code rate_limit_exceeded]")) + .toBe("OpenAI rate-limited the request"); + expect(headline("model request failed (server): provider anthropic Overloaded [provider anthropic, status 529, code overloaded_error]")) + .toBe("Anthropic was unavailable"); + expect(headline("model request failed (timeout): provider openai request timed out")) + .toBe("The request to OpenAI timed out"); + }); + + test("falls back to the provider's status and code", () => { + const failure = view( + failedRun({ + message: "model request failed (invalid_request): provider openai Unsupported parameter: 'temperature' is not supported with this model. [provider openai, status 400, code unsupported_parameter]", + }), + ); + expect(failure?.headline).toBe("OpenAI returned 400 unsupported parameter"); + }); + + test("never reads a provider, status or billing problem out of a command's own words", () => { + const failure = view( + failedRun({ + message: "script exited with status 401", + exec_output_tail: { stdout: "test billing::invoice ... FAILED\nretry the provider list later\n", stderr: null }, + }), + ); + expect(failure).toMatchObject({ + headline: "The step's command exited with status 401", + provider: null, + nextStep: "Its output, under Details, usually says why.", + }); + expect(view(failedRun({ message: "Ran 512 tests; line 503 failed." }))?.headline).toBe("Ran 512 tests; line 503 failed."); + }); + + test("names no step when the run recovered from an earlier failed one", () => { + const failure = view( + failedRun({ message: "push rejected: remote branch moved" }, { + reason: "publish_failed", + runStages: stages( + ["toolchain@1", "failed", "2026-10-10T15:20:45Z"], + ["fixup@1", "succeeded", "2026-10-10T15:21:00Z"], + ["verify@1", "succeeded", "2026-10-10T15:25:00.500Z"], + ), + }), + ); + expect(failure?.stage).toBeNull(); + expect(failure?.headline).toBe("The steps finished, but publishing the result failed"); + }); + + test("shows nothing for a cancelled run, and says what happened for other endings", () => { + expect(view(failedRun({ message: "Run cancelled before worker launch completed" }, { reason: "cancelled" }))).toBeNull(); + expect(view(failedRun({ message: "denied at review" }, { reason: "approval_denied" }))?.headline).toBe("Approval was denied"); + expect(view(failedRun({ message: "budget of $5 reached" }, { reason: "budget_exhausted" }))?.headline).toBe("The run used up its budget"); + expect(view(failedRun({ message: "sandbox image not found" }, { reason: "sandbox_init_failed" }))?.headline).toBe("The run couldn't start"); + expect(view(failedRun({ message: "worker exited unexpectedly" }, { kind: "dead" }))?.headline).toBe("Worker exited unexpectedly"); + }); + + test("shows the end of a command's output, from whichever stream has it", () => { + const tail = (exec_output_tail: Record) => + view(failedRun({ message: "script exited with status 1", exec_output_tail }))?.output; + expect(tail({ stderr: "\n \n", stdout: "real error\n" })).toBe("real error"); + expect(tail({ stderr: "line one\r\nline two\r\n", stdout: null })).toBe("line one\nline two"); + expect(tail({ stdout: "10%\r50%\r100% done\n" })).toBe("100% done"); + const many = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`).join("\n"); + expect(tail({ stdout: many })?.split("\n")).toEqual(Array.from({ length: 12 }, (_, i) => `line ${i + 19}`)); + }); + + test("cuts a long fallback headline without splitting a character", () => { + const headline = view(failedRun({ message: `${"🔥".repeat(200)} and more` }))!.headline; + expect(Array.from(headline)).toHaveLength(111); + expect(headline.endsWith("🔥…")).toBe(true); + }); + + test("shows nothing for runs that didn't fail", () => { + expect(buildRunFailureView({ status: { kind: "succeeded" }, stages: {} } as State)).toBeNull(); + expect(buildRunFailureView(null)).toBeNull(); + }); +}); diff --git a/apps/fabro-web/app/routes/run-detail/failure.ts b/apps/fabro-web/app/routes/run-detail/failure.ts new file mode 100644 index 000000000..645e2f415 --- /dev/null +++ b/apps/fabro-web/app/routes/run-detail/failure.ts @@ -0,0 +1,179 @@ +import { + FailureCategory, + FailureReason, + type ExecOutputTail, + type RunProjection, +} from "@qltysh/fabro-api-client"; + +/** Lines of a failed command's output shown on the page. */ +const OUTPUT_LINES = 12; +/** Fallback headlines are cut to this many characters. */ +const HEADLINE_CHARS = 110; + +export interface RunFailureView { + /** The step the run stopped at (`spec`, `review@3`), when it stopped at one. */ + stage: string | null; + headline: string; + nextStep: string; + /** The recorded failure message, verbatim. */ + message: string; + /** Provider id, when the failure is a model request the provider refused. */ + provider: string | null; + /** Last OUTPUT_LINES of the failed command's stderr, else its stdout. */ + output: string | null; +} + +type FailureState = Pick; + +/** + * Reads the failure from run state (`conclusion.failure`); the run summary the + * status popover reads doesn't carry the message or the command's output. + */ +export function buildRunFailureView( + state: FailureState | null | undefined, + providerNames: ReadonlyMap = new Map(), +): RunFailureView | null { + const kind = state?.status?.kind; + if (!state || (kind !== "failed" && kind !== "dead")) return null; + const failure = state.conclusion?.failure; + if (!failure?.detail.message || failure.reason === FailureReason.CANCELLED) return null; + + const { message, category } = failure.detail; + const output = outputTail(failure.detail.exec_output_tail); + const request = parseModelRequest(message); + const name = (id: string) => providerNames.get(id) ?? id; + const advice = + reasonAdvice(failure.reason, category) ?? + (request ? requestAdvice(request, message, name(request.provider)) : stepAdvice(message, category, output)); + return { + stage: stoppedAt(state.stages), + ...advice, + message, + provider: request?.provider ?? null, + output, + }; +} + +type Advice = Pick; + +const RETRY = "choose Retry from Actions"; + +function reasonAdvice(reason: FailureReason, category: FailureCategory): Advice | null { + if (reason === FailureReason.APPROVAL_DENIED) { + return { headline: "Approval was denied", nextStep: `The run stops where it was denied. To run it again, ${RETRY}.` }; + } + if (reason === FailureReason.BUDGET_EXHAUSTED || category === FailureCategory.BUDGET_EXHAUSTED) { + return { headline: "The run used up its budget", nextStep: `Raise the run's budget, then ${RETRY}.` }; + } + if (reason === FailureReason.TERMINATED) { + return { headline: "The run was terminated", nextStep: `To run it again, ${RETRY}.` }; + } + if (reason === FailureReason.PUBLISH_FAILED) { + return { + headline: "The steps finished, but publishing the result failed", + nextStep: "The work is on the run branch. The message under Details says why publishing failed.", + }; + } + if ( + reason === FailureReason.LAUNCH_FAILED || + reason === FailureReason.BOOTSTRAP_FAILED || + reason === FailureReason.SANDBOX_INIT_FAILED + ) { + return { headline: "The run couldn't start", nextStep: "The message under Details says why." }; + } + return null; +} + +interface ModelRequest { + provider: string; + kind: string | null; + status: number | null; + code: string | null; +} + +/** + * A refused model request, as the server records it: + * `model request failed (): provider [provider , status , code ]`. + * Only this shape is read, so numbers and words in a command's output can't + * pass for a provider's status or name. + */ +function parseModelRequest(message: string): ModelRequest | null { + const lead = /^model request failed \(([a-z_]+)\): provider (\S+)/i.exec(message); + const tail = /\[provider ([^,\]]+), status (\d{3})(?:, code ([^\]]+))?\]\s*$/i.exec(message); + const provider = tail?.[1] ?? lead?.[2]; + if (!provider) return null; + return { + provider: provider.toLowerCase(), + kind: lead?.[1]?.toLowerCase() ?? null, + status: tail ? Number(tail[2]) : null, + code: tail?.[3] ?? null, + }; +} + +function requestAdvice({ kind, status, code }: ModelRequest, message: string, who: string): Advice { + if (/credit balance|insufficient[_ ]quota|payment required|billing|spend limit/i.test(message) || status === 402) { + return { + headline: `${who} declined the request for billing reasons`, + nextStep: `Add credit or raise the spend limit with ${who}, then ${RETRY}.`, + }; + } + if (kind === "authentication" || status === 401 || status === 403) { + return { headline: `${who} rejected the API key`, nextStep: `Check the key under Settings → Secrets, then ${RETRY}.` }; + } + if (kind === "rate_limit" || status === 429) { + return { headline: `${who} rate-limited the request`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." }; + } + if (kind === "timeout" || /timed out/i.test(message)) { + return { headline: `The request to ${who} timed out`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." }; + } + if ((status !== null && status >= 500) || /overloaded/i.test(message)) { + return { headline: `${who} was unavailable`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." }; + } + const what = [status, code?.replace(/_/g, " ")].filter(Boolean).join(" "); + return { + headline: what ? `${who} returned ${what}` : `${who} rejected the request`, + nextStep: `${who}'s message, under Details, says why.`, + }; +} + +function stepAdvice(message: string, category: FailureCategory, output: string | null): Advice { + const exit = /exited with (?:status|code) (\d+)/i.exec(message); + if (exit) { + return { + headline: `The step's command exited with status ${exit[1]}`, + nextStep: output ? "Its output, under Details, usually says why." : "Open the step's logs for its output.", + }; + } + if (category === FailureCategory.TRANSIENT_INFRA) { + return { headline: "The step hit a temporary infrastructure error", nextStep: "Choose Retry from Actions in a few minutes." }; + } + return { headline: firstSentence(message), nextStep: "Open the failed step's logs for details." }; +} + +/** + * The last step to start, if it failed. An earlier failure that a later step + * recovered from isn't where the run stopped, so it isn't named. + */ +function stoppedAt(stages: FailureState["stages"]): string | null { + let last: [string, number] | null = null; + for (const [key, stage] of Object.entries(stages)) { + const started = stage.started_at ? Date.parse(stage.started_at) : NaN; + if (!Number.isNaN(started) && (!last || started > last[1])) last = [key, started]; + } + if (!last || stages[last[0]]?.state !== "failed") return null; + return last[0].replace(/@1$/, ""); +} + +function outputTail(tail: ExecOutputTail | null | undefined): string | null { + const stream = [tail?.stderr, tail?.stdout].find((text) => text?.trim()); + if (!stream) return null; + // Carriage-return progress lines keep only what was drawn last. + const lines = stream.trimEnd().split(/\r?\n/).map((line) => line.slice(line.lastIndexOf("\r") + 1)); + return lines.slice(-OUTPUT_LINES).join("\n"); +} + +function firstSentence(message: string): string { + const sentence = message.split(/(?<=[.!?])\s+/)[0]!.trim().replace(/\s+/g, " "); + const chars = Array.from(sentence.charAt(0).toUpperCase() + sentence.slice(1)); + return chars.length > HEADLINE_CHARS ? `${chars.slice(0, HEADLINE_CHARS).join("").trimEnd()}…` : chars.join(""); +}