mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-11-02 07:12:34 +00:00
Say why a failed run stopped, above its tabs
The status popover shows the reason code ("Workflow error") and
lifecycle.error, which is empty when the failure lives in the run's
state, so a run stopped by the provider's billing refusal showed no
reason anywhere. The message is in conclusion.failure.detail, which the
run page already loads.
Show a quiet banner on failed and dead runs: the step the run stopped
at, a headline and the next thing to do, with the verbatim message and
the tail of a failed command's output in a closed <details>. The advice
comes from the failure's reason and category first (approval denied,
budget, publish, launch), then from the structured model request
message the server records (provider, kind, status, code), then from a
command's exit status. Command output is never searched for provider
or billing words. Provider names come from /providers, cancelled runs
get no banner, and a step that failed earlier but was recovered from
isn't named as the stop.
This commit is contained in:
James Robert Somers
parent
1906d7c9ea
commit
123e0944bd
5 changed files with 393 additions and 1 deletions
|
|
@ -86,6 +86,7 @@ mock.module("../lib/queries", () => ({
|
|||
isValidating: false,
|
||||
mutate: mock(() => Promise.resolve(null)),
|
||||
}),
|
||||
useProviders: () => ({ data: null }),
|
||||
}));
|
||||
|
||||
mock.module("../lib/run-events", () => ({
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import {
|
||||
useCallback,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
type CSSProperties,
|
||||
|
|
@ -34,7 +35,7 @@ import {
|
|||
} from "../lib/mutations";
|
||||
import { useRunEvents } from "../lib/run-events";
|
||||
import { useRunToasts } from "../hooks/use-run-toasts";
|
||||
import { useRun, useRunQuestions, useRunState } from "../lib/queries";
|
||||
import { useProviders, useRun, useRunQuestions, useRunState } from "../lib/queries";
|
||||
import {
|
||||
canApprove,
|
||||
deleteErrorMessage,
|
||||
|
|
@ -50,6 +51,8 @@ import {
|
|||
RunDetailAskFabroShell,
|
||||
RunDetailDockedControls,
|
||||
} from "./run-detail/docked-controls";
|
||||
import { buildRunFailureView } from "./run-detail/failure";
|
||||
import { RunFailureBanner } from "./run-detail/failure-banner";
|
||||
import { RunDetailHeader } from "./run-detail/header";
|
||||
import {
|
||||
createLifecycleToastState,
|
||||
|
|
@ -148,6 +151,12 @@ export default function RunDetail({ params }: { params: { id: string } }) {
|
|||
|
||||
useRunEvents(params.id);
|
||||
useRunToasts(params.id);
|
||||
const providersQuery = useProviders();
|
||||
const providerNames = useMemo(
|
||||
() => new Map((providersQuery.data?.data ?? []).map((p) => [p.id, p.display_name || p.id])),
|
||||
[providersQuery.data],
|
||||
);
|
||||
const failure = buildRunFailureView(runStateQuery.data, providerNames);
|
||||
|
||||
const handleLifecycleMutationResult = useCallback(
|
||||
(
|
||||
|
|
@ -382,6 +391,8 @@ export default function RunDetail({ params }: { params: { id: string } }) {
|
|||
}}
|
||||
/>
|
||||
|
||||
{failure ? <RunFailureBanner failure={failure} /> : null}
|
||||
|
||||
{retryDialogOpen && (
|
||||
<RetryRunDialog
|
||||
key={params.id}
|
||||
|
|
|
|||
62
apps/fabro-web/app/routes/run-detail/failure-banner.tsx
Normal file
62
apps/fabro-web/app/routes/run-detail/failure-banner.tsx
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { ChevronDownIcon, ExclamationTriangleIcon } from "@heroicons/react/20/solid";
|
||||
|
||||
import type { RunFailureView } from "./failure";
|
||||
|
||||
export function RunFailureBanner({ failure }: { failure: RunFailureView }) {
|
||||
return (
|
||||
<section aria-label="Why this run failed" className="mb-6">
|
||||
<details className="group rounded-lg bg-panel-alt outline-1 -outline-offset-1 outline-line">
|
||||
<summary className="flex cursor-pointer list-none items-start gap-3 rounded-lg px-4 py-3.5 transition-colors hover:bg-overlay focus-visible:outline-2 focus-visible:-outline-offset-1 focus-visible:outline-teal-500 [&::-webkit-details-marker]:hidden">
|
||||
<ExclamationTriangleIcon
|
||||
className="mt-0.5 size-4 shrink-0 text-coral"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block text-sm/5 font-medium text-fg">
|
||||
{failure.stage ? (
|
||||
<>
|
||||
Stopped at <span className="font-mono text-[0.8125rem]">{failure.stage}</span>
|
||||
<span className="mx-2 text-fg-muted" aria-hidden="true">
|
||||
·
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
{failure.headline}
|
||||
</span>
|
||||
<span className="mt-1 block text-sm/5 text-fg-3">{failure.nextStep}</span>
|
||||
</span>
|
||||
<span className="inline-flex shrink-0 items-center gap-1 pt-0.5 text-xs font-medium text-fg-muted transition-colors group-hover:text-fg-3">
|
||||
<span className="group-open:hidden">Details</span>
|
||||
<span className="hidden group-open:inline">Hide</span>
|
||||
<ChevronDownIcon
|
||||
className="size-3.5 shrink-0 transition-transform group-open:-rotate-180"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
</span>
|
||||
</summary>
|
||||
<dl className="space-y-3 px-4 pt-0.5 pb-4 pl-11">
|
||||
<div>
|
||||
<dt className="mb-1 font-mono text-[0.6875rem] tracking-wide text-fg-muted uppercase">
|
||||
{failure.provider ? "Provider message" : "Message"}
|
||||
</dt>
|
||||
<dd className="rounded-md bg-overlay px-3 py-2 font-mono text-xs/5 wrap-break-word whitespace-pre-wrap text-fg-3">
|
||||
{failure.message}
|
||||
</dd>
|
||||
</div>
|
||||
{failure.output ? (
|
||||
<div>
|
||||
<dt className="mb-1 font-mono text-[0.6875rem] tracking-wide text-fg-muted uppercase">
|
||||
Output
|
||||
</dt>
|
||||
<dd>
|
||||
<pre className="max-h-48 overflow-auto rounded-md bg-overlay px-3 py-2 font-mono text-xs/5 text-fg-3">
|
||||
{failure.output}
|
||||
</pre>
|
||||
</dd>
|
||||
</div>
|
||||
) : null}
|
||||
</dl>
|
||||
</details>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
139
apps/fabro-web/app/routes/run-detail/failure.test.ts
Normal file
139
apps/fabro-web/app/routes/run-detail/failure.test.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import type { RunProjection } from "@qltysh/fabro-api-client";
|
||||
|
||||
import { buildRunFailureView } from "./failure";
|
||||
|
||||
type State = Pick<RunProjection, "status" | "conclusion" | "stages">;
|
||||
|
||||
const NAMES = new Map([
|
||||
["anthropic", "Anthropic"],
|
||||
["openai", "OpenAI"],
|
||||
]);
|
||||
|
||||
const CREDIT =
|
||||
"model request failed (invalid_request): provider anthropic Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits. [provider anthropic, status 400, code invalid_request_error]";
|
||||
|
||||
function stages(...entries: Array<[string, string, string]>): State["stages"] {
|
||||
return Object.fromEntries(
|
||||
entries.map(([key, state, started_at]) => [key, { state, started_at } as State["stages"][string]]),
|
||||
);
|
||||
}
|
||||
|
||||
function failedRun(
|
||||
detail: Record<string, unknown>,
|
||||
{ reason = "workflow_error", kind = "failed", runStages = {} as State["stages"] } = {},
|
||||
): State {
|
||||
return {
|
||||
status: (kind === "failed" ? { kind, reason } : { kind }) as State["status"],
|
||||
conclusion: {
|
||||
failure: { reason, detail: { category: "deterministic", ...detail } },
|
||||
} as State["conclusion"],
|
||||
stages: runStages,
|
||||
};
|
||||
}
|
||||
|
||||
const view = (state: State) => buildRunFailureView(state, NAMES);
|
||||
|
||||
describe("buildRunFailureView", () => {
|
||||
test("names the step the run stopped at and the provider's real message", () => {
|
||||
const failure = view(
|
||||
failedRun({ message: CREDIT }, {
|
||||
runStages: stages(["pick@1", "succeeded", "2026-10-10T18:05:42.783Z"], ["spec@1", "failed", "2026-10-10T18:41:53.513Z"]),
|
||||
}),
|
||||
);
|
||||
expect(failure).toMatchObject({
|
||||
stage: "spec",
|
||||
headline: "Anthropic declined the request for billing reasons",
|
||||
provider: "anthropic",
|
||||
message: CREDIT,
|
||||
});
|
||||
expect(failure?.nextStep).toBe("Add credit or raise the spend limit with Anthropic, then choose Retry from Actions.");
|
||||
});
|
||||
|
||||
test("names an unknown provider by its id", () => {
|
||||
const failure = view(
|
||||
failedRun({ message: "model request failed (authentication): provider acme bad key [provider acme, status 401, code unauthorized]" }),
|
||||
);
|
||||
expect(failure?.headline).toBe("acme rejected the API key");
|
||||
});
|
||||
|
||||
test("reads auth, rate limits, timeouts and outages from the request's own status and kind", () => {
|
||||
const headline = (message: string) => view(failedRun({ message }))?.headline;
|
||||
expect(headline("model request failed (authentication): provider anthropic invalid x-api-key [provider anthropic, status 401, code authentication_error]"))
|
||||
.toBe("Anthropic rejected the API key");
|
||||
expect(headline("model request failed (rate_limit): provider openai Too many requests [provider openai, status 429, code rate_limit_exceeded]"))
|
||||
.toBe("OpenAI rate-limited the request");
|
||||
expect(headline("model request failed (server): provider anthropic Overloaded [provider anthropic, status 529, code overloaded_error]"))
|
||||
.toBe("Anthropic was unavailable");
|
||||
expect(headline("model request failed (timeout): provider openai request timed out"))
|
||||
.toBe("The request to OpenAI timed out");
|
||||
});
|
||||
|
||||
test("falls back to the provider's status and code", () => {
|
||||
const failure = view(
|
||||
failedRun({
|
||||
message: "model request failed (invalid_request): provider openai Unsupported parameter: 'temperature' is not supported with this model. [provider openai, status 400, code unsupported_parameter]",
|
||||
}),
|
||||
);
|
||||
expect(failure?.headline).toBe("OpenAI returned 400 unsupported parameter");
|
||||
});
|
||||
|
||||
test("never reads a provider, status or billing problem out of a command's own words", () => {
|
||||
const failure = view(
|
||||
failedRun({
|
||||
message: "script exited with status 401",
|
||||
exec_output_tail: { stdout: "test billing::invoice ... FAILED\nretry the provider list later\n", stderr: null },
|
||||
}),
|
||||
);
|
||||
expect(failure).toMatchObject({
|
||||
headline: "The step's command exited with status 401",
|
||||
provider: null,
|
||||
nextStep: "Its output, under Details, usually says why.",
|
||||
});
|
||||
expect(view(failedRun({ message: "Ran 512 tests; line 503 failed." }))?.headline).toBe("Ran 512 tests; line 503 failed.");
|
||||
});
|
||||
|
||||
test("names no step when the run recovered from an earlier failed one", () => {
|
||||
const failure = view(
|
||||
failedRun({ message: "push rejected: remote branch moved" }, {
|
||||
reason: "publish_failed",
|
||||
runStages: stages(
|
||||
["toolchain@1", "failed", "2026-10-10T15:20:45Z"],
|
||||
["fixup@1", "succeeded", "2026-10-10T15:21:00Z"],
|
||||
["verify@1", "succeeded", "2026-10-10T15:25:00.500Z"],
|
||||
),
|
||||
}),
|
||||
);
|
||||
expect(failure?.stage).toBeNull();
|
||||
expect(failure?.headline).toBe("The steps finished, but publishing the result failed");
|
||||
});
|
||||
|
||||
test("shows nothing for a cancelled run, and says what happened for other endings", () => {
|
||||
expect(view(failedRun({ message: "Run cancelled before worker launch completed" }, { reason: "cancelled" }))).toBeNull();
|
||||
expect(view(failedRun({ message: "denied at review" }, { reason: "approval_denied" }))?.headline).toBe("Approval was denied");
|
||||
expect(view(failedRun({ message: "budget of $5 reached" }, { reason: "budget_exhausted" }))?.headline).toBe("The run used up its budget");
|
||||
expect(view(failedRun({ message: "sandbox image not found" }, { reason: "sandbox_init_failed" }))?.headline).toBe("The run couldn't start");
|
||||
expect(view(failedRun({ message: "worker exited unexpectedly" }, { kind: "dead" }))?.headline).toBe("Worker exited unexpectedly");
|
||||
});
|
||||
|
||||
test("shows the end of a command's output, from whichever stream has it", () => {
|
||||
const tail = (exec_output_tail: Record<string, unknown>) =>
|
||||
view(failedRun({ message: "script exited with status 1", exec_output_tail }))?.output;
|
||||
expect(tail({ stderr: "\n \n", stdout: "real error\n" })).toBe("real error");
|
||||
expect(tail({ stderr: "line one\r\nline two\r\n", stdout: null })).toBe("line one\nline two");
|
||||
expect(tail({ stdout: "10%\r50%\r100% done\n" })).toBe("100% done");
|
||||
const many = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`).join("\n");
|
||||
expect(tail({ stdout: many })?.split("\n")).toEqual(Array.from({ length: 12 }, (_, i) => `line ${i + 19}`));
|
||||
});
|
||||
|
||||
test("cuts a long fallback headline without splitting a character", () => {
|
||||
const headline = view(failedRun({ message: `${"🔥".repeat(200)} and more` }))!.headline;
|
||||
expect(Array.from(headline)).toHaveLength(111);
|
||||
expect(headline.endsWith("🔥…")).toBe(true);
|
||||
});
|
||||
|
||||
test("shows nothing for runs that didn't fail", () => {
|
||||
expect(buildRunFailureView({ status: { kind: "succeeded" }, stages: {} } as State)).toBeNull();
|
||||
expect(buildRunFailureView(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
179
apps/fabro-web/app/routes/run-detail/failure.ts
Normal file
179
apps/fabro-web/app/routes/run-detail/failure.ts
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import {
|
||||
FailureCategory,
|
||||
FailureReason,
|
||||
type ExecOutputTail,
|
||||
type RunProjection,
|
||||
} from "@qltysh/fabro-api-client";
|
||||
|
||||
/** Lines of a failed command's output shown on the page. */
|
||||
const OUTPUT_LINES = 12;
|
||||
/** Fallback headlines are cut to this many characters. */
|
||||
const HEADLINE_CHARS = 110;
|
||||
|
||||
export interface RunFailureView {
|
||||
/** The step the run stopped at (`spec`, `review@3`), when it stopped at one. */
|
||||
stage: string | null;
|
||||
headline: string;
|
||||
nextStep: string;
|
||||
/** The recorded failure message, verbatim. */
|
||||
message: string;
|
||||
/** Provider id, when the failure is a model request the provider refused. */
|
||||
provider: string | null;
|
||||
/** Last OUTPUT_LINES of the failed command's stderr, else its stdout. */
|
||||
output: string | null;
|
||||
}
|
||||
|
||||
type FailureState = Pick<RunProjection, "status" | "conclusion" | "stages">;
|
||||
|
||||
/**
|
||||
* Reads the failure from run state (`conclusion.failure`); the run summary the
|
||||
* status popover reads doesn't carry the message or the command's output.
|
||||
*/
|
||||
export function buildRunFailureView(
|
||||
state: FailureState | null | undefined,
|
||||
providerNames: ReadonlyMap<string, string> = new Map(),
|
||||
): RunFailureView | null {
|
||||
const kind = state?.status?.kind;
|
||||
if (!state || (kind !== "failed" && kind !== "dead")) return null;
|
||||
const failure = state.conclusion?.failure;
|
||||
if (!failure?.detail.message || failure.reason === FailureReason.CANCELLED) return null;
|
||||
|
||||
const { message, category } = failure.detail;
|
||||
const output = outputTail(failure.detail.exec_output_tail);
|
||||
const request = parseModelRequest(message);
|
||||
const name = (id: string) => providerNames.get(id) ?? id;
|
||||
const advice =
|
||||
reasonAdvice(failure.reason, category) ??
|
||||
(request ? requestAdvice(request, message, name(request.provider)) : stepAdvice(message, category, output));
|
||||
return {
|
||||
stage: stoppedAt(state.stages),
|
||||
...advice,
|
||||
message,
|
||||
provider: request?.provider ?? null,
|
||||
output,
|
||||
};
|
||||
}
|
||||
|
||||
type Advice = Pick<RunFailureView, "headline" | "nextStep">;
|
||||
|
||||
const RETRY = "choose Retry from Actions";
|
||||
|
||||
function reasonAdvice(reason: FailureReason, category: FailureCategory): Advice | null {
|
||||
if (reason === FailureReason.APPROVAL_DENIED) {
|
||||
return { headline: "Approval was denied", nextStep: `The run stops where it was denied. To run it again, ${RETRY}.` };
|
||||
}
|
||||
if (reason === FailureReason.BUDGET_EXHAUSTED || category === FailureCategory.BUDGET_EXHAUSTED) {
|
||||
return { headline: "The run used up its budget", nextStep: `Raise the run's budget, then ${RETRY}.` };
|
||||
}
|
||||
if (reason === FailureReason.TERMINATED) {
|
||||
return { headline: "The run was terminated", nextStep: `To run it again, ${RETRY}.` };
|
||||
}
|
||||
if (reason === FailureReason.PUBLISH_FAILED) {
|
||||
return {
|
||||
headline: "The steps finished, but publishing the result failed",
|
||||
nextStep: "The work is on the run branch. The message under Details says why publishing failed.",
|
||||
};
|
||||
}
|
||||
if (
|
||||
reason === FailureReason.LAUNCH_FAILED ||
|
||||
reason === FailureReason.BOOTSTRAP_FAILED ||
|
||||
reason === FailureReason.SANDBOX_INIT_FAILED
|
||||
) {
|
||||
return { headline: "The run couldn't start", nextStep: "The message under Details says why." };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
interface ModelRequest {
|
||||
provider: string;
|
||||
kind: string | null;
|
||||
status: number | null;
|
||||
code: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A refused model request, as the server records it:
|
||||
* `model request failed (<kind>): provider <id> <text> [provider <id>, status <n>, code <code>]`.
|
||||
* Only this shape is read, so numbers and words in a command's output can't
|
||||
* pass for a provider's status or name.
|
||||
*/
|
||||
function parseModelRequest(message: string): ModelRequest | null {
|
||||
const lead = /^model request failed \(([a-z_]+)\): provider (\S+)/i.exec(message);
|
||||
const tail = /\[provider ([^,\]]+), status (\d{3})(?:, code ([^\]]+))?\]\s*$/i.exec(message);
|
||||
const provider = tail?.[1] ?? lead?.[2];
|
||||
if (!provider) return null;
|
||||
return {
|
||||
provider: provider.toLowerCase(),
|
||||
kind: lead?.[1]?.toLowerCase() ?? null,
|
||||
status: tail ? Number(tail[2]) : null,
|
||||
code: tail?.[3] ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function requestAdvice({ kind, status, code }: ModelRequest, message: string, who: string): Advice {
|
||||
if (/credit balance|insufficient[_ ]quota|payment required|billing|spend limit/i.test(message) || status === 402) {
|
||||
return {
|
||||
headline: `${who} declined the request for billing reasons`,
|
||||
nextStep: `Add credit or raise the spend limit with ${who}, then ${RETRY}.`,
|
||||
};
|
||||
}
|
||||
if (kind === "authentication" || status === 401 || status === 403) {
|
||||
return { headline: `${who} rejected the API key`, nextStep: `Check the key under Settings → Secrets, then ${RETRY}.` };
|
||||
}
|
||||
if (kind === "rate_limit" || status === 429) {
|
||||
return { headline: `${who} rate-limited the request`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." };
|
||||
}
|
||||
if (kind === "timeout" || /timed out/i.test(message)) {
|
||||
return { headline: `The request to ${who} timed out`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." };
|
||||
}
|
||||
if ((status !== null && status >= 500) || /overloaded/i.test(message)) {
|
||||
return { headline: `${who} was unavailable`, nextStep: "This is usually temporary. Choose Retry from Actions in a few minutes." };
|
||||
}
|
||||
const what = [status, code?.replace(/_/g, " ")].filter(Boolean).join(" ");
|
||||
return {
|
||||
headline: what ? `${who} returned ${what}` : `${who} rejected the request`,
|
||||
nextStep: `${who}'s message, under Details, says why.`,
|
||||
};
|
||||
}
|
||||
|
||||
function stepAdvice(message: string, category: FailureCategory, output: string | null): Advice {
|
||||
const exit = /exited with (?:status|code) (\d+)/i.exec(message);
|
||||
if (exit) {
|
||||
return {
|
||||
headline: `The step's command exited with status ${exit[1]}`,
|
||||
nextStep: output ? "Its output, under Details, usually says why." : "Open the step's logs for its output.",
|
||||
};
|
||||
}
|
||||
if (category === FailureCategory.TRANSIENT_INFRA) {
|
||||
return { headline: "The step hit a temporary infrastructure error", nextStep: "Choose Retry from Actions in a few minutes." };
|
||||
}
|
||||
return { headline: firstSentence(message), nextStep: "Open the failed step's logs for details." };
|
||||
}
|
||||
|
||||
/**
|
||||
* The last step to start, if it failed. An earlier failure that a later step
|
||||
* recovered from isn't where the run stopped, so it isn't named.
|
||||
*/
|
||||
function stoppedAt(stages: FailureState["stages"]): string | null {
|
||||
let last: [string, number] | null = null;
|
||||
for (const [key, stage] of Object.entries(stages)) {
|
||||
const started = stage.started_at ? Date.parse(stage.started_at) : NaN;
|
||||
if (!Number.isNaN(started) && (!last || started > last[1])) last = [key, started];
|
||||
}
|
||||
if (!last || stages[last[0]]?.state !== "failed") return null;
|
||||
return last[0].replace(/@1$/, "");
|
||||
}
|
||||
|
||||
function outputTail(tail: ExecOutputTail | null | undefined): string | null {
|
||||
const stream = [tail?.stderr, tail?.stdout].find((text) => text?.trim());
|
||||
if (!stream) return null;
|
||||
// Carriage-return progress lines keep only what was drawn last.
|
||||
const lines = stream.trimEnd().split(/\r?\n/).map((line) => line.slice(line.lastIndexOf("\r") + 1));
|
||||
return lines.slice(-OUTPUT_LINES).join("\n");
|
||||
}
|
||||
|
||||
function firstSentence(message: string): string {
|
||||
const sentence = message.split(/(?<=[.!?])\s+/)[0]!.trim().replace(/\s+/g, " ");
|
||||
const chars = Array.from(sentence.charAt(0).toUpperCase() + sentence.slice(1));
|
||||
return chars.length > HEADLINE_CHARS ? `${chars.slice(0, HEADLINE_CHARS).join("").trimEnd()}…` : chars.join("");
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue