scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-08-28 05:27:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge remote stage billing cleanup

Browse source
This commit is contained in:
Bryan Helmkamp 2026-07-28 15:10:55 -04:00
commit 75da0873f6
No known key found for this signature in database
14 changed files with 140 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
apps/fabro-web/app/components/stage-popover.test.tsx
View file

@ -9,7 +9,7 @@ import { StagePopover } from "./stage-popover";
import { deriveStageSummary } from "./stage-popover-summary";
import type { Stage } from "../lib/stage-sidebar";
import { generatedAxios } from "../lib/api-client";
import { testBilledTokenCounts } from "../lib/test-fixtures";
import { makeBilledTokenCounts } from "../lib/test-fixtures";
function makeEvent(overrides: Partial<EventEnvelope>): EventEnvelope {
return {
@ -35,7 +35,7 @@ function makeStage(overrides: Partial<Stage> = {}): Stage {
duration: "1m 30s",
startedAt: "2026-05-24T11:58:30Z",
providerUsed: { mode: "policy", model: "claude-opus-4-7", reasoning_effort: "high" },
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
...overrides,
};
}

4
apps/fabro-web/app/components/stage-renderers/fan-in-results.test.tsx
View file

@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import type { EventEnvelope } from "@qltysh/fabro-api-client";
import TestRenderer, { act } from "react-test-renderer";
import { testBilledTokenCounts } from "../../lib/test-fixtures";
import { makeBilledTokenCounts } from "../../lib/test-fixtures";
import { makeEventEnvelope, setupReactTestEnv } from "../../lib/test-utils";
import type { Stage } from "../stage-sidebar";
import { FanInResults } from "./fan-in-results";
@ -25,7 +25,7 @@ const fanInStage: Stage = {
resumedFromStageId: null,
startedAt: "2026-04-09T12:00:00Z",
providerUsed: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
};
function event(seq: number, partial: Partial<EventEnvelope>): EventEnvelope {

4
apps/fabro-web/app/components/stage-renderers/parallel-children.test.tsx
View file

@ -3,7 +3,7 @@ import type { EventEnvelope } from "@qltysh/fabro-api-client";
import TestRenderer, { act } from "react-test-renderer";
import { MemoryRouter } from "react-router";
import { testBilledTokenCounts } from "../../lib/test-fixtures";
import { makeBilledTokenCounts } from "../../lib/test-fixtures";
import { makeEventEnvelope, setupReactTestEnv } from "../../lib/test-utils";
import type { Stage } from "../stage-sidebar";
import { ParallelChildren } from "./parallel-children";
@ -26,7 +26,7 @@ const parallelStage: Stage = {
resumedFromStageId: null,
startedAt: "2026-04-09T12:00:00Z",
providerUsed: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
};
function event(partial: Partial<EventEnvelope>): EventEnvelope {

4
apps/fabro-web/app/components/stage-sidebar.test.tsx
View file

@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test";
import TestRenderer, { act } from "react-test-renderer";
import { MemoryRouter } from "react-router";
import { testBilledTokenCounts } from "../lib/test-fixtures";
import { makeBilledTokenCounts } from "../lib/test-fixtures";
import { StageSidebar, type Stage } from "./stage-sidebar";
function makeStage(overrides: Partial<Stage> = {}): Stage {
@ -18,7 +18,7 @@ function makeStage(overrides: Partial<Stage> = {}): Stage {
duration: "--",
startedAt: null,
providerUsed: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
...overrides,
};
}

13
apps/fabro-web/app/lib/billing.test.ts
View file

@ -5,13 +5,13 @@ import {
formatBillingTokenCount,
hasBillingUsage,
} from "./billing";
import { testBilledTokenCounts } from "./test-fixtures";
import { makeBilledTokenCounts } from "./test-fixtures";
describe("billingTokenBuckets", () => {
test("returns the shared display order and folds reasoning into output", () => {
expect(
billingTokenBuckets(
testBilledTokenCounts({
makeBilledTokenCounts({
input_tokens: 10,
output_tokens: 20,
reasoning_tokens: 5,
@ -30,12 +30,15 @@ describe("billingTokenBuckets", () => {
describe("hasBillingUsage", () => {
test("includes cache-only and cost-only usage", () => {
expect(hasBillingUsage(testBilledTokenCounts())).toBe(false);
expect(hasBillingUsage(makeBilledTokenCounts())).toBe(false);
expect(
hasBillingUsage(testBilledTokenCounts({ cache_read_tokens: 1 })),
hasBillingUsage(makeBilledTokenCounts({ cache_read_tokens: 1 })),
).toBe(true);
expect(
hasBillingUsage(testBilledTokenCounts({ total_usd_micros: 1 })),
hasBillingUsage(makeBilledTokenCounts({ total_usd_micros: 1 })),
).toBe(true);
expect(
hasBillingUsage(makeBilledTokenCounts({ total_tokens: 1 })),
).toBe(true);
});
});

10
apps/fabro-web/app/lib/billing.ts
View file

@ -7,6 +7,10 @@ export interface BillingTokenBucket {
value: number;
}
export function billableOutputTokens(billing: BilledTokenCounts): number {
return billing.output_tokens + billing.reasoning_tokens;
}
/**
* Return the disjoint billing buckets in their shared display order.
* Reasoning tokens are billed as output tokens.
@ -16,10 +20,7 @@ export function billingTokenBuckets(billing: BilledTokenCounts): BillingTokenBuc
{ label: "Cache read", value: billing.cache_read_tokens },
{ label: "Cache creation", value: billing.cache_write_tokens },
{ label: "Uncached", value: billing.input_tokens },
{
label: "Output",
value: billing.output_tokens + billing.reasoning_tokens,
},
{ label: "Output", value: billableOutputTokens(billing) },
];
}
@ -30,6 +31,7 @@ export function hasBillingUsage(billing: BilledTokenCounts): boolean {
billing.reasoning_tokens !== 0 ||
billing.cache_read_tokens !== 0 ||
billing.cache_write_tokens !== 0 ||
billing.total_tokens !== 0 ||
(billing.total_usd_micros ?? 0) !== 0
);
}

8
apps/fabro-web/app/lib/stage-sidebar.test.ts
View file

@ -8,13 +8,13 @@ import type {
import type { Stage } from "../components/stage-sidebar";
import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar";
import { testBilledTokenCounts } from "./test-fixtures";
import { makeBilledTokenCounts } from "./test-fixtures";
function runStage(
stage: Omit<RunStage, "billing"> & Partial<Pick<RunStage, "billing">>,
): RunStage {
return {
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
...stage,
};
}
@ -32,7 +32,7 @@ function makeStage(nodeId: string, visit: number, status: StageState): Stage {
duration: "--",
startedAt: null,
providerUsed: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
};
}
@ -54,7 +54,7 @@ describe("mapRunStagesToSidebarStages", () => {
model: "gpt-5.5",
reasoning_effort: "high",
},
billing: testBilledTokenCounts({
billing: makeBilledTokenCounts({
input_tokens: 28_640,
output_tokens: 7_550,
total_tokens: 43_690,

10
apps/fabro-web/app/lib/test-fixtures.ts
View file

@ -7,16 +7,16 @@ export const TEST_PRINCIPAL: Principal = {
auth_method: "dev_token",
};
export function testBilledTokenCounts(
export function makeBilledTokenCounts(
overrides: Partial<BilledTokenCounts> = {},
): BilledTokenCounts {
return {
input_tokens: 0,
output_tokens: 0,
total_tokens: 0,
reasoning_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
input_tokens: 0,
output_tokens: 0,
reasoning_tokens: 0,
total_tokens: 0,
...overrides,
};
}

24
apps/fabro-web/app/routes/run-billing.test.tsx
View file

@ -3,7 +3,7 @@ import TestRenderer from "react-test-renderer";
import type { RunBilling, StageTiming } from "@qltysh/fabro-api-client";
import { testBilledTokenCounts } from "../lib/test-fixtures";
import { makeBilledTokenCounts } from "../lib/test-fixtures";
function stageTiming(wall_time_ms = 0, inference_time_ms = 0, tool_time_ms = 0): StageTiming {
return {
@ -27,7 +27,7 @@ function billing(overrides: Partial<RunBilling> = {}): RunBilling {
stages: [],
totals: {
timing: stageTiming(),
...testBilledTokenCounts(),
...makeBilledTokenCounts(),
},
by_model: [],
...overrides,
@ -71,21 +71,21 @@ describe("RunBilling", () => {
{
stage: { id: "start", name: "start" },
model: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
timing: stageTiming(),
state: "succeeded",
},
{
stage: { id: "command", name: "command" },
model: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
timing: stageTiming(61000),
state: "succeeded",
},
],
totals: {
timing: stageTiming(61000),
...testBilledTokenCounts(),
...makeBilledTokenCounts(),
},
}),
);
@ -106,7 +106,7 @@ describe("RunBilling", () => {
{
stage: { id: "start", name: "start" },
model: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
timing: stageTiming(),
state: "succeeded",
},
@ -116,7 +116,7 @@ describe("RunBilling", () => {
provider: "anthropic",
model_id: "claude-sonnet-4-5",
},
billing: testBilledTokenCounts({
billing: makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,
@ -128,7 +128,7 @@ describe("RunBilling", () => {
],
totals: {
timing: stageTiming(42000),
...testBilledTokenCounts({
...makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,
@ -142,7 +142,7 @@ describe("RunBilling", () => {
model_id: "claude-sonnet-4-5",
},
stages: 1,
billing: testBilledTokenCounts({
billing: makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,
@ -189,7 +189,7 @@ describe("RunBilling", () => {
model_id: "claude-opus-4-6",
speed: "fast",
},
billing: testBilledTokenCounts({
billing: makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,
@ -202,7 +202,7 @@ describe("RunBilling", () => {
],
totals: {
timing: stageTiming(),
...testBilledTokenCounts({
...makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,
@ -217,7 +217,7 @@ describe("RunBilling", () => {
speed: "fast",
},
stages: 1,
billing: testBilledTokenCounts({
billing: makeBilledTokenCounts({
input_tokens: 1200,
output_tokens: 300,
total_tokens: 1500,

30
apps/fabro-web/app/routes/run-billing.tsx
View file

@ -3,6 +3,7 @@ import { Fragment, useMemo } from "react";
import { EmptyState } from "../components/state";
import { Tooltip } from "../components/ui";
import {
billableOutputTokens,
billingTokenBuckets,
formatBillingTokenCount,
hasBillingUsage,
@ -75,18 +76,18 @@ function mapStageRow(stage: RunBillingStage, wallTimeMs: number): MappedStageRow
/** Hover breakdown of the disjoint token buckets behind an `in / out` count. */
function TokenBreakdown({ billing }: { billing: BilledTokenCounts }) {
const rows = billingTokenBuckets(billing);
const buckets = billingTokenBuckets(billing);
return (
<div className="min-w-44 py-0.5">
<div className="mb-1.5 border-b border-line pb-1 font-medium text-fg-2">
<div className="border-line text-fg-2 mb-1.5 border-b pb-1 font-medium">
Tokens in / out
</div>
<dl className="grid grid-cols-[1fr_auto] gap-x-6 gap-y-1">
{rows.map((row) => (
<Fragment key={row.label}>
<dt className="text-fg-3">{row.label}</dt>
<dd className="text-right font-mono tabular-nums text-fg">
{formatTokens(row.value)}
{buckets.map((bucket) => (
<Fragment key={bucket.label}>
<dt className="text-fg-3">{bucket.label}</dt>
<dd className="text-fg text-right font-mono tabular-nums">
{formatTokens(bucket.value)}
</dd>
</Fragment>
))}
@ -99,21 +100,14 @@ function TokenBreakdown({ billing }: { billing: BilledTokenCounts }) {
* Renders an `input / output` token count. When the row has model usage,
* hovering the count reveals the cache breakdown.
*/
function TokensCell({
billing,
}: {
billing: BilledTokenCounts | null;
}) {
const inputTokens = billing?.input_tokens;
const outputTokens =
billing == null ? null : billing.output_tokens + billing.reasoning_tokens;
function TokensCell({ billing }: { billing: BilledTokenCounts | null }) {
const display = (
<>
{formatTokens(inputTokens)} <span className="text-fg-muted">/</span>{" "}
{formatTokens(outputTokens)}
{formatTokens(billing?.input_tokens)} <span className="text-fg-muted">/</span>{" "}
{formatTokens(billing ? billableOutputTokens(billing) : null)}
</>
);
if (billing == null) return display;
if (!billing) return display;
return (
<Tooltip label={<TokenBreakdown billing={billing} />}>
<span>{display}</span>

4
apps/fabro-web/app/routes/run-stages-chat.test.tsx
View file

@ -3,7 +3,7 @@ import { renderToStaticMarkup } from "react-dom/server";
import { StageState } from "@qltysh/fabro-api-client";
import type { Stage } from "../lib/stage-sidebar";
import { testBilledTokenCounts } from "../lib/test-fixtures";
import { makeBilledTokenCounts } from "../lib/test-fixtures";
import { StageChatView } from "./run-stages";
function stage(overrides: Partial<Stage> = {}): Stage {
@ -19,7 +19,7 @@ function stage(overrides: Partial<Stage> = {}): Stage {
resumedFromStageId: null,
startedAt: "2026-04-09T12:00:00Z",
providerUsed: null,
billing: testBilledTokenCounts(),
billing: makeBilledTokenCounts(),
...overrides,
};
}

14
apps/fabro-web/app/routes/run-stages-details.test.tsx
View file

@ -7,7 +7,7 @@ import type {
StageModelUsage,
} from "@qltysh/fabro-api-client";
import { testBilledTokenCounts } from "../lib/test-fixtures";
import { makeBilledTokenCounts } from "../lib/test-fixtures";
import { EventDetails, ModelUsagePopover } from "./run-stages";
const RUN_START = "2026-04-09T12:00:00Z";
@ -94,7 +94,7 @@ function popoverMarkup(counts: BilledTokenCounts): string {
describe("ModelUsagePopover billing", () => {
test("shows the visit's token buckets and cost next to the model", () => {
const html = popoverMarkup(
testBilledTokenCounts({
makeBilledTokenCounts({
input_tokens: 28_640,
output_tokens: 7_550,
reasoning_tokens: 1_200,
@ -120,7 +120,7 @@ describe("ModelUsagePopover billing", () => {
});
test("omits the token section for a stage that called no model", () => {
const html = popoverMarkup(testBilledTokenCounts());
const html = popoverMarkup(makeBilledTokenCounts());
expect(html).toContain("kimi-k3");
expect(html).not.toContain("Tokens");
@ -129,7 +129,7 @@ describe("ModelUsagePopover billing", () => {
test("still shows tokens when nothing priced the stage", () => {
const html = popoverMarkup(
testBilledTokenCounts({
makeBilledTokenCounts({
input_tokens: 1_000,
output_tokens: 500,
total_tokens: 1_500,
@ -141,14 +141,14 @@ describe("ModelUsagePopover billing", () => {
expect(html).not.toContain("Cost");
});
test("shows a reported cost when token buckets are empty", () => {
test("shows a provider-reported cost when token counts are unavailable", () => {
const html = popoverMarkup(
testBilledTokenCounts({ total_usd_micros: 1_000_000 }),
makeBilledTokenCounts({ total_usd_micros: 720_000 }),
);
expect(html).toContain("kimi-k3");
expect(html).toContain("Tokens");
expect(html).toContain("Cost");
expect(html).toContain("$1.00");
expect(html).toContain("$0.72");
});
});

152
lib/apps/fabro-server/src/server/tests.rs
View file

@ -5259,6 +5259,64 @@ fn test_billed_usage(
.unwrap()
}
async fn create_billed_retry_run(state: &Arc<AppState>, run_id: RunId) {
create_durable_run_with_events(state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
state,
run_id,
"verify",
1,
&workflow_event::Event::StageFailed {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
failure: FailureDetail::new("try again", FailureCategory::TransientInfra),
will_retry: true,
timing: fabro_types::StageTiming::wall_only(1200),
billing: Some(test_billed_usage("gpt-old", 100, 10)),
actor: None,
},
)
.await;
append_scoped_stage_event(
state,
run_id,
"verify",
2,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
timing: fabro_types::StageTiming::wall_only(800),
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(test_billed_usage("gpt-new", 200, 20)),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 2,
max_attempts: 2,
},
)
.await;
}
#[tokio::test]
async fn list_run_stages_distinguishes_visits() {
let state = test_app_state_with_isolated_storage();
@ -5650,69 +5708,13 @@ async fn run_billing_dedups_retried_nodes_and_sums_their_durations() {
}
#[tokio::test]
async fn billing_endpoints_report_retry_usage_per_node_and_per_visit() {
async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
let failed_usage = test_billed_usage("gpt-old", 100, 10);
create_billed_retry_run(&state, run_id).await;
let success_usage = test_billed_usage("gpt-new", 200, 20);
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
&state,
run_id,
"verify",
1,
&workflow_event::Event::StageFailed {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
failure: FailureDetail::new("try again", FailureCategory::TransientInfra),
will_retry: true,
timing: fabro_types::StageTiming::wall_only(1200),
billing: Some(failed_usage),
actor: None,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"verify",
2,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
timing: fabro_types::StageTiming::wall_only(800),
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(success_usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 2,
max_attempts: 2,
},
)
.await;
let mut latest_outcome: Outcome<Option<fabro_model::BilledModelUsage>> = Outcome::success();
latest_outcome.usage = Some(success_usage);
latest_outcome.timing = Some(fabro_types::StageTiming::wall_only(800));
@ -5746,7 +5748,6 @@ async fn billing_endpoints_report_retry_usage_per_node_and_per_visit() {
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
@ -5789,6 +5790,18 @@ async fn billing_endpoints_report_retry_usage_per_node_and_per_visit() {
assert_eq!(old_model["billing"]["input_tokens"], 100);
assert_eq!(new_model["stages"], 1);
assert_eq!(new_model["billing"]["input_tokens"], 200);
}
/// The stage popover reads `billing` off the stages list, so it must be scoped
/// to one visit — unlike the Billing tab's rows, which sum every visit of a
/// node.
#[tokio::test]
async fn list_run_stages_reports_billing_per_visit() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
create_billed_retry_run(&state, run_id).await;
let response = app
.oneshot(
@ -5827,23 +5840,8 @@ async fn list_run_stages_reports_zero_billing_for_a_stage_that_called_no_model()
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
&state,
run_id,
"script",
1,
&workflow_event::Event::StageStarted {
graph_visit: None,
resumed_from_stage_id: None,
node_id: "script".to_string(),
name: "Script".to_string(),
index: 0,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 1,
},
)
.await;
let started = stage_started_event("script", "command");
append_scoped_stage_event(&state, run_id, "script", 1, &started).await;
let response = app
.oneshot(

2
lib/foundation/fabro-types/src/run_projection.rs
View file

@ -1054,7 +1054,7 @@ mod iter_stages_tests {
}
#[test]
fn billed_usage_leaves_an_unused_model_uncosted() {
fn billed_usage_leaves_zero_tokens_uncosted() {
let mut stage = priced_stage(None);
stage.usage = BilledTokenCounts::default();