mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-11-14 09:01:16 +00:00
Merge remote-tracking branch 'origin/main' into codex/adopt-stage-files
This commit is contained in:
Scott Werner
commit
d78a46f741
34 changed files with 1482 additions and 603 deletions
29
.github/workflows/installer.yml
vendored
Normal file
29
.github/workflows/installer.yml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
name: Installer
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "installer/**"
|
||||
- ".github/workflows/installer.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "installer/**"
|
||||
- ".github/workflows/installer.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
homebrew-push:
|
||||
name: Homebrew push retries
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: python3 -m unittest discover -s installer/tests -v
|
||||
10
.github/workflows/release.yml
vendored
10
.github/workflows/release.yml
vendored
|
|
@ -319,7 +319,10 @@ jobs:
|
|||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add Formula/fabro.rb versions.json
|
||||
git diff --cached --quiet || (git commit -m "Update fabro to ${TAG_NAME}" && git push)
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "Update fabro to ${TAG_NAME}"
|
||||
bash ../installer/push-homebrew.sh
|
||||
fi
|
||||
|
||||
update-homebrew-nightly:
|
||||
name: Update Homebrew Nightly Formula
|
||||
|
|
@ -368,4 +371,7 @@ jobs:
|
|||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add Formula/fabro-nightly.rb versions.json
|
||||
git diff --cached --quiet || (git commit -m "Update fabro-nightly to ${TAG_NAME}" && git push)
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "Update fabro-nightly to ${TAG_NAME}"
|
||||
bash ../installer/push-homebrew.sh
|
||||
fi
|
||||
|
|
|
|||
90
Cargo.lock
generated
90
Cargo.lock
generated
|
|
@ -1992,7 +1992,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-api"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"fabro-automation",
|
||||
|
|
@ -2018,7 +2018,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-auth"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2043,7 +2043,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-automation"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2064,11 +2064,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-build-support"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
|
||||
[[package]]
|
||||
name = "fabro-checkpoint"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"fabro-config",
|
||||
"fabro-types",
|
||||
|
|
@ -2076,7 +2076,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-cli"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"assert_cmd",
|
||||
|
|
@ -2172,7 +2172,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-client"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
|
|
@ -2201,7 +2201,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-config"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2230,7 +2230,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-db"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2243,7 +2243,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-dev"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"assert_cmd",
|
||||
|
|
@ -2262,7 +2262,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-dot"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"fabro-template",
|
||||
"fabro-types",
|
||||
|
|
@ -2274,7 +2274,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-dump"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
|
|
@ -2288,7 +2288,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-environment"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2310,7 +2310,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-github"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2335,7 +2335,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-graphviz"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fabro-dot",
|
||||
|
|
@ -2345,7 +2345,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-http"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"fabro-static",
|
||||
"http 1.4.0",
|
||||
|
|
@ -2355,7 +2355,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-install"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fabro-config",
|
||||
|
|
@ -2372,7 +2372,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-interview"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"fabro-types",
|
||||
|
|
@ -2384,7 +2384,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-llm"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
|
|
@ -2409,7 +2409,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-macros"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"fabro-options-metadata",
|
||||
|
|
@ -2420,7 +2420,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-manifest"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2449,7 +2449,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-mcp-server"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fabro-client",
|
||||
|
|
@ -2469,7 +2469,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-mcp-store"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"fabro-db",
|
||||
|
|
@ -2487,7 +2487,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-oauth"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
|
|
@ -2509,7 +2509,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-options-metadata"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2517,7 +2517,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-petri"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2572,7 +2572,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-proc"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"tempfile",
|
||||
|
|
@ -2584,7 +2584,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-redact"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"pebble-coding-agent",
|
||||
|
|
@ -2602,7 +2602,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-server"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2697,7 +2697,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-slack"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"fabro-http",
|
||||
"fabro-interview",
|
||||
|
|
@ -2718,18 +2718,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-spa"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"rust-embed",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fabro-static"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
|
||||
[[package]]
|
||||
name = "fabro-store"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
|
|
@ -2757,7 +2757,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-telemetry"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
|
|
@ -2783,7 +2783,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-template"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fabro-types",
|
||||
|
|
@ -2797,7 +2797,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-test"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"assert_cmd",
|
||||
|
|
@ -2822,7 +2822,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-tool"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2846,7 +2846,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-tracker"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2860,7 +2860,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-types"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"clap",
|
||||
|
|
@ -2887,7 +2887,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-util"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"console 0.15.11",
|
||||
|
|
@ -2910,7 +2910,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-variable"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2927,7 +2927,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-vault"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
|
|
@ -2946,7 +2946,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-workflow"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2991,7 +2991,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fabro-workflow-version"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"fabro-config",
|
||||
"fabro-dot",
|
||||
|
|
@ -7931,7 +7931,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "twin-github"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ resolver = "2"
|
|||
|
||||
[workspace.package]
|
||||
edition = "2021"
|
||||
version = "0.403.0-nightly.0"
|
||||
version = "0.407.0-nightly.0"
|
||||
license = "MIT"
|
||||
|
||||
[workspace.dependencies]
|
||||
|
|
|
|||
|
|
@ -116,6 +116,8 @@ interface SubscribeOptions<TPayload extends EventPayload> {
|
|||
resyncKeys: () => KeyOrMatcher[];
|
||||
fallbackSubscribe: () => () => void;
|
||||
debounceMs?: number;
|
||||
/** Flush pending keys within this interval, even while events keep arriving. */
|
||||
maxWaitMs?: number;
|
||||
}
|
||||
|
||||
export interface SubscribeToCrossTabSseOptions<TPayload extends EventPayload>
|
||||
|
|
@ -136,6 +138,8 @@ interface LocalSubscription {
|
|||
pendingKeys: Map<string, KeyOrMatcher>;
|
||||
debounceTimer: ReturnType<typeof setTimeout> | null;
|
||||
debounceMs: number;
|
||||
maxWaitMs?: number;
|
||||
maxWaitTimer: ReturnType<typeof setTimeout> | null;
|
||||
resolveInvalidation: (payload: EventPayload) => EventInvalidation;
|
||||
resyncKeys: () => KeyOrMatcher[];
|
||||
}
|
||||
|
|
@ -295,6 +299,8 @@ export class CrossTabSseCoordinator {
|
|||
pendingKeys: new Map(),
|
||||
debounceTimer: null,
|
||||
debounceMs: options.debounceMs ?? 300,
|
||||
maxWaitMs: options.maxWaitMs,
|
||||
maxWaitTimer: null,
|
||||
resolveInvalidation: options.resolveInvalidation as (payload: EventPayload) => EventInvalidation,
|
||||
resyncKeys: options.resyncKeys,
|
||||
};
|
||||
|
|
@ -304,6 +310,7 @@ export class CrossTabSseCoordinator {
|
|||
options.resolveInvalidation as (payload: EventPayload) => EventInvalidation;
|
||||
subscription.resyncKeys = options.resyncKeys;
|
||||
subscription.debounceMs = options.debounceMs ?? subscription.debounceMs;
|
||||
subscription.maxWaitMs = options.maxWaitMs ?? subscription.maxWaitMs;
|
||||
}
|
||||
|
||||
subscription.refcount += 1;
|
||||
|
|
@ -702,6 +709,12 @@ export class CrossTabSseCoordinator {
|
|||
return;
|
||||
}
|
||||
|
||||
if (subscription.maxWaitMs !== undefined && subscription.maxWaitTimer === null) {
|
||||
subscription.maxWaitTimer = setTimeout(
|
||||
() => this.flushInvalidations(subscription),
|
||||
subscription.maxWaitMs,
|
||||
);
|
||||
}
|
||||
if (subscription.debounceTimer) {
|
||||
clearTimeout(subscription.debounceTimer);
|
||||
}
|
||||
|
|
@ -712,6 +725,7 @@ export class CrossTabSseCoordinator {
|
|||
}
|
||||
|
||||
private flushInvalidations(subscription: LocalSubscription) {
|
||||
this.clearSubscriptionTimer(subscription);
|
||||
if (subscription.pendingKeys.size === 0) return;
|
||||
const keys = [...subscription.pendingKeys.values()];
|
||||
subscription.pendingKeys.clear();
|
||||
|
|
@ -940,9 +954,10 @@ export class CrossTabSseCoordinator {
|
|||
}
|
||||
|
||||
private clearSubscriptionTimer(subscription: LocalSubscription) {
|
||||
if (!subscription.debounceTimer) return;
|
||||
clearTimeout(subscription.debounceTimer);
|
||||
if (subscription.debounceTimer !== null) clearTimeout(subscription.debounceTimer);
|
||||
if (subscription.maxWaitTimer !== null) clearTimeout(subscription.maxWaitTimer);
|
||||
subscription.debounceTimer = null;
|
||||
subscription.maxWaitTimer = null;
|
||||
}
|
||||
|
||||
private resetIdleState() {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import type { Key } from "swr";
|
||||
import { describe, expect, jest, test } from "bun:test";
|
||||
import { unstable_serialize, type Key } from "swr";
|
||||
|
||||
import { loadPetriFixture } from "./petri-fixtures";
|
||||
import { streamItemName } from "./petri-stream";
|
||||
|
|
@ -431,3 +431,94 @@ async function waitFor(condition: () => boolean, timeoutMs = 200) {
|
|||
}
|
||||
throw new Error("condition did not become true before timeout");
|
||||
}
|
||||
|
||||
for (const mode of ["coordinated", "fallback"] as const) {
|
||||
describe(`${mode} live usage batching`, () => {
|
||||
function subscribe() {
|
||||
const source = new FakeEventSource();
|
||||
const coordinator = mode === "coordinated"
|
||||
? createCoordinator(() => source)
|
||||
: createFallbackCoordinator();
|
||||
const keys: Key[] = [];
|
||||
const usageKey = unstable_serialize(queryKeys.runs.usage("run-live"));
|
||||
const cleanup = subscribeToRunEvents("run-live", (key) => {
|
||||
keys.push(key);
|
||||
return Promise.resolve();
|
||||
}, () => source, { coordinator });
|
||||
jest.advanceTimersByTime(100);
|
||||
expect(source.onmessage).not.toBeNull();
|
||||
keys.length = 0;
|
||||
let seq = 0;
|
||||
return {
|
||||
source, keys, cleanup,
|
||||
close: () => { cleanup(); coordinator.close(); },
|
||||
usageRefreshes: () => keys.filter((key) => unstable_serialize(key) === usageKey).length,
|
||||
progress: () => source.emit(makePetriItem(++seq, { event: "step.progress.recorded" }, {
|
||||
run_id: "run-live", stage: { name: "agent" },
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
test("refreshes bursts after 300ms and continuous progress at least once per second", () => {
|
||||
jest.useFakeTimers();
|
||||
const subscription = subscribe();
|
||||
try {
|
||||
subscription.progress();
|
||||
subscription.progress();
|
||||
jest.advanceTimersByTime(299);
|
||||
expect(subscription.usageRefreshes()).toBe(0);
|
||||
jest.advanceTimersByTime(1);
|
||||
expect(subscription.usageRefreshes()).toBe(1);
|
||||
// No new event means no polling or leftover maximum-wait flush.
|
||||
jest.advanceTimersByTime(1_000);
|
||||
expect(subscription.usageRefreshes()).toBe(1);
|
||||
|
||||
for (let batch = 0; batch < 2; batch += 1) {
|
||||
for (let event = 0; event < 10; event += 1) {
|
||||
subscription.progress();
|
||||
expect(subscription.usageRefreshes()).toBe(batch + 1);
|
||||
jest.advanceTimersByTime(100);
|
||||
}
|
||||
expect(subscription.usageRefreshes()).toBe(batch + 2);
|
||||
}
|
||||
} finally {
|
||||
subscription.close();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("flushes terminal usage immediately without a later duplicate refresh", () => {
|
||||
jest.useFakeTimers();
|
||||
const subscription = subscribe();
|
||||
try {
|
||||
subscription.progress();
|
||||
subscription.source.emit(makePlatformItem(2, {
|
||||
kind: "run.lifecycle", transition: "succeeded", status: { kind: "succeeded" },
|
||||
}, { run_id: "run-live" }));
|
||||
expect(subscription.usageRefreshes()).toBe(1);
|
||||
const count = subscription.keys.length;
|
||||
jest.advanceTimersByTime(1_000);
|
||||
expect(subscription.keys).toHaveLength(count);
|
||||
expect(subscription.source.closed).toBe(mode === "fallback");
|
||||
} finally {
|
||||
subscription.close();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("unsubscribing discards both pending timers", () => {
|
||||
jest.useFakeTimers();
|
||||
const subscription = subscribe();
|
||||
try {
|
||||
subscription.progress();
|
||||
subscription.cleanup();
|
||||
jest.advanceTimersByTime(2_000);
|
||||
expect(subscription.keys).toEqual([]);
|
||||
expect(subscription.source.closed).toBe(true);
|
||||
} finally {
|
||||
subscription.close();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ export interface RunEventPayload extends EventPayload {
|
|||
|
||||
interface RunEventOptions {
|
||||
debounceMs?: number;
|
||||
/** Keep live queries updating even when progress never pauses. */
|
||||
maxWaitMs?: number;
|
||||
coordinator?: CrossTabSseCoordinator;
|
||||
/** Called with every well-formed stream item of the run, before invalidation. */
|
||||
onItem?: (item: RunStreamItem) => void;
|
||||
|
|
@ -131,7 +133,7 @@ export function queryKeysForStreamItem(
|
|||
immediate: false,
|
||||
};
|
||||
}
|
||||
return { keys: [queryKeys.runs.state(runId), stream, ...stageKeys], immediate: false };
|
||||
return { keys: [queryKeys.runs.state(runId), queryKeys.runs.usage(runId), stream, ...stageKeys], immediate: false };
|
||||
}
|
||||
case "control.requested":
|
||||
return {
|
||||
|
|
@ -196,13 +198,14 @@ export function subscribeToRunEvents(
|
|||
runId: string,
|
||||
mutate: MutateFn,
|
||||
eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource,
|
||||
{ debounceMs = 300, coordinator, onItem }: RunEventOptions = {},
|
||||
{ debounceMs = 300, maxWaitMs = 1_000, coordinator, onItem }: RunEventOptions = {},
|
||||
): () => void {
|
||||
return subscribeToCrossTabSse<RunEventPayload>({
|
||||
coordinator,
|
||||
subscriptionKey: `run:${runId}`,
|
||||
mutate,
|
||||
debounceMs,
|
||||
maxWaitMs,
|
||||
resyncKeys: () => resyncKeysForRun(runId),
|
||||
resolveInvalidation: (payload) => {
|
||||
if (payload.run_id !== runId) return { keys: [] };
|
||||
|
|
@ -216,6 +219,7 @@ export function subscribeToRunEvents(
|
|||
mutate,
|
||||
eventSourceFactory,
|
||||
debounceMs,
|
||||
maxWaitMs,
|
||||
resolveInvalidation: (payload) => {
|
||||
const result = runInvalidation(runId, payload, onItem);
|
||||
return { ...result, close: result.immediate };
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ export interface SharedEventSubscription {
|
|||
resolvers: Map<symbol, EventResolver>;
|
||||
pendingKeys: Map<string, KeyOrMatcher>;
|
||||
debounceTimer: ReturnType<typeof setTimeout> | null;
|
||||
maxWaitTimer: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
|
||||
export function sseKeyDedupeId(key: KeyOrMatcher): string {
|
||||
|
|
@ -47,6 +48,7 @@ export function subscribeToSharedEventSource<TPayload extends EventPayload>({
|
|||
resolveInvalidation,
|
||||
eventSourceFactory = createBrowserEventSource,
|
||||
debounceMs = 300,
|
||||
maxWaitMs,
|
||||
}: {
|
||||
subscriptions: Map<string, SharedEventSubscription>;
|
||||
subscriptionKey: string;
|
||||
|
|
@ -55,6 +57,8 @@ export function subscribeToSharedEventSource<TPayload extends EventPayload>({
|
|||
resolveInvalidation: (payload: TPayload) => EventInvalidation;
|
||||
eventSourceFactory?: (url: string) => EventSourceLike;
|
||||
debounceMs?: number;
|
||||
/** Flush pending keys within this interval, even while events keep arriving. */
|
||||
maxWaitMs?: number;
|
||||
}): () => void {
|
||||
let subscription = subscriptions.get(subscriptionKey);
|
||||
if (!subscription) {
|
||||
|
|
@ -66,6 +70,7 @@ export function subscribeToSharedEventSource<TPayload extends EventPayload>({
|
|||
resolvers: new Map(),
|
||||
pendingKeys: new Map(),
|
||||
debounceTimer: null,
|
||||
maxWaitTimer: null,
|
||||
};
|
||||
subscriptions.set(subscriptionKey, subscription);
|
||||
|
||||
|
|
@ -92,7 +97,7 @@ export function subscribeToSharedEventSource<TPayload extends EventPayload>({
|
|||
immediate ||= Boolean(invalidation.immediate);
|
||||
}
|
||||
|
||||
queueInvalidations(current, [...keys.values()], { debounceMs, immediate });
|
||||
queueInvalidations(current, [...keys.values()], { debounceMs, maxWaitMs, immediate });
|
||||
|
||||
if (close) {
|
||||
closeSharedEventSource(subscriptions, subscriptionKey, { flushPending: true });
|
||||
|
|
@ -133,9 +138,11 @@ function queueInvalidations(
|
|||
keys: KeyOrMatcher[],
|
||||
{
|
||||
debounceMs,
|
||||
maxWaitMs,
|
||||
immediate,
|
||||
}: {
|
||||
debounceMs: number;
|
||||
maxWaitMs?: number;
|
||||
immediate?: boolean;
|
||||
},
|
||||
) {
|
||||
|
|
@ -149,6 +156,9 @@ function queueInvalidations(
|
|||
return;
|
||||
}
|
||||
|
||||
if (maxWaitMs !== undefined && subscription.maxWaitTimer === null) {
|
||||
subscription.maxWaitTimer = setTimeout(() => flushInvalidations(subscription), maxWaitMs);
|
||||
}
|
||||
if (subscription.debounceTimer) {
|
||||
clearTimeout(subscription.debounceTimer);
|
||||
}
|
||||
|
|
@ -159,6 +169,7 @@ function queueInvalidations(
|
|||
}
|
||||
|
||||
function flushInvalidations(subscription: SharedEventSubscription) {
|
||||
clearInvalidationTimers(subscription);
|
||||
if (subscription.pendingKeys.size === 0) return;
|
||||
const keys = [...subscription.pendingKeys.values()];
|
||||
subscription.pendingKeys.clear();
|
||||
|
|
@ -181,13 +192,18 @@ function closeSharedEventSource(
|
|||
if (flushPending) {
|
||||
flushInvalidations(subscription);
|
||||
}
|
||||
if (subscription.debounceTimer) {
|
||||
clearTimeout(subscription.debounceTimer);
|
||||
}
|
||||
clearInvalidationTimers(subscription);
|
||||
subscription.source.close();
|
||||
subscriptions.delete(subscriptionKey);
|
||||
}
|
||||
|
||||
function clearInvalidationTimers(subscription: SharedEventSubscription) {
|
||||
if (subscription.debounceTimer !== null) clearTimeout(subscription.debounceTimer);
|
||||
if (subscription.maxWaitTimer !== null) clearTimeout(subscription.maxWaitTimer);
|
||||
subscription.debounceTimer = null;
|
||||
subscription.maxWaitTimer = null;
|
||||
}
|
||||
|
||||
function stringifyKeyValue(value: unknown): string {
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((item) => stringifyKeyValue(item)).join(",")}]`;
|
||||
|
|
|
|||
102
apps/fabro-web/app/routes/run-usage.live.test.tsx
Normal file
102
apps/fabro-web/app/routes/run-usage.live.test.tsx
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import { expect, jest, spyOn, test } from "bun:test";
|
||||
import TestRenderer, { act } from "react-test-renderer";
|
||||
import { SWRConfig, useSWRConfig } from "swr";
|
||||
import { AxiosHeaders } from "axios";
|
||||
import type { RunUsage } from "@qltysh/fabro-api-client";
|
||||
|
||||
import { runOutputsApi } from "../lib/api-client";
|
||||
import { createCrossTabSseCoordinator } from "../lib/cross-tab-sse";
|
||||
import { subscribeToRunEvents } from "../lib/run-events";
|
||||
import type { EventSourceLike, MutateFn } from "../lib/sse";
|
||||
import { makeUsage } from "../lib/test-fixtures";
|
||||
import { makePebbleItem, setupReactTestEnv, textContent } from "../lib/test-utils";
|
||||
import RunUsageRoute from "./run-usage";
|
||||
|
||||
function usage(responses: number): RunUsage {
|
||||
const model = { provider: "anthropic", model_id: "claude-sonnet-5" };
|
||||
const totals = makeUsage({ input: responses * 10_000, output: responses * 8_000 }, responses * 108_500);
|
||||
const timing = { wall_time_ms: 0, inference_time_ms: 0, tool_time_ms: 0, active_time_ms: 0 };
|
||||
return {
|
||||
stages: [{
|
||||
stage: { id: "agent", name: "agent" }, model, usage: totals, timing,
|
||||
state: "running", started_at: "2026-10-08T16:57:29Z",
|
||||
}],
|
||||
by_model: [{ model, stages: 1, usage: totals }],
|
||||
totals: { usage: totals, timing },
|
||||
};
|
||||
}
|
||||
|
||||
for (const mode of ["coordinated", "fallback"] as const) {
|
||||
test(`${mode} progress updates the mounted Usage page through its real query cache`, async () => {
|
||||
jest.useFakeTimers();
|
||||
const teardown = setupReactTestEnv();
|
||||
let remote = usage(1);
|
||||
const retrieve = spyOn(runOutputsApi, "retrieveRunUsage").mockImplementation(async () => ({
|
||||
data: remote, status: 200, statusText: "OK", headers: new AxiosHeaders(),
|
||||
config: { headers: new AxiosHeaders() },
|
||||
}));
|
||||
const source: EventSourceLike = { onmessage: null, close() {} };
|
||||
const coordinator = createCrossTabSseCoordinator({
|
||||
channelFactory: () => {
|
||||
if (mode === "fallback") throw new Error("BroadcastChannel unavailable");
|
||||
return { onmessage: null, postMessage() {}, close() {} };
|
||||
},
|
||||
eventSourceFactory: () => source,
|
||||
addVisibilityChangeListener: () => () => {},
|
||||
addPagehideListener: () => () => {},
|
||||
timing: { electionJitterMs: 0 },
|
||||
});
|
||||
let mutate!: MutateFn;
|
||||
function CacheAccess() {
|
||||
mutate = useSWRConfig().mutate as MutateFn;
|
||||
return null;
|
||||
}
|
||||
let renderer: TestRenderer.ReactTestRenderer | undefined;
|
||||
let cleanup: (() => void) | undefined;
|
||||
try {
|
||||
await act(async () => {
|
||||
renderer = TestRenderer.create(
|
||||
<SWRConfig value={{
|
||||
provider: () => new Map(), dedupingInterval: 0,
|
||||
revalidateOnFocus: false, revalidateOnReconnect: false, shouldRetryOnError: false,
|
||||
}}>
|
||||
<CacheAccess />
|
||||
<RunUsageRoute params={{ id: "run-1" }} />
|
||||
</SWRConfig>,
|
||||
);
|
||||
});
|
||||
expect(retrieve).toHaveBeenCalledTimes(1);
|
||||
expect(textContent(renderer!.root)).toContain("$0.11");
|
||||
await act(async () => {
|
||||
cleanup = subscribeToRunEvents("run-1", mutate, () => source, { coordinator });
|
||||
jest.advanceTimersByTime(100);
|
||||
});
|
||||
expect(source.onmessage).not.toBeNull();
|
||||
const fetchesBeforeProgress = retrieve.mock.calls.length;
|
||||
remote = usage(2);
|
||||
await act(async () => {
|
||||
source.onmessage!({ data: JSON.stringify(makePebbleItem(
|
||||
1, "2026-10-08T16:57:43Z", { name: "agent" }, "AssistantMessage", {},
|
||||
)) });
|
||||
jest.advanceTimersByTime(300);
|
||||
});
|
||||
expect(retrieve).toHaveBeenCalledTimes(fetchesBeforeProgress + 1);
|
||||
// Stage row, page total, model row and model total all use the new snapshot.
|
||||
const rows = renderer!.root.findAllByType("tr").filter((row) => textContent(row).includes("$0.22"));
|
||||
expect(rows).toHaveLength(4);
|
||||
for (const row of rows) {
|
||||
expect(textContent(row)).toContain("20.0k");
|
||||
expect(textContent(row)).toContain("16.0k");
|
||||
}
|
||||
expect(textContent(renderer!.root)).not.toContain("$0.11");
|
||||
expect(remote.stages[0].state).toBe("running");
|
||||
} finally {
|
||||
cleanup?.();
|
||||
coordinator.close();
|
||||
act(() => renderer?.unmount());
|
||||
retrieve.mockRestore();
|
||||
teardown();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -310,22 +310,19 @@ function ModelsSection({ providers }: { providers: Provider[] }) {
|
|||
modelsApi.testModel(...modelOfferingTestArgs(offering)),
|
||||
);
|
||||
if (
|
||||
result.provider !== offering.provider ||
|
||||
result.model_id !== offering.id
|
||||
result.route !== null &&
|
||||
result.route !== `${offering.provider}/${offering.id}`
|
||||
) {
|
||||
outcome = {
|
||||
phase: "error",
|
||||
message: `Server tested unexpected offering ${result.provider}/${result.model_id}`,
|
||||
phase: "error",
|
||||
message: `Server tested unexpected offering ${result.route}`,
|
||||
};
|
||||
} else if (result.status === "ok") {
|
||||
} else if (result.outcome.status === "passed") {
|
||||
outcome = { phase: "ok" };
|
||||
} else if (result.status === "error") {
|
||||
outcome = {
|
||||
phase: "error",
|
||||
message: result.error_message ?? "Failed",
|
||||
};
|
||||
} else if (result.outcome.status === "failed") {
|
||||
outcome = { phase: "error", message: result.outcome.error.message };
|
||||
} else {
|
||||
outcome = { phase: "error", message: "provider not configured" };
|
||||
outcome = { phase: "error", message: result.outcome.detail };
|
||||
}
|
||||
} catch (err) {
|
||||
outcome = {
|
||||
|
|
|
|||
|
|
@ -5911,24 +5911,28 @@ paths:
|
|||
operationId: testModel
|
||||
tags: [Models]
|
||||
summary: Test Model
|
||||
description: Tests a model by sending a simple prompt and reporting pass/fail.
|
||||
description: >-
|
||||
Runs the lithos-llm probe and returns its versioned report, including the
|
||||
canonical route, outcome, elapsed time and token usage. Unknown or
|
||||
unavailable models and provider failures produce HTTP 200 reports with
|
||||
a failed outcome. Invalid query options remain HTTP 400 errors.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
description: The canonical model ID or an alias.
|
||||
description: A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
|
||||
- $ref: "#/components/parameters/ModelTestProviderParam"
|
||||
- $ref: "#/components/parameters/ModelTestModeParam"
|
||||
- $ref: "#/components/parameters/ModelTestReasoningEffortParam"
|
||||
responses:
|
||||
"200":
|
||||
description: Test result
|
||||
description: Probe report (passed, failed or incorrect)
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ModelTestResult"
|
||||
$ref: "#/components/schemas/ProbeReport"
|
||||
"400":
|
||||
description: Invalid test mode or reasoning effort
|
||||
headers:
|
||||
|
|
@ -5938,15 +5942,6 @@ paths:
|
|||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
"404":
|
||||
description: Model not found
|
||||
headers:
|
||||
x-request-id:
|
||||
$ref: "#/components/headers/XRequestId"
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
|
||||
/api/v1/providers:
|
||||
get:
|
||||
|
|
@ -8875,30 +8870,99 @@ components:
|
|||
key. Does NOT imply authentication or model requests will succeed;
|
||||
call `POST /models/{id}/test` to test usability.
|
||||
|
||||
ModelTestResult:
|
||||
description: Result of testing a model in `basic` or `deep` mode.
|
||||
ProbeReport:
|
||||
description: >-
|
||||
The lithos-llm version-1 probe report. The route is the canonical
|
||||
provider/model selected by the probe, or null if selection failed.
|
||||
Probe failures are findings in a successful HTTP response; API
|
||||
authentication, malformed options and server setup failures remain HTTP errors.
|
||||
Provider errors are redacted at the server disclosure boundary.
|
||||
type: object
|
||||
required:
|
||||
- model_id
|
||||
- provider
|
||||
- status
|
||||
required: [version, route, outcome, latency_ms, usage]
|
||||
properties:
|
||||
model_id:
|
||||
version:
|
||||
type: integer
|
||||
enum: [1]
|
||||
route:
|
||||
type: [string, "null"]
|
||||
pattern: '^[^/]+/.+$'
|
||||
outcome:
|
||||
$ref: "#/components/schemas/ProbeOutcome"
|
||||
latency_ms:
|
||||
type: integer
|
||||
format: uint64
|
||||
minimum: 0
|
||||
usage:
|
||||
$ref: "#/components/schemas/TokenCounts"
|
||||
|
||||
ProbeOutcome:
|
||||
oneOf:
|
||||
- $ref: "#/components/schemas/ProbePassed"
|
||||
- $ref: "#/components/schemas/ProbeFailed"
|
||||
- $ref: "#/components/schemas/ProbeIncorrect"
|
||||
discriminator:
|
||||
propertyName: status
|
||||
mapping:
|
||||
passed: "#/components/schemas/ProbePassed"
|
||||
failed: "#/components/schemas/ProbeFailed"
|
||||
incorrect: "#/components/schemas/ProbeIncorrect"
|
||||
|
||||
ProbePassed:
|
||||
type: object
|
||||
required: [status]
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
enum: [passed]
|
||||
|
||||
ProbeFailed:
|
||||
type: object
|
||||
required: [status, error]
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
enum: [failed]
|
||||
error:
|
||||
$ref: "#/components/schemas/LlmErrorData"
|
||||
|
||||
ProbeIncorrect:
|
||||
type: object
|
||||
required: [status, detail]
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
enum: [incorrect]
|
||||
detail:
|
||||
type: string
|
||||
|
||||
LlmErrorData:
|
||||
description: Structured lithos-llm failure, subject to the server's disclosure policy.
|
||||
type: object
|
||||
required: [kind, message, retry]
|
||||
properties:
|
||||
kind:
|
||||
$ref: "#/components/schemas/LlmErrorKind"
|
||||
message:
|
||||
type: string
|
||||
description: The canonical model ID that was tested.
|
||||
example: "claude-opus-4-6"
|
||||
provider:
|
||||
$ref: "#/components/schemas/ProviderId"
|
||||
status:
|
||||
type: integer
|
||||
minimum: 0
|
||||
maximum: 65535
|
||||
provider_code:
|
||||
type: string
|
||||
enum:
|
||||
- ok
|
||||
- error
|
||||
- skip
|
||||
description: Whether the model responded successfully, failed, or was skipped because its provider is not configured.
|
||||
error_message:
|
||||
type: ["string", "null"]
|
||||
description: Error details when status is "error".
|
||||
retry:
|
||||
$ref: "#/components/schemas/LlmRetryClassification"
|
||||
raw_data:
|
||||
description: Provider response data; omitted by the model-test endpoint.
|
||||
provider_retry_after_millis:
|
||||
type: integer
|
||||
format: uint64
|
||||
minimum: 0
|
||||
source_message:
|
||||
type: string
|
||||
description: Internal source detail; omitted by the model-test endpoint.
|
||||
|
||||
ModelTestMode:
|
||||
description: Single-model test mode.
|
||||
|
|
|
|||
|
|
@ -344,3 +344,24 @@ fabro model test --provider openai
|
|||
```
|
||||
|
||||
This is useful for confirming connectivity after setup or when adding a new provider key.
|
||||
|
||||
With `--model`, Fabro sends the selector directly to the server's lithos-llm
|
||||
probe. Normal output retains the model table: model, provider, aliases, context,
|
||||
cost, speed, and result. Catalog metadata is fetched only for presentation;
|
||||
unavailable or missing metadata appears as `—` and does not change the probe's
|
||||
result. Add `--tools` to check a multi-turn tool exchange. Timing and token usage
|
||||
remain available in `--json`, which does not fetch catalog metadata.
|
||||
|
||||
`fabro model test --model <selector> --json` returns the version-1 lithos-llm
|
||||
probe report directly: `version`, `route`, `outcome`, `latency_ms`, and `usage`.
|
||||
The outcome is `passed`, `failed` (with a structured `error`), or `incorrect`
|
||||
(with a `detail` explaining the model's behavior). A selection failure has a
|
||||
null route. Failed and incorrect probes print their report and exit nonzero.
|
||||
This replaces the previous single-model JSON summary and `ok/error/skip` API
|
||||
result.
|
||||
|
||||
Without `--model`, Fabro lists models and tests configured offerings with the
|
||||
requested concurrency. Bulk JSON retains `total`, `failures`, `skipped`, and
|
||||
`results`; each completed probe's row also contains its full `report`.
|
||||
Unconfigured offerings are skipped before probing. An offering that becomes
|
||||
unavailable after listing is a failed probe.
|
||||
|
|
|
|||
18
installer/push-homebrew.sh
Normal file
18
installer/push-homebrew.sh
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Run from the tap checkout after committing the formula and versions.json.
|
||||
# Retry the same commit: if GitHub accepted it but lost the response, the next
|
||||
# push is a no-op. Never force or rebase over another publisher's update.
|
||||
for delay in 0 10 30; do
|
||||
if (( delay > 0 )); then
|
||||
echo "Homebrew push failed; retrying in ${delay}s" >&2
|
||||
sleep "$delay"
|
||||
fi
|
||||
if git push origin HEAD:refs/heads/main; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "::error::Homebrew publication failed after 3 push attempts" >&2
|
||||
exit 1
|
||||
131
installer/tests/test_push_homebrew.py
Normal file
131
installer/tests/test_push_homebrew.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
"""Exercise publication against local Git repositories; no network or credentials."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "push-homebrew.sh"
|
||||
REAL_GIT = shutil.which("git")
|
||||
|
||||
|
||||
class HomebrewPushTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.tap = self.root / "tap"
|
||||
self.remote = self.root / "remote.git"
|
||||
self.bin = self.root / "bin"
|
||||
self.bin.mkdir()
|
||||
self.env = {
|
||||
**os.environ,
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_CONFIG_GLOBAL": os.devnull,
|
||||
"GIT_TERMINAL_PROMPT": "0",
|
||||
"REAL_GIT": REAL_GIT,
|
||||
"TEST_ROOT": str(self.root),
|
||||
}
|
||||
for key in list(self.env):
|
||||
if key.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_")) or key in (
|
||||
"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_CONFIG_COUNT"
|
||||
):
|
||||
del self.env[key]
|
||||
self.git("init", "--bare", "--initial-branch=main", str(self.remote))
|
||||
self.git("init", "--initial-branch=main", str(self.tap))
|
||||
self.git("config", "user.name", "Test")
|
||||
self.git("config", "user.email", "test@example.invalid")
|
||||
self.git("remote", "add", "origin", str(self.remote))
|
||||
self.commit("old version")
|
||||
self.git("push", "origin", "HEAD:main")
|
||||
self.before = self.git("rev-parse", "HEAD")
|
||||
self.commit("new version")
|
||||
self.expected = self.git("rev-parse", "HEAD")
|
||||
# Keep tests fast and record the backoff without changing production code.
|
||||
self.write_executable(self.bin / "sleep", '#!/bin/sh\necho "$1" >> "$TEST_ROOT/delays"\n')
|
||||
self.write_executable(self.bin / "git", '''#!/bin/sh
|
||||
printf 'attempt\n' >> "$TEST_ROOT/attempts"
|
||||
"$REAL_GIT" "$@"
|
||||
status=$?
|
||||
if [ "$status" -eq 0 ] && [ -f "$TEST_ROOT/lose-response" ]; then
|
||||
rm "$TEST_ROOT/lose-response"
|
||||
exit 1
|
||||
fi
|
||||
exit "$status"
|
||||
''')
|
||||
|
||||
def git(self, *args):
|
||||
result = subprocess.run(
|
||||
[REAL_GIT, *args], cwd=self.tap if self.tap.exists() else self.root,
|
||||
env=self.env, text=True, capture_output=True, check=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
def commit(self, value):
|
||||
(self.tap / "formula").write_text(value)
|
||||
self.git("add", "formula")
|
||||
self.git("commit", "-m", value)
|
||||
|
||||
def write_executable(self, path, content):
|
||||
path.write_text(content)
|
||||
path.chmod(0o755)
|
||||
|
||||
def reject_pushes(self, failures):
|
||||
self.write_executable(self.remote / "hooks" / "pre-receive", f'''#!/bin/sh
|
||||
count=0
|
||||
if [ -f reject-count ]; then count=$(cat reject-count); fi
|
||||
count=$((count + 1))
|
||||
echo "$count" > reject-count
|
||||
if [ "$count" -le {failures} ]; then
|
||||
echo "Internal Server Error" >&2
|
||||
exit 1
|
||||
fi
|
||||
''')
|
||||
|
||||
def run_push(self, success, attempts, delays):
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT)], cwd=self.tap, text=True, capture_output=True,
|
||||
env={**self.env, "PATH": str(self.bin) + os.pathsep + os.environ["PATH"]},
|
||||
timeout=10,
|
||||
)
|
||||
self.assertEqual(result.returncode == 0, success, result.stderr)
|
||||
self.assertEqual((self.root / "attempts").read_text().splitlines(), ["attempt"] * attempts)
|
||||
delay_file = self.root / "delays"
|
||||
self.assertEqual(delay_file.read_text().splitlines() if delay_file.exists() else [], delays)
|
||||
return self.git("--git-dir", str(self.remote), "rev-parse", "main")
|
||||
|
||||
def test_success_needs_no_retry(self):
|
||||
self.assertEqual(self.run_push(True, 1, []), self.expected)
|
||||
|
||||
def test_transient_server_rejection_recovers(self):
|
||||
self.reject_pushes(2)
|
||||
self.assertEqual(self.run_push(True, 3, ["10", "30"]), self.expected)
|
||||
|
||||
def test_persistent_failure_remains_failure(self):
|
||||
self.reject_pushes(99)
|
||||
self.assertEqual(self.run_push(False, 3, ["10", "30"]), self.before)
|
||||
|
||||
def test_lost_success_response_is_idempotent(self):
|
||||
(self.root / "lose-response").touch()
|
||||
self.assertEqual(self.run_push(True, 2, ["10"]), self.expected)
|
||||
self.assertEqual(self.git("rev-parse", "HEAD"), self.expected)
|
||||
|
||||
def test_already_published_commit_is_noop(self):
|
||||
self.git("push", "origin", "HEAD:main")
|
||||
self.assertEqual(self.run_push(True, 1, []), self.expected)
|
||||
|
||||
def test_concurrent_publication_is_not_overwritten(self):
|
||||
self.git("checkout", "--detach", self.before)
|
||||
self.commit("competing version")
|
||||
competing = self.git("rev-parse", "HEAD")
|
||||
self.git("push", "origin", "HEAD:main")
|
||||
self.git("checkout", "main")
|
||||
self.assertEqual(self.run_push(False, 3, ["10", "30"]), competing)
|
||||
self.assertEqual(self.git("rev-parse", "HEAD"), self.expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use anyhow::{Result, bail};
|
||||
use cli_table::format::{Border, Justify, Separator};
|
||||
use cli_table::{Cell, CellStruct, Color, Style, Table};
|
||||
use fabro_api::types as api_types;
|
||||
use fabro_types::{Model, ModelTestMode};
|
||||
use fabro_util::terminal::Styles;
|
||||
use futures::{StreamExt, stream};
|
||||
use lithos_llm::catalog::ProviderId;
|
||||
use lithos_llm::client::{ProbeOutcome, ProbeReport};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::args::{ModelListArgs, ModelTestArgs, ModelsCommand};
|
||||
|
|
@ -29,6 +29,8 @@ struct ModelTestRow {
|
|||
detail: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
report: Option<ProbeReport>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -44,24 +46,7 @@ struct CompletedModelTest {
|
|||
model: Model,
|
||||
result_color: Color,
|
||||
status: String,
|
||||
}
|
||||
|
||||
fn model_matches_selector(model: &Model, selector: &str) -> bool {
|
||||
model.id.as_str() == selector || model.aliases.iter().any(|alias| alias == selector)
|
||||
}
|
||||
|
||||
fn find_model_by_id_or_alias(
|
||||
models: &[Model],
|
||||
id: &str,
|
||||
provider: Option<&ProviderId>,
|
||||
) -> Option<Model> {
|
||||
models
|
||||
.iter()
|
||||
.find(|model| {
|
||||
provider.is_none_or(|provider| &model.provider == provider)
|
||||
&& model_matches_selector(model, id)
|
||||
})
|
||||
.cloned()
|
||||
report: Option<ProbeReport>,
|
||||
}
|
||||
|
||||
pub(crate) async fn execute(
|
||||
|
|
@ -183,34 +168,62 @@ fn print_models_table(models: &[Model], styles: &Styles) {
|
|||
);
|
||||
}
|
||||
|
||||
fn configured_model_test_status(
|
||||
expected: &Model,
|
||||
result: Result<api_types::ModelTestResult>,
|
||||
) -> (Color, String, bool) {
|
||||
match result {
|
||||
Ok(resp) if resp.provider != expected.provider || resp.model_id != expected.id.as_str() => {
|
||||
(
|
||||
Color::Red,
|
||||
format!(
|
||||
"error: server tested unexpected offering {}/{}",
|
||||
resp.provider, resp.model_id
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => {
|
||||
(Color::Green, "ok".to_string(), false)
|
||||
}
|
||||
Ok(resp) if resp.status == api_types::ModelTestResultStatus::Skip => (
|
||||
/// Catalog data decorates the report; it never selects a model or changes the
|
||||
/// finding.
|
||||
async fn probe_display_metadata(
|
||||
client: &server_client::Client,
|
||||
report: &ProbeReport,
|
||||
selector: &str,
|
||||
provider: Option<&str>,
|
||||
) -> Option<Model> {
|
||||
let (provider, query) = report.route.as_ref().map_or((provider, selector), |route| {
|
||||
(Some(route.provider().as_str()), route.model().as_str())
|
||||
});
|
||||
// Missing or unavailable metadata must not turn a successful probe into a
|
||||
// failure.
|
||||
let models = client.list_models(provider, Some(query)).await.ok()?;
|
||||
let mut matches = models.into_iter().filter(|model| {
|
||||
provider.is_none_or(|provider| model.provider.as_str() == provider)
|
||||
&& (model.id.as_str() == query
|
||||
|| (report.route.is_none() && model.aliases.iter().any(|alias| alias == query)))
|
||||
});
|
||||
let model = matches.next()?;
|
||||
// An unresolved selector can name multiple offerings. Do not invent a selected
|
||||
// route.
|
||||
matches.next().is_none().then_some(model)
|
||||
}
|
||||
|
||||
fn probe_status(report: &ProbeReport) -> (Color, String, bool) {
|
||||
match &report.outcome {
|
||||
ProbeOutcome::Passed => (Color::Green, "ok".to_string(), false),
|
||||
ProbeOutcome::Failed(error) => (
|
||||
Color::Red,
|
||||
"error: provider became unconfigured after listing".to_string(),
|
||||
format!("error: {}: {}", error.kind.as_str(), error.message),
|
||||
true,
|
||||
),
|
||||
Ok(resp) => {
|
||||
let message = resp
|
||||
.error_message
|
||||
.unwrap_or_else(|| "unknown error".to_string());
|
||||
(Color::Red, format!("error: {message}"), true)
|
||||
ProbeOutcome::Incorrect { detail } => (Color::Red, format!("incorrect: {detail}"), true),
|
||||
_ => (Color::Red, "error: unknown probe outcome".to_string(), true),
|
||||
}
|
||||
}
|
||||
|
||||
fn configured_model_test_status(
|
||||
expected: &Model,
|
||||
result: &Result<ProbeReport>,
|
||||
) -> (Color, String, bool) {
|
||||
match result {
|
||||
Ok(report) => {
|
||||
// A bulk job names an exact offering. Keep this correlation check;
|
||||
// a single selector is resolved exclusively by the server's probe.
|
||||
if let Some(route) = &report.route {
|
||||
if route.provider() != &expected.provider || route.model() != &expected.id {
|
||||
return (
|
||||
Color::Red,
|
||||
format!("error: server tested unexpected offering {route}"),
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
probe_status(report)
|
||||
}
|
||||
Err(err) => (Color::Red, format!("error: {err}"), true),
|
||||
}
|
||||
|
|
@ -225,6 +238,7 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color)
|
|||
result: ModelTestResultKind::Pass,
|
||||
detail: None,
|
||||
error: None,
|
||||
report: None,
|
||||
},
|
||||
Color::Yellow => ModelTestRow {
|
||||
model: model.id.to_string(),
|
||||
|
|
@ -232,12 +246,14 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color)
|
|||
result: ModelTestResultKind::Skip,
|
||||
detail: Some(trimmed.to_string()),
|
||||
error: None,
|
||||
report: None,
|
||||
},
|
||||
_ => ModelTestRow {
|
||||
model: model.id.to_string(),
|
||||
provider: model.provider.clone(),
|
||||
result: ModelTestResultKind::Fail,
|
||||
detail: None,
|
||||
report: None,
|
||||
error: Some(
|
||||
trimmed
|
||||
.strip_prefix("error: ")
|
||||
|
|
@ -265,6 +281,66 @@ async fn test_models_via_server(
|
|||
let reasoning_effort = args.reasoning_effort;
|
||||
let request_mode = args.tools.then_some(ModelTestMode::Deep);
|
||||
|
||||
if let Some(selector) = model {
|
||||
let requested_provider = provider.map(ProviderId::new);
|
||||
if !json_output {
|
||||
eprint!("Testing {selector}...");
|
||||
}
|
||||
let result = client
|
||||
.test_model(
|
||||
selector,
|
||||
requested_provider.as_ref(),
|
||||
request_mode,
|
||||
reasoning_effort,
|
||||
)
|
||||
.await;
|
||||
if !json_output {
|
||||
eprintln!(" done");
|
||||
}
|
||||
let report = result?;
|
||||
if json_output {
|
||||
println!("{}", serde_json::to_string_pretty(&report)?);
|
||||
} else {
|
||||
let use_color = styles.use_color;
|
||||
let metadata = probe_display_metadata(client, &report, selector, provider).await;
|
||||
let mut row = if let Some(metadata) = metadata {
|
||||
model_row(&metadata, use_color)
|
||||
} else {
|
||||
let (model, provider) = report
|
||||
.route
|
||||
.as_ref()
|
||||
.map_or((selector, provider.unwrap_or("—")), |route| {
|
||||
(route.model().as_str(), route.provider().as_str())
|
||||
});
|
||||
vec![
|
||||
model.cell().bold(use_color),
|
||||
provider
|
||||
.cell()
|
||||
.foreground_color(color_if(use_color, Color::Ansi256(8))),
|
||||
"—".cell(),
|
||||
"—".cell().justify(Justify::Right),
|
||||
"—".cell().justify(Justify::Right),
|
||||
"—".cell().justify(Justify::Right),
|
||||
]
|
||||
};
|
||||
let (color, status, _) = probe_status(&report);
|
||||
row.push(status.cell().foreground_color(color_if(use_color, color)));
|
||||
let mut title = models_title(use_color);
|
||||
title.push("RESULT".cell().bold(use_color));
|
||||
let table = vec![row]
|
||||
.table()
|
||||
.title(title)
|
||||
.color_choice(color_choice(use_color))
|
||||
.border(Border::builder().build())
|
||||
.separator(Separator::builder().build());
|
||||
println!("{}", table.display()?);
|
||||
}
|
||||
if !report.passed() {
|
||||
bail!("1 model(s) failed");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let use_color = styles.use_color;
|
||||
let mut title = models_title(use_color);
|
||||
title.push("RESULT".cell().bold(use_color));
|
||||
|
|
@ -274,162 +350,81 @@ async fn test_models_via_server(
|
|||
let mut failures = 0u32;
|
||||
let mut skipped = 0u32;
|
||||
let mut skipped_providers: Vec<String> = Vec::new();
|
||||
if let Some(model_id) = model {
|
||||
let requested_provider = provider.map(ProviderId::new);
|
||||
let listed_models = client.list_models(provider, Some(model_id)).await?;
|
||||
let listed_info = find_model_by_id_or_alias(&listed_models, model_id, None);
|
||||
if !json_output {
|
||||
eprint!("Testing {model_id}...");
|
||||
}
|
||||
let has_configured_match = listed_models
|
||||
.iter()
|
||||
.any(|model| model.configured && model_matches_selector(model, model_id));
|
||||
let result =
|
||||
if requested_provider.is_none() && listed_info.is_some() && !has_configured_match {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
client
|
||||
.test_model(
|
||||
model_id,
|
||||
requested_provider.as_ref(),
|
||||
request_mode,
|
||||
reasoning_effort,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
};
|
||||
if !json_output {
|
||||
eprintln!(" done");
|
||||
}
|
||||
let models_to_test = client.list_models(provider, None).await?;
|
||||
if models_to_test.is_empty() {
|
||||
bail!("No models found");
|
||||
}
|
||||
|
||||
let (info, result_color, status) = match result {
|
||||
None => {
|
||||
let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?;
|
||||
failures += 1;
|
||||
skipped += 1;
|
||||
(info, Color::Yellow, "not configured".to_string())
|
||||
}
|
||||
Some(Ok(resp)) => {
|
||||
let info =
|
||||
find_model_by_id_or_alias(&listed_models, &resp.model_id, Some(&resp.provider))
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Unknown model returned by server: {}/{}",
|
||||
resp.provider, resp.model_id
|
||||
)
|
||||
})?;
|
||||
if resp.status == api_types::ModelTestResultStatus::Ok {
|
||||
(info, Color::Green, "ok".to_string())
|
||||
} else if resp.status == api_types::ModelTestResultStatus::Skip {
|
||||
failures += 1;
|
||||
skipped += 1;
|
||||
(info, Color::Yellow, "not configured".to_string())
|
||||
} else {
|
||||
failures += 1;
|
||||
let message = resp
|
||||
.error_message
|
||||
.unwrap_or_else(|| "unknown error".to_string());
|
||||
(info, Color::Red, format!("error: {message}"))
|
||||
let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test
|
||||
.into_iter()
|
||||
.partition(|model| model.configured);
|
||||
|
||||
for info in &unconfigured {
|
||||
skipped += 1;
|
||||
let provider_name = info.provider.to_string();
|
||||
if !skipped_providers.contains(&provider_name) {
|
||||
skipped_providers.push(provider_name);
|
||||
}
|
||||
if json_output {
|
||||
json_rows.push(model_test_row_from_status(
|
||||
info,
|
||||
"not configured",
|
||||
Color::Yellow,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut completed = stream::iter(configured.into_iter().enumerate())
|
||||
.map(|(index, info)| {
|
||||
let client = client.clone();
|
||||
async move {
|
||||
let result = client
|
||||
.test_model(
|
||||
info.id.as_str(),
|
||||
Some(&info.provider),
|
||||
request_mode,
|
||||
reasoning_effort,
|
||||
)
|
||||
.await;
|
||||
if !json_output {
|
||||
eprintln!("Testing {}... done", info.id);
|
||||
}
|
||||
let (result_color, status, failed) = configured_model_test_status(&info, &result);
|
||||
(
|
||||
CompletedModelTest {
|
||||
index,
|
||||
model: info,
|
||||
result_color,
|
||||
status,
|
||||
report: result.ok(),
|
||||
},
|
||||
failed,
|
||||
)
|
||||
}
|
||||
Some(Err(err)) if err.to_string().contains("Model not found") => {
|
||||
bail!("Unknown model: {model_id}");
|
||||
}
|
||||
Some(Err(err)) => {
|
||||
let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?;
|
||||
failures += 1;
|
||||
(info, Color::Red, format!("error: {err}"))
|
||||
}
|
||||
};
|
||||
})
|
||||
.buffer_unordered(jobs)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let mut row = model_row(&info, use_color);
|
||||
completed.sort_by_key(|(completed, _)| completed.index);
|
||||
|
||||
for (completed, failed) in completed {
|
||||
if failed {
|
||||
failures += 1;
|
||||
}
|
||||
|
||||
let mut row = model_row(&completed.model, use_color);
|
||||
let mut json_row =
|
||||
model_test_row_from_status(&completed.model, &completed.status, completed.result_color);
|
||||
json_row.report = completed.report;
|
||||
json_rows.push(json_row);
|
||||
row.push(
|
||||
status
|
||||
.clone()
|
||||
completed
|
||||
.status
|
||||
.cell()
|
||||
.foreground_color(color_if(use_color, result_color)),
|
||||
.foreground_color(color_if(use_color, completed.result_color)),
|
||||
);
|
||||
rows.push(row);
|
||||
json_rows.push(model_test_row_from_status(&info, &status, result_color));
|
||||
} else {
|
||||
let models_to_test = client.list_models(provider, None).await?;
|
||||
if models_to_test.is_empty() {
|
||||
bail!("No models found");
|
||||
}
|
||||
|
||||
let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test
|
||||
.into_iter()
|
||||
.partition(|model| model.configured);
|
||||
|
||||
for info in &unconfigured {
|
||||
skipped += 1;
|
||||
let provider_name = info.provider.to_string();
|
||||
if !skipped_providers.contains(&provider_name) {
|
||||
skipped_providers.push(provider_name);
|
||||
}
|
||||
if json_output {
|
||||
json_rows.push(model_test_row_from_status(
|
||||
info,
|
||||
"not configured",
|
||||
Color::Yellow,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut completed = stream::iter(configured.into_iter().enumerate())
|
||||
.map(|(index, info)| {
|
||||
let client = client.clone();
|
||||
async move {
|
||||
let result = client
|
||||
.test_model(
|
||||
info.id.as_str(),
|
||||
Some(&info.provider),
|
||||
request_mode,
|
||||
reasoning_effort,
|
||||
)
|
||||
.await;
|
||||
if !json_output {
|
||||
eprintln!("Testing {}... done", info.id);
|
||||
}
|
||||
let (result_color, status, failed) =
|
||||
configured_model_test_status(&info, result);
|
||||
(
|
||||
CompletedModelTest {
|
||||
index,
|
||||
model: info,
|
||||
result_color,
|
||||
status,
|
||||
},
|
||||
failed,
|
||||
)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(jobs)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
completed.sort_by_key(|(completed, _)| completed.index);
|
||||
|
||||
for (completed, failed) in completed {
|
||||
if failed {
|
||||
failures += 1;
|
||||
}
|
||||
|
||||
let mut row = model_row(&completed.model, use_color);
|
||||
json_rows.push(model_test_row_from_status(
|
||||
&completed.model,
|
||||
&completed.status,
|
||||
completed.result_color,
|
||||
));
|
||||
row.push(
|
||||
completed
|
||||
.status
|
||||
.cell()
|
||||
.foreground_color(color_if(use_color, completed.result_color)),
|
||||
);
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
if json_output {
|
||||
|
|
@ -458,7 +453,7 @@ async fn test_models_via_server(
|
|||
println!("{}", table.display()?);
|
||||
}
|
||||
|
||||
if skipped > 0 && model.is_none() {
|
||||
if skipped > 0 {
|
||||
eprintln!(
|
||||
"Skipped {} model(s) (no credentials: {})",
|
||||
skipped,
|
||||
|
|
@ -516,7 +511,7 @@ impl Default for ModelsCommand {
|
|||
mod tests {
|
||||
use fabro_types::{ModelControls, ModelCosts, ModelFeatures, ModelLimits};
|
||||
use lithos_llm::catalog::builtin;
|
||||
use lithos_llm::types::ReasoningEffort;
|
||||
use lithos_llm::types::{ErrorKind, ReasoningEffort, TokenCounts};
|
||||
|
||||
use super::*;
|
||||
|
||||
|
|
@ -657,9 +652,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "test-model",
|
||||
"provider": "anthropic",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "anthropic/test-model",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
@ -672,8 +669,7 @@ mod tests {
|
|||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status, api_types::ModelTestResultStatus::Ok);
|
||||
assert!(response.error_message.is_none());
|
||||
assert!(response.passed());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -689,10 +685,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "test-model",
|
||||
"provider": "anthropic",
|
||||
"status": "error",
|
||||
"error_message": "timeout"
|
||||
"version": 1,
|
||||
"route": "anthropic/test-model",
|
||||
"outcome": {"status": "failed", "error": {"kind": "timeout", "message": "timeout", "retry": {"type": "never"}}},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
@ -710,12 +707,13 @@ mod tests {
|
|||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status, api_types::ModelTestResultStatus::Error);
|
||||
assert_eq!(response.error_message.as_deref(), Some("timeout"));
|
||||
assert!(
|
||||
matches!(response.outcome, ProbeOutcome::Failed(error) if error.message == "timeout")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_via_server_parses_skip() {
|
||||
async fn test_model_via_server_parses_selection_failure() {
|
||||
let server = httpmock::MockServer::start_async().await;
|
||||
server
|
||||
.mock_async(|when, then| {
|
||||
|
|
@ -724,9 +722,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "kimi-k2.5",
|
||||
"provider": "moonshot",
|
||||
"status": "skip"
|
||||
"version": 1,
|
||||
"route": "moonshot/kimi-k2.5",
|
||||
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
@ -739,8 +739,9 @@ mod tests {
|
|||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status, api_types::ModelTestResultStatus::Skip);
|
||||
assert!(response.error_message.is_none());
|
||||
assert!(
|
||||
matches!(response.outcome, ProbeOutcome::Failed(error) if error.kind == ErrorKind::ModelSelection)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -767,26 +768,8 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_model_test_uses_server_model_metadata_for_custom_models() {
|
||||
async fn single_model_test_uses_probe_identity_without_catalog_metadata() {
|
||||
let server = httpmock::MockServer::start_async().await;
|
||||
server
|
||||
.mock_async(|when, then| {
|
||||
when.method("GET")
|
||||
.path("/api/v1/models")
|
||||
.query_param("page[limit]", "100")
|
||||
.query_param("page[offset]", "0")
|
||||
.query_param("query", "venice-large");
|
||||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"data": [custom_model_json("venice-large", "venice")],
|
||||
"meta": { "has_more": false }
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
server
|
||||
.mock_async(|when, then| {
|
||||
when.method("POST").path("/api/v1/models/venice-large/test");
|
||||
|
|
@ -794,9 +777,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "venice-large",
|
||||
"provider": "venice",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "venice/venice-large",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
@ -847,9 +832,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "portable-model",
|
||||
"provider": "openai",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "openai/portable-model",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
@ -864,9 +851,11 @@ mod tests {
|
|||
.header("Content-Type", "application/json")
|
||||
.body(
|
||||
serde_json::json!({
|
||||
"model_id": "portable-model",
|
||||
"provider": "openrouter",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "openrouter/portable-model",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ use axum::routing::{get, post};
|
|||
use axum::{Json, Router};
|
||||
use fabro_test::{fabro_snapshot, test_context};
|
||||
use httpmock::MockServer;
|
||||
use lithos_llm::types::TokenCounts;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::{Semaphore, oneshot};
|
||||
|
||||
|
|
@ -129,9 +130,11 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)]
|
|||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.json_body(serde_json::json!({
|
||||
"model_id": "test-model",
|
||||
"provider": "anthropic",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "anthropic/test-model",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
|
||||
|
|
@ -146,7 +149,7 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)]
|
|||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
list.assert();
|
||||
list.assert_calls(1);
|
||||
test.assert();
|
||||
}
|
||||
|
||||
|
|
@ -164,38 +167,197 @@ fn model_test_deep_remains_an_alias_for_tools() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn model_test_unknown_model_errors() {
|
||||
fn single_model_json_reports_canonical_alias_without_listing() {
|
||||
let context = test_context!();
|
||||
let server = MockServer::start();
|
||||
context.set_http_target(&server.base_url());
|
||||
let listing = server.mock(|when, then| {
|
||||
when.method("GET").path("/api/v1/models");
|
||||
then.status(500);
|
||||
});
|
||||
let report = serde_json::json!({
|
||||
"version": 1,
|
||||
"route": "anthropic/claude-sonnet-4.6",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default(),
|
||||
});
|
||||
let probe = server.mock(|when, then| {
|
||||
when.method("POST")
|
||||
.path("/api/v1/models/claude-sonnet-4-6/test")
|
||||
.query_param("provider", "anthropic");
|
||||
then.status(200).json_body(report.clone());
|
||||
});
|
||||
let mut cmd = context.command();
|
||||
cmd.args(["model", "test", "--model", "nonexistent-model-xyz"]);
|
||||
|
||||
fabro_snapshot!(context.filters(), cmd, @"
|
||||
success: false
|
||||
exit_code: 1
|
||||
----- stdout -----
|
||||
----- stderr -----
|
||||
Testing nonexistent-model-xyz... done
|
||||
× Unknown model: nonexistent-model-xyz
|
||||
");
|
||||
cmd.args([
|
||||
"model",
|
||||
"test",
|
||||
"--model",
|
||||
"claude-sonnet-4-6",
|
||||
"--provider",
|
||||
"anthropic",
|
||||
"--json",
|
||||
]);
|
||||
let output = cmd
|
||||
.assert()
|
||||
.success()
|
||||
.stderr("")
|
||||
.get_output()
|
||||
.stdout
|
||||
.clone();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&output).unwrap(),
|
||||
report
|
||||
);
|
||||
listing.assert_calls(0);
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_model_skip_exits_nonzero() {
|
||||
fn single_model_text_restores_the_metadata_table() {
|
||||
let context = test_context!();
|
||||
let server = MockServer::start();
|
||||
context.set_http_target(&server.base_url());
|
||||
let mut model = model_json("test-model", "anthropic", false);
|
||||
model["aliases"] = serde_json::json!(["tm"]);
|
||||
let listing = server.mock(|when, then| {
|
||||
when.method("GET")
|
||||
.path("/api/v1/models")
|
||||
.query_param("provider", "anthropic")
|
||||
.query_param("query", "test-model");
|
||||
then.status(200).json_body(serde_json::json!({
|
||||
"data": [model_json("test-model", "other-provider", true), model],
|
||||
"meta": {"has_more": false}
|
||||
}));
|
||||
});
|
||||
let probe = server.mock(|when, then| {
|
||||
when.method("POST").path("/api/v1/models/tm/test");
|
||||
then.status(200).json_body(serde_json::json!({
|
||||
"version": 1, "route": "anthropic/test-model", "outcome": {"status": "passed"},
|
||||
"latency_ms": 12, "usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
let mut cmd = context.command();
|
||||
cmd.args(["model", "test", "--model", "gemini-3.1-pro-preview"]);
|
||||
remove_provider_env(&mut cmd);
|
||||
cmd.args(["model", "test", "--model", "tm"]);
|
||||
fabro_snapshot!(context.filters(), cmd, @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT
|
||||
test-model anthropic tm 131k $1.0 / $2.0 42 tok/s ok
|
||||
----- stderr -----
|
||||
Testing tm... done
|
||||
");
|
||||
listing.assert_calls(1);
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_model_text_preserves_success_when_metadata_is_missing_or_unavailable() {
|
||||
for (status, models) in [
|
||||
(500, vec![]),
|
||||
(200, vec![]),
|
||||
(200, vec![model_json("test-model", "other-provider", true)]),
|
||||
] {
|
||||
let context = test_context!();
|
||||
let server = MockServer::start();
|
||||
context.set_http_target(&server.base_url());
|
||||
let listing = server.mock(|when, then| {
|
||||
when.method("GET").path("/api/v1/models");
|
||||
then.status(status).json_body(serde_json::json!({
|
||||
"data": models, "meta": {"has_more": false}
|
||||
}));
|
||||
});
|
||||
let probe = server.mock(|when, then| {
|
||||
when.method("POST").path("/api/v1/models/tm/test");
|
||||
then.status(200).json_body(serde_json::json!({
|
||||
"version": 1, "route": "anthropic/test-model", "outcome": {"status": "passed"},
|
||||
"latency_ms": 12, "usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
let mut cmd = context.command();
|
||||
cmd.args(["model", "test", "--model", "tm"]);
|
||||
insta::allow_duplicates! {
|
||||
fabro_snapshot!(context.filters(), cmd, @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT
|
||||
test-model anthropic — — — — ok
|
||||
----- stderr -----
|
||||
Testing tm... done
|
||||
");
|
||||
}
|
||||
listing.assert_calls(1);
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_model_text_preserves_failure_when_metadata_is_unavailable() {
|
||||
let context = test_context!();
|
||||
let server = MockServer::start();
|
||||
context.set_http_target(&server.base_url());
|
||||
let listing = server.mock(|when, then| {
|
||||
when.method("GET").path("/api/v1/models");
|
||||
then.status(500);
|
||||
});
|
||||
let probe = server.mock(|when, then| {
|
||||
when.method("POST")
|
||||
.path("/api/v1/models/unknown-model/test");
|
||||
then.status(200).json_body(serde_json::json!({
|
||||
"version": 1, "route": null,
|
||||
"outcome": {"status": "failed", "error": {
|
||||
"kind": "model_selection", "message": "unknown model", "retry": {"type": "never"}
|
||||
}},
|
||||
"latency_ms": 0, "usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
let mut cmd = context.command();
|
||||
cmd.args(["model", "test", "--model", "unknown-model"]);
|
||||
fabro_snapshot!(context.filters(), cmd, @"
|
||||
success: false
|
||||
exit_code: 1
|
||||
----- stdout -----
|
||||
MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT
|
||||
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s not configured
|
||||
MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT
|
||||
unknown-model — — — — — error: model_selection: unknown model
|
||||
----- stderr -----
|
||||
Testing gemini-3.1-pro-preview... done
|
||||
Testing unknown-model... done
|
||||
× 1 model(s) failed
|
||||
");
|
||||
listing.assert_calls(1);
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_model_preserves_failed_and_incorrect_findings() {
|
||||
for outcome in [
|
||||
serde_json::json!({"status": "failed", "error": {"kind": "model_selection", "message": "unknown model", "retry": {"type": "never"}}}),
|
||||
serde_json::json!({"status": "incorrect", "detail": "model did not call the tool"}),
|
||||
] {
|
||||
let context = test_context!();
|
||||
let server = MockServer::start();
|
||||
context.set_http_target(&server.base_url());
|
||||
let report = serde_json::json!({
|
||||
"version": 1,
|
||||
"route": null,
|
||||
"outcome": outcome,
|
||||
"latency_ms": 0,
|
||||
"usage": TokenCounts::default(),
|
||||
});
|
||||
let probe = server.mock(|when, then| {
|
||||
when.method("POST").path("/api/v1/models/test-model/test");
|
||||
then.status(200).json_body(report.clone());
|
||||
});
|
||||
let mut cmd = context.command();
|
||||
cmd.args(["model", "test", "--model", "test-model", "--json"]);
|
||||
let output = cmd.assert().failure().get_output().stdout.clone();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&output).unwrap(),
|
||||
report
|
||||
);
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -224,34 +386,6 @@ fn bulk_skip_exits_zero_and_prints_summary() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_output_includes_skipped_models() {
|
||||
let context = test_context!();
|
||||
let mut cmd = context.command();
|
||||
cmd.args([
|
||||
"model",
|
||||
"test",
|
||||
"--model",
|
||||
"gemini-3.1-pro-preview",
|
||||
"--json",
|
||||
]);
|
||||
remove_provider_env(&mut cmd);
|
||||
|
||||
let output = cmd.output().expect("failed to execute model test");
|
||||
assert!(
|
||||
!output.status.success(),
|
||||
"expected single-model skip to exit non-zero:\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let json: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output");
|
||||
|
||||
assert_eq!(json["failures"], 1);
|
||||
assert_eq!(json["skipped"], 1);
|
||||
assert_eq!(json["results"][0]["result"], "skip");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_test_does_not_announce_unconfigured() {
|
||||
let context = test_context!();
|
||||
|
|
@ -267,9 +401,11 @@ fn model_test_does_not_announce_unconfigured() {
|
|||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.json_body(serde_json::json!({
|
||||
"model_id": "claude-opus-4-7",
|
||||
"provider": "anthropic",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "anthropic/claude-opus-4-7",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
let unconfigured_test = server.mock(|when, then| {
|
||||
|
|
@ -317,9 +453,11 @@ fn model_test_skipped_footer_sources_from_listing() {
|
|||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.json_body(serde_json::json!({
|
||||
"model_id": "claude-opus-4-7",
|
||||
"provider": "anthropic",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": "anthropic/claude-opus-4-7",
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
|
||||
|
|
@ -349,9 +487,11 @@ fn model_test_post_list_race_is_a_failure() {
|
|||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.json_body(serde_json::json!({
|
||||
"model_id": "claude-opus-4-7",
|
||||
"provider": "anthropic",
|
||||
"status": "skip"
|
||||
"version": 1,
|
||||
"route": "anthropic/claude-opus-4-7",
|
||||
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
|
||||
|
|
@ -367,7 +507,7 @@ fn model_test_post_list_race_is_a_failure() {
|
|||
);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(stdout.contains("provider became unconfigured after listing"));
|
||||
assert!(stdout.contains("model_selection: provider unavailable"));
|
||||
assert!(stderr.contains("1 model(s) failed"));
|
||||
assert!(!stderr.contains("Skipped"));
|
||||
}
|
||||
|
|
@ -391,9 +531,11 @@ fn model_test_json_partitions_skip_and_fail() {
|
|||
then.status(200)
|
||||
.header("Content-Type", "application/json")
|
||||
.json_body(serde_json::json!({
|
||||
"model_id": "claude-opus-4-7",
|
||||
"provider": "anthropic",
|
||||
"status": "skip"
|
||||
"version": 1,
|
||||
"route": "anthropic/claude-opus-4-7",
|
||||
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}));
|
||||
});
|
||||
|
||||
|
|
@ -420,8 +562,13 @@ fn model_test_json_partitions_skip_and_fail() {
|
|||
assert_eq!(json["results"][1]["result"], "fail");
|
||||
assert_eq!(
|
||||
json["results"][1]["error"],
|
||||
"provider became unconfigured after listing"
|
||||
"model_selection: provider unavailable"
|
||||
);
|
||||
assert_eq!(
|
||||
json["results"][1]["report"]["outcome"]["error"]["kind"],
|
||||
"model_selection"
|
||||
);
|
||||
assert!(json["results"][0].get("report").is_none());
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -598,9 +745,11 @@ async fn concurrent_test_model(
|
|||
state.gate.exit();
|
||||
|
||||
Json(serde_json::json!({
|
||||
"model_id": id,
|
||||
"provider": "anthropic",
|
||||
"status": "ok"
|
||||
"version": 1,
|
||||
"route": format!("anthropic/{id}"),
|
||||
"outcome": {"status": "passed"},
|
||||
"latency_ms": 12,
|
||||
"usage": TokenCounts::default()
|
||||
}))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@ use std::collections::HashSet;
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use fabro_llm::api;
|
||||
use fabro_llm::lithos_catalog::Catalog;
|
||||
use fabro_llm::probe::{self, ApiKeyProbeError, ModelTestStatus};
|
||||
use fabro_llm::{ModelSelectionError, api, selection};
|
||||
use fabro_redact::redact_string;
|
||||
use lithos_llm::client::{ProbeOptions, ProbeOutcome};
|
||||
use lithos_llm::types::ReasoningEffort;
|
||||
|
||||
use super::super::{
|
||||
|
|
@ -228,85 +229,25 @@ async fn test_model(
|
|||
.into_response();
|
||||
}
|
||||
};
|
||||
let catalog = state.catalog();
|
||||
let eligible = llm_result
|
||||
.provider_ids()
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
let explicit_provider = params.provider.map(ProviderId::new);
|
||||
let info = if let Some(provider) = explicit_provider.as_ref() {
|
||||
match selection::resolve_on_provider(&catalog, provider, &id) {
|
||||
Ok(info) => info,
|
||||
Err(error) => return model_selection_response(&error),
|
||||
}
|
||||
} else {
|
||||
match selection::select(&catalog, &id, None, &eligible) {
|
||||
Ok(info) => info,
|
||||
Err(error) => return model_selection_response(&error),
|
||||
}
|
||||
};
|
||||
let provider_id = info.provider.id().clone();
|
||||
let model_id = info.model.id().clone();
|
||||
if let Some((_, issue)) = llm_result
|
||||
.auth_issues
|
||||
.iter()
|
||||
.find(|(provider, _)| provider == &provider_id)
|
||||
{
|
||||
return ApiError::bad_request(issue.to_string()).into_response();
|
||||
}
|
||||
if !llm_result.has_provider(&provider_id) {
|
||||
return Json(serde_json::json!({
|
||||
"model_id": model_id,
|
||||
"provider": provider_id,
|
||||
"status": "skip",
|
||||
}))
|
||||
.into_response();
|
||||
}
|
||||
let selector = params
|
||||
.provider
|
||||
.map_or_else(|| id.clone(), |provider| format!("{provider}/{id}"));
|
||||
let mut options = ProbeOptions::new().tools(mode == ModelTestMode::Deep);
|
||||
if let Some(effort) = reasoning_effort {
|
||||
let capabilities = info.model.capabilities();
|
||||
if !capabilities.reasoning_effort(effort).is_supported() {
|
||||
let allowed = ReasoningEffort::ALL
|
||||
.into_iter()
|
||||
.filter(|candidate| capabilities.reasoning_effort(*candidate).is_supported())
|
||||
.map(ReasoningEffort::as_str)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
return ApiError::bad_request(format!(
|
||||
"model '{model_id}' does not support reasoning_effort '{effort}'; allowed values: {allowed}"
|
||||
))
|
||||
.into_response();
|
||||
}
|
||||
options = options.reasoning_effort(effort);
|
||||
}
|
||||
|
||||
let outcome = probe::run_model_test(
|
||||
&llm_result.client,
|
||||
&format!("{provider_id}/{model_id}"),
|
||||
mode,
|
||||
reasoning_effort,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
Json(serde_json::json!({
|
||||
"model_id": model_id,
|
||||
"provider": provider_id,
|
||||
"status": <&'static str>::from(outcome.status),
|
||||
"error_message": outcome.error_message,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn model_selection_response(error: &ModelSelectionError) -> Response {
|
||||
match error {
|
||||
ModelSelectionError::UnknownProvider { .. }
|
||||
| ModelSelectionError::UnknownSelector { .. }
|
||||
| ModelSelectionError::UnknownSelectorOnProvider { .. }
|
||||
| ModelSelectionError::ProviderUsedAsModel { .. } => {
|
||||
ApiError::not_found(error.to_string()).into_response()
|
||||
}
|
||||
ModelSelectionError::ProviderUnavailable { .. }
|
||||
| ModelSelectionError::NoEligibleOffering { .. }
|
||||
| ModelSelectionError::NoDefaultModel { .. } => {
|
||||
ApiError::bad_request(error.to_string()).into_response()
|
||||
let mut report = llm_result.client.probe(selector, options).await;
|
||||
// Model selection and probe findings belong to lithos. Fabro only owns
|
||||
// disclosure of provider diagnostics across its HTTP boundary.
|
||||
match &mut report.outcome {
|
||||
ProbeOutcome::Failed(error) => {
|
||||
error.message = redact_string(&error.message);
|
||||
error.provider_code = error.provider_code.as_deref().map(redact_string);
|
||||
error.raw_data = None;
|
||||
error.source_message = None;
|
||||
}
|
||||
ProbeOutcome::Incorrect { detail } => *detail = redact_string(detail),
|
||||
_ => {}
|
||||
}
|
||||
Json(report).into_response()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5194,7 +5194,7 @@ async fn pr_test_app_with_minimal_run(
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_unknown_returns_404() {
|
||||
async fn test_model_unknown_returns_a_probe_finding() {
|
||||
let app = test_app_with();
|
||||
|
||||
let req = Request::builder()
|
||||
|
|
@ -5205,11 +5205,14 @@ async fn test_model_unknown_returns_404() {
|
|||
.unwrap();
|
||||
|
||||
let response = app.clone().oneshot(req).await.unwrap();
|
||||
assert_status!(response, StatusCode::NOT_FOUND).await;
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
|
||||
assert!(body["route"].is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() {
|
||||
async fn test_model_explicit_provider_unavailable_returns_a_probe_finding() {
|
||||
let state = test_app_state_with_env_lookup(
|
||||
default_test_server_settings(),
|
||||
RunLayer::default(),
|
||||
|
|
@ -5227,9 +5230,8 @@ async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unav
|
|||
|
||||
let response = app.oneshot(req).await.unwrap();
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["model_id"], "claude-sonnet-5");
|
||||
assert_eq!(body["provider"], "anthropic");
|
||||
assert_eq!(body["status"], "skip");
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -5253,7 +5255,9 @@ async fn test_model_unqualified_known_alias_requires_a_ready_provider() {
|
|||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_status!(response, StatusCode::BAD_REQUEST).await;
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -5388,30 +5392,118 @@ capabilities = {{ text = true }}
|
|||
assert_eq!(filtered["data"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(filtered["data"][0]["provider"], "aggregator");
|
||||
|
||||
for (query, expected_provider) in [
|
||||
("?provider=direct", "direct"),
|
||||
("?provider=aggregator", "aggregator"),
|
||||
("", "direct"),
|
||||
for (selector, query, expected_provider) in [
|
||||
("portable", "?provider=direct", "direct"),
|
||||
("portable", "?provider=aggregator", "aggregator"),
|
||||
("portable", "", "direct"),
|
||||
(
|
||||
"vendor%2Fportable-model",
|
||||
"?provider=aggregator",
|
||||
"aggregator",
|
||||
),
|
||||
] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(api(&format!("/models/portable/test{query}")))
|
||||
.uri(api(&format!("/models/{selector}/test{query}")))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["model_id"], "portable-model");
|
||||
assert_eq!(body["provider"], expected_provider);
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert_eq!(body["route"], format!("{expected_provider}/portable-model"));
|
||||
assert_eq!(body["outcome"]["status"], "passed");
|
||||
assert_eq!(body["usage"]["input"], 1);
|
||||
}
|
||||
|
||||
direct_probe.assert_calls(2);
|
||||
aggregator_probe.assert_calls(1);
|
||||
aggregator_probe.assert_calls(2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_preserves_error_classification_and_redacts_provider_details() {
|
||||
let leaked_key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
let upstream = MockServer::start();
|
||||
let probe = upstream.mock(|when, then| {
|
||||
when.method(POST).path("/v1/chat/completions");
|
||||
then.status(401).json_body(json!({
|
||||
"error": {
|
||||
"message": format!("invalid api key {leaked_key}"),
|
||||
"code": format!("invalid_key_{leaked_key}"),
|
||||
"type": "invalid_request_error"
|
||||
},
|
||||
"private_detail": "must not cross the API boundary"
|
||||
}));
|
||||
});
|
||||
let state = TestAppStateBuilder::new()
|
||||
.llm_overlay_toml(&acme_overlay(&upstream.base_url()))
|
||||
.vault_entries([("ACME_API_KEY", leaked_key)])
|
||||
.build();
|
||||
let app = crate::test_support::build_test_router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(api("/models/acme-large/test?provider=acme"))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["version"], 1);
|
||||
assert_eq!(body["route"], "acme/acme-large");
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
let error = &body["outcome"]["error"];
|
||||
assert_eq!(error["kind"], "authentication");
|
||||
assert_eq!(error["status"], 401);
|
||||
assert_eq!(error["provider"], "acme");
|
||||
assert!(error.get("raw_data").is_none());
|
||||
assert!(error.get("source_message").is_none());
|
||||
let serialized = body.to_string();
|
||||
assert!(!serialized.contains(leaked_key));
|
||||
assert!(!serialized.contains("must not cross"));
|
||||
assert!(serialized.contains("REDACTED"));
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_tool_probe_preserves_incorrect_outcome_and_usage() {
|
||||
let upstream = MockServer::start();
|
||||
let probe = upstream.mock(|when, then| {
|
||||
when.method(POST).path("/v1/chat/completions");
|
||||
then.status(200).json_body(json!({
|
||||
"id": "chatcmpl-test", "model": "acme-large",
|
||||
"choices": [{"message": {"role": "assistant", "content": "84 is even"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 12, "completion_tokens": 7, "total_tokens": 19}
|
||||
}));
|
||||
});
|
||||
let state = TestAppStateBuilder::new()
|
||||
.llm_overlay_toml(&acme_overlay(&upstream.base_url()))
|
||||
.vault_entries([("ACME_API_KEY", "test-key")])
|
||||
.build();
|
||||
let app = crate::test_support::build_test_router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(api("/models/acme-large/test?provider=acme&mode=deep"))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["route"], "acme/acme-large");
|
||||
assert_eq!(body["outcome"]["status"], "incorrect");
|
||||
assert!(!body["outcome"]["detail"].as_str().unwrap().is_empty());
|
||||
assert_eq!(body["usage"]["input"], 12);
|
||||
assert_eq!(body["usage"]["output"], 7);
|
||||
assert!(body["latency_ms"].is_u64());
|
||||
probe.assert_calls(1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -5516,7 +5608,7 @@ protocol_options = {{ reasoning_effort_levels = true }}
|
|||
|
||||
let response = app.clone().oneshot(req).await.unwrap();
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert_eq!(body["outcome"]["status"], "passed");
|
||||
|
||||
let unsupported = Request::builder()
|
||||
.method("POST")
|
||||
|
|
@ -5526,11 +5618,9 @@ protocol_options = {{ reasoning_effort_levels = true }}
|
|||
.body(Body::empty())
|
||||
.unwrap();
|
||||
let response = app.oneshot(unsupported).await.unwrap();
|
||||
let body = response_json!(response, StatusCode::BAD_REQUEST).await;
|
||||
assert_eq!(
|
||||
body["errors"][0]["detail"],
|
||||
"model 'acme-reasoner' does not support reasoning_effort 'medium'; allowed values: low, high"
|
||||
);
|
||||
let body = response_json!(response, StatusCode::OK).await;
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "invalid_request");
|
||||
completion.assert_calls(1);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ async fn dry_run_serve_starts_and_runs_workflow() {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_known_but_unavailable_returns_bad_request() {
|
||||
async fn test_model_known_but_unavailable_returns_a_probe_finding() {
|
||||
let app = test_app_with_no_providers();
|
||||
|
||||
let req = Request::builder()
|
||||
|
|
@ -71,15 +71,12 @@ async fn test_model_known_but_unavailable_returns_bad_request() {
|
|||
let response = app.oneshot(req).await.unwrap();
|
||||
let body = response_json(
|
||||
response,
|
||||
StatusCode::BAD_REQUEST,
|
||||
StatusCode::OK,
|
||||
"POST /api/v1/models/claude-opus-4.6/test",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
body["errors"][0]["detail"]
|
||||
.as_str()
|
||||
.is_some_and(|detail| detail.contains("no offering on an eligible provider"))
|
||||
);
|
||||
assert_eq!(body["outcome"]["status"], "failed");
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -94,12 +91,13 @@ async fn test_model_unknown_via_full_router() {
|
|||
.unwrap();
|
||||
|
||||
let response = app.oneshot(req).await.unwrap();
|
||||
response_status(
|
||||
let body = response_json(
|
||||
response,
|
||||
StatusCode::NOT_FOUND,
|
||||
StatusCode::OK,
|
||||
"POST /api/v1/models/nonexistent-model-xyz/test",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -4,10 +4,8 @@ use std::sync::Arc;
|
|||
use std::time::Duration;
|
||||
|
||||
use fabro_auth::ApiKeyCredentialSource;
|
||||
use fabro_types::ModelTestMode;
|
||||
use lithos_llm::catalog::{Catalog, ProviderId};
|
||||
use lithos_llm::client::{Client, ProbeOptions, ProbeOutcome};
|
||||
use lithos_llm::types::ReasoningEffort;
|
||||
use strum::IntoStaticStr;
|
||||
|
||||
use crate::client::{ClientOptions, LlmSetupError, build_client};
|
||||
|
|
@ -43,25 +41,16 @@ impl ModelTestOutcome {
|
|||
}
|
||||
}
|
||||
|
||||
/// Probes `selector` (a `provider/model` route or bare selector) in `mode`.
|
||||
///
|
||||
/// `Basic` asks for one word; `Deep` runs a two-step tool exchange and checks
|
||||
/// the total. The probe resolves, authenticates, and retries exactly as a
|
||||
/// real request would.
|
||||
pub async fn run_model_test(
|
||||
/// A credential-health projection of the library's basic probe.
|
||||
/// Model-test endpoints return the full library report instead.
|
||||
pub async fn run_basic_probe(
|
||||
client: &Client,
|
||||
selector: &str,
|
||||
mode: ModelTestMode,
|
||||
reasoning_effort: Option<ReasoningEffort>,
|
||||
timeout: Option<Duration>,
|
||||
timeout: Duration,
|
||||
) -> ModelTestOutcome {
|
||||
let mut options = ProbeOptions::new()
|
||||
.tools(mode == ModelTestMode::Deep)
|
||||
.timeout(timeout.unwrap_or_else(|| Duration::from_secs(mode.timeout_secs())));
|
||||
if let Some(effort) = reasoning_effort {
|
||||
options = options.reasoning_effort(effort);
|
||||
}
|
||||
let report = client.probe(selector, options).await;
|
||||
let report = client
|
||||
.probe(selector, ProbeOptions::new().timeout(timeout))
|
||||
.await;
|
||||
match report.outcome {
|
||||
ProbeOutcome::Passed => ModelTestOutcome::ok(),
|
||||
ProbeOutcome::Failed(data) => ModelTestOutcome::error(data.message),
|
||||
|
|
@ -70,15 +59,6 @@ pub async fn run_model_test(
|
|||
}
|
||||
}
|
||||
|
||||
/// A basic probe of `selector` bounded by `timeout`.
|
||||
pub async fn run_basic_probe(
|
||||
client: &Client,
|
||||
selector: &str,
|
||||
timeout: Duration,
|
||||
) -> ModelTestOutcome {
|
||||
run_model_test(client, selector, ModelTestMode::Basic, None, Some(timeout)).await
|
||||
}
|
||||
|
||||
/// Why an API key could not be probed.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ApiKeyProbeError {
|
||||
|
|
|
|||
|
|
@ -606,6 +606,8 @@ fn main() {
|
|||
&[],
|
||||
),
|
||||
("TokenCounts", "lithos_llm::types::TokenCounts", &[]),
|
||||
("ProbeReport", "lithos_llm::client::ProbeReport", &[]),
|
||||
("LlmErrorData", "lithos_llm::types::ErrorData", &[]),
|
||||
("Usage", "lithos_llm::types::Usage", &[]),
|
||||
("UsageModelRef", "fabro_types::ModelRef", &[]),
|
||||
("Speed", "lithos_llm::types::Speed", &[]),
|
||||
|
|
|
|||
|
|
@ -72,9 +72,10 @@ pub mod types {
|
|||
WorkflowVersionId,
|
||||
};
|
||||
pub use lithos_llm::catalog::{ModelHandle, ProviderId};
|
||||
pub use lithos_llm::client::ProbeReport;
|
||||
pub use lithos_llm::types::{
|
||||
ContentPart, Cost, CostSource, ErrorKind as LlmErrorKind, Message, ReasoningEffort,
|
||||
ReasoningOutput, ResponseFormat as CompletionResponseFormat,
|
||||
ContentPart, Cost, CostSource, ErrorData as LlmErrorData, ErrorKind as LlmErrorKind,
|
||||
Message, ReasoningEffort, ReasoningOutput, ResponseFormat as CompletionResponseFormat,
|
||||
RetryClassification as LlmRetryClassification, Role, Speed, TokenCounts,
|
||||
ToolChoice as CompletionToolChoice, ToolDefinition as CompletionToolDefinition,
|
||||
ToolDefinitionKind as CompletionToolDefinitionKind, Usage,
|
||||
|
|
|
|||
77
lib/foundation/fabro-api/tests/probe_report_round_trip.rs
Normal file
77
lib/foundation/fabro-api/tests/probe_report_round_trip.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
use std::any::TypeId;
|
||||
|
||||
use fabro_api::types;
|
||||
use lithos_llm::client::ProbeReport;
|
||||
use lithos_llm::types::{ErrorData, TokenCounts};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn validator(name: &str) -> jsonschema::Validator {
|
||||
let yaml: serde_yaml::Value = serde_yaml::from_str(include_str!(
|
||||
"../../../../docs/public/api-reference/fabro-api.yaml"
|
||||
))
|
||||
.unwrap();
|
||||
let mut spec = serde_json::to_value(yaml).unwrap();
|
||||
spec["$ref"] = json!(format!("#/components/schemas/{name}"));
|
||||
jsonschema::validator_for(&spec).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_api_reuses_library_types_and_preserves_every_outcome() {
|
||||
assert_eq!(
|
||||
TypeId::of::<types::ProbeReport>(),
|
||||
TypeId::of::<ProbeReport>()
|
||||
);
|
||||
assert_eq!(
|
||||
TypeId::of::<types::LlmErrorData>(),
|
||||
TypeId::of::<ErrorData>()
|
||||
);
|
||||
let error = json!({
|
||||
"kind": "authentication", "message": "invalid key", "provider": "acme",
|
||||
"status": 401, "provider_code": "invalid_key", "retry": {"type": "never"},
|
||||
"raw_data": {"error": "detail"}, "source_message": "source",
|
||||
"provider_retry_after_millis": 12
|
||||
});
|
||||
let parsed: types::LlmErrorData = serde_json::from_value(error.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(parsed).unwrap(), error);
|
||||
assert!(validator("LlmErrorData").is_valid(&error));
|
||||
|
||||
let schema = validator("ProbeReport");
|
||||
for (route, outcome) in [
|
||||
(json!("acme/vendor/model"), json!({"status": "passed"})),
|
||||
(Value::Null, json!({"status": "failed", "error": error})),
|
||||
(
|
||||
json!("acme/model"),
|
||||
json!({"status": "incorrect", "detail": "no tool call"}),
|
||||
),
|
||||
] {
|
||||
let value = json!({
|
||||
"version": 1, "route": route, "outcome": outcome, "latency_ms": 123,
|
||||
"usage": TokenCounts { input: 12, output: 7, ..TokenCounts::default() }
|
||||
});
|
||||
let report: types::ProbeReport = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(report).unwrap(), value);
|
||||
assert!(schema.is_valid(&value), "{value}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_schema_and_rust_reject_unsupported_versions_routes_and_outcomes() {
|
||||
let schema = validator("ProbeReport");
|
||||
let valid = json!({
|
||||
"version": 1, "route": "acme/model", "outcome": {"status": "passed"},
|
||||
"latency_ms": 0, "usage": TokenCounts::default()
|
||||
});
|
||||
for (field, invalid) in [
|
||||
("version", json!(2)),
|
||||
("route", json!("model")),
|
||||
("route", json!("acme/")),
|
||||
("outcome", json!({"status": "skip"})),
|
||||
("outcome", json!({"status": "failed"})),
|
||||
("latency_ms", json!(-1)),
|
||||
] {
|
||||
let mut value = valid.clone();
|
||||
value[field] = invalid;
|
||||
assert!(!schema.is_valid(&value), "{value}");
|
||||
assert!(serde_json::from_value::<types::ProbeReport>(value).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ use fabro_util::exit::{ErrorExt, ExitClass};
|
|||
use futures::future::BoxFuture;
|
||||
use futures::{Stream, StreamExt};
|
||||
use lithos_llm::catalog::ProviderId;
|
||||
use lithos_llm::client::ProbeReport;
|
||||
use lithos_llm::types::ReasoningEffort;
|
||||
use serde::Deserialize;
|
||||
use tokio::sync::Mutex;
|
||||
|
|
@ -912,7 +913,7 @@ impl Client {
|
|||
provider: Option<&ProviderId>,
|
||||
mode: Option<ModelTestMode>,
|
||||
reasoning_effort: Option<ReasoningEffort>,
|
||||
) -> Result<types::ModelTestResult> {
|
||||
) -> Result<ProbeReport> {
|
||||
let response = self
|
||||
.send_api(|client| async move {
|
||||
let mut request = client.test_model().id(id.to_string());
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ models/interview-option.ts
|
|||
models/interview-provider-settings.ts
|
||||
models/interview-question-record.ts
|
||||
models/link-run-pull-request-request.ts
|
||||
models/llm-error-data.ts
|
||||
models/llm-output-kind.ts
|
||||
models/llm-retry-classification-after.ts
|
||||
models/llm-retry-classification-never.ts
|
||||
|
|
@ -252,7 +253,6 @@ models/model-handle.ts
|
|||
models/model-limits.ts
|
||||
models/model-reference.ts
|
||||
models/model-test-mode.ts
|
||||
models/model-test-result.ts
|
||||
models/model-usage.ts
|
||||
models/model.ts
|
||||
models/none-run-target.ts
|
||||
|
|
@ -321,6 +321,11 @@ models/principal-user.ts
|
|||
models/principal-webhook.ts
|
||||
models/principal-worker.ts
|
||||
models/principal.ts
|
||||
models/probe-failed.ts
|
||||
models/probe-incorrect.ts
|
||||
models/probe-outcome.ts
|
||||
models/probe-passed.ts
|
||||
models/probe-report.ts
|
||||
models/project-namespace.ts
|
||||
models/provider-credential-test-request.ts
|
||||
models/provider-credential-test-response.ts
|
||||
|
|
|
|||
24
lib/packages/fabro-api-client/src/api/models-api.ts
generated
24
lib/packages/fabro-api-client/src/api/models-api.ts
generated
|
|
@ -26,10 +26,10 @@ import type { ErrorResponse } from '../models';
|
|||
// @ts-ignore
|
||||
import type { ModelTestMode } from '../models';
|
||||
// @ts-ignore
|
||||
import type { ModelTestResult } from '../models';
|
||||
// @ts-ignore
|
||||
import type { PaginatedModelList } from '../models';
|
||||
// @ts-ignore
|
||||
import type { ProbeReport } from '../models';
|
||||
// @ts-ignore
|
||||
import type { ProviderCredentialTestRequest } from '../models';
|
||||
// @ts-ignore
|
||||
import type { ProviderCredentialTestResponse } from '../models';
|
||||
|
|
@ -137,9 +137,9 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio
|
|||
};
|
||||
},
|
||||
/**
|
||||
* Tests a model by sending a simple prompt and reporting pass/fail.
|
||||
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
|
||||
* @summary Test Model
|
||||
* @param {string} id The canonical model ID or an alias.
|
||||
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
|
||||
* @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority.
|
||||
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`.
|
||||
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
|
||||
|
|
@ -310,16 +310,16 @@ export const ModelsApiFp = function(configuration?: Configuration) {
|
|||
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
|
||||
},
|
||||
/**
|
||||
* Tests a model by sending a simple prompt and reporting pass/fail.
|
||||
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
|
||||
* @summary Test Model
|
||||
* @param {string} id The canonical model ID or an alias.
|
||||
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
|
||||
* @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority.
|
||||
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`.
|
||||
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<ModelTestResult>> {
|
||||
async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<ProbeReport>> {
|
||||
const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, reasoningEffort, options);
|
||||
const localVarOperationServerIndex = configuration?.serverIndex ?? 0;
|
||||
const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url;
|
||||
|
|
@ -383,16 +383,16 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat
|
|||
return localVarFp.listProviders(options).then((request) => request(axios, basePath));
|
||||
},
|
||||
/**
|
||||
* Tests a model by sending a simple prompt and reporting pass/fail.
|
||||
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
|
||||
* @summary Test Model
|
||||
* @param {string} id The canonical model ID or an alias.
|
||||
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
|
||||
* @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority.
|
||||
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`.
|
||||
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
|
||||
* @param {*} [options] Override http request option.
|
||||
* @throws {RequiredError}
|
||||
*/
|
||||
testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise<ModelTestResult> {
|
||||
testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise<ProbeReport> {
|
||||
return localVarFp.testModel(id, provider, mode, reasoningEffort, options).then((request) => request(axios, basePath));
|
||||
},
|
||||
/**
|
||||
|
|
@ -447,9 +447,9 @@ export class ModelsApi extends BaseAPI {
|
|||
}
|
||||
|
||||
/**
|
||||
* Tests a model by sending a simple prompt and reporting pass/fail.
|
||||
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
|
||||
* @summary Test Model
|
||||
* @param {string} id The canonical model ID or an alias.
|
||||
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
|
||||
* @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority.
|
||||
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`.
|
||||
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
|
||||
|
|
|
|||
|
|
@ -184,6 +184,7 @@ export * from './interview-option';
|
|||
export * from './interview-provider-settings';
|
||||
export * from './interview-question-record';
|
||||
export * from './link-run-pull-request-request';
|
||||
export * from './llm-error-data';
|
||||
export * from './llm-output-kind';
|
||||
export * from './llm-retry-classification';
|
||||
export * from './llm-retry-classification-after';
|
||||
|
|
@ -223,7 +224,6 @@ export * from './model-handle';
|
|||
export * from './model-limits';
|
||||
export * from './model-reference';
|
||||
export * from './model-test-mode';
|
||||
export * from './model-test-result';
|
||||
export * from './model-usage';
|
||||
export * from './none-run-target';
|
||||
export * from './notification-provider-settings';
|
||||
|
|
@ -291,6 +291,11 @@ export * from './principal-system';
|
|||
export * from './principal-user';
|
||||
export * from './principal-webhook';
|
||||
export * from './principal-worker';
|
||||
export * from './probe-failed';
|
||||
export * from './probe-incorrect';
|
||||
export * from './probe-outcome';
|
||||
export * from './probe-passed';
|
||||
export * from './probe-report';
|
||||
export * from './project-namespace';
|
||||
export * from './provider';
|
||||
export * from './provider-credential-test-request';
|
||||
|
|
|
|||
42
lib/packages/fabro-api-client/src/models/llm-error-data.ts
generated
Normal file
42
lib/packages/fabro-api-client/src/models/llm-error-data.ts
generated
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { LlmRetryClassification } from './llm-retry-classification';
|
||||
|
||||
/**
|
||||
* Structured lithos-llm failure, subject to the server\'s disclosure policy.
|
||||
*/
|
||||
export interface LlmErrorData {
|
||||
/**
|
||||
* The model layer\'s stable failure category. Known values are `configuration`, `model_selection`, `authentication`, `access_denied`, `not_found`, `invalid_request`, `context_length`, `rate_limit`, `quota_exceeded`, `content_filter`, `server`, `provider`, `network`, `timeout`, `stream_decode`, `response_decode`, `resource_limit`, `middleware`, and `cancelled`. A category written by a newer model layer is carried as its own spelling.
|
||||
*/
|
||||
'kind': string;
|
||||
'message': string;
|
||||
/**
|
||||
* LLM provider identifier.
|
||||
*/
|
||||
'provider'?: string;
|
||||
'status'?: number;
|
||||
'provider_code'?: string;
|
||||
'retry': LlmRetryClassification;
|
||||
'raw_data'?: any;
|
||||
'provider_retry_after_millis'?: number;
|
||||
/**
|
||||
* Internal source detail; omitted by the model-test endpoint.
|
||||
*/
|
||||
'source_message'?: string;
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Result of testing a model in `basic` or `deep` mode.
|
||||
*/
|
||||
export interface ModelTestResult {
|
||||
/**
|
||||
* The canonical model ID that was tested.
|
||||
*/
|
||||
'model_id': string;
|
||||
/**
|
||||
* LLM provider identifier.
|
||||
*/
|
||||
'provider': string;
|
||||
/**
|
||||
* Whether the model responded successfully, failed, or was skipped because its provider is not configured.
|
||||
*/
|
||||
'status': ModelTestResultStatusEnum;
|
||||
/**
|
||||
* Error details when status is \"error\".
|
||||
*/
|
||||
'error_message'?: string | null;
|
||||
}
|
||||
|
||||
export const ModelTestResultStatusEnum = {
|
||||
OK: 'ok',
|
||||
ERROR: 'error',
|
||||
SKIP: 'skip'
|
||||
} as const;
|
||||
|
||||
export type ModelTestResultStatusEnum = typeof ModelTestResultStatusEnum[keyof typeof ModelTestResultStatusEnum];
|
||||
29
lib/packages/fabro-api-client/src/models/probe-failed.ts
generated
Normal file
29
lib/packages/fabro-api-client/src/models/probe-failed.ts
generated
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { LlmErrorData } from './llm-error-data';
|
||||
|
||||
export interface ProbeFailed {
|
||||
'status': ProbeFailedStatusEnum;
|
||||
'error': LlmErrorData;
|
||||
}
|
||||
|
||||
export const ProbeFailedStatusEnum = {
|
||||
FAILED: 'failed'
|
||||
} as const;
|
||||
|
||||
export type ProbeFailedStatusEnum = typeof ProbeFailedStatusEnum[keyof typeof ProbeFailedStatusEnum];
|
||||
26
lib/packages/fabro-api-client/src/models/probe-incorrect.ts
generated
Normal file
26
lib/packages/fabro-api-client/src/models/probe-incorrect.ts
generated
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
export interface ProbeIncorrect {
|
||||
'status': ProbeIncorrectStatusEnum;
|
||||
'detail': string;
|
||||
}
|
||||
|
||||
export const ProbeIncorrectStatusEnum = {
|
||||
INCORRECT: 'incorrect'
|
||||
} as const;
|
||||
|
||||
export type ProbeIncorrectStatusEnum = typeof ProbeIncorrectStatusEnum[keyof typeof ProbeIncorrectStatusEnum];
|
||||
32
lib/packages/fabro-api-client/src/models/probe-outcome.ts
generated
Normal file
32
lib/packages/fabro-api-client/src/models/probe-outcome.ts
generated
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { LlmErrorData } from './llm-error-data';
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { ProbeFailed } from './probe-failed';
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { ProbeIncorrect } from './probe-incorrect';
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { ProbePassed } from './probe-passed';
|
||||
|
||||
/**
|
||||
* @type ProbeOutcome
|
||||
*/
|
||||
export type ProbeOutcome = { status: 'failed' } & ProbeFailed | { status: 'incorrect' } & ProbeIncorrect | { status: 'passed' } & ProbePassed;
|
||||
25
lib/packages/fabro-api-client/src/models/probe-passed.ts
generated
Normal file
25
lib/packages/fabro-api-client/src/models/probe-passed.ts
generated
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
export interface ProbePassed {
|
||||
'status': ProbePassedStatusEnum;
|
||||
}
|
||||
|
||||
export const ProbePassedStatusEnum = {
|
||||
PASSED: 'passed'
|
||||
} as const;
|
||||
|
||||
export type ProbePassedStatusEnum = typeof ProbePassedStatusEnum[keyof typeof ProbePassedStatusEnum];
|
||||
38
lib/packages/fabro-api-client/src/models/probe-report.ts
generated
Normal file
38
lib/packages/fabro-api-client/src/models/probe-report.ts
generated
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
* Fabro Run API
|
||||
* HTTP API for managing Fabro workflow run executions.
|
||||
*
|
||||
* The version of the OpenAPI document: 0.2.0
|
||||
*
|
||||
*
|
||||
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
||||
* https://openapi-generator.tech
|
||||
* Do not edit the class manually.
|
||||
*/
|
||||
|
||||
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { ProbeOutcome } from './probe-outcome';
|
||||
// May contain unused imports in some cases
|
||||
// @ts-ignore
|
||||
import type { TokenCounts } from './token-counts';
|
||||
|
||||
/**
|
||||
* The lithos-llm version-1 probe report. The route is the canonical provider/model selected by the probe, or null if selection failed. Probe failures are findings in a successful HTTP response; API authentication, malformed options and server setup failures remain HTTP errors. Provider errors are redacted at the server disclosure boundary.
|
||||
*/
|
||||
export interface ProbeReport {
|
||||
'version': ProbeReportVersionEnum;
|
||||
'route': string | null;
|
||||
'outcome': ProbeOutcome;
|
||||
'latency_ms': number;
|
||||
'usage': TokenCounts;
|
||||
}
|
||||
|
||||
export const ProbeReportVersionEnum = {
|
||||
NUMBER_1: 1
|
||||
} as const;
|
||||
|
||||
export type ProbeReportVersionEnum = typeof ProbeReportVersionEnum[keyof typeof ProbeReportVersionEnum];
|
||||
Loading…
Add table
Reference in a new issue