mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
feat(ui): add System One (Jev) tab to the playground (#44043)
* feat(ui): add System One (Jev) playground tab Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): validate System One inputs and refresh request context Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): validate System One response payloads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): fall back to requested model for System One Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(ui): use useMutation and zod schemas for the System One tab, mark it Beta Replace the hand-written request and response guards with zod schemas, which also provide the types. Send requests through useMutation instead of manual loading, error and race-guard state. Unknown spec fields now pass through to the upstream model, and validation errors point at the exact offending key * feat(ui): flag the System One tab as a TypeSafe-only beta * feat(ui): highlighted JSON editor for the System One tab Line numbers, JSON syntax highlighting through the existing react-syntax-highlighter dependency, a valid or issue-count status badge, and a compact path plus message issue list replace the bare textarea and stacked alerts. Answer card type badges now sit on the header row * fix(ui): let the System One results scroll to the bottom The tab panel was viewport height but sat below the tab bar, so its bottom was cut off. On wide screens the editor and results now scroll independently, answers render above the question breakdown, and the raw response no longer nests its own scroll area * feat(ui): color the model, state and questions blocks in the System One editor Tints each top-level request block in the JSON editor and marks the matching breakdown sections with the same color, so it is clear which part of the payload feeds which panel * feat(ui): wrap long lines in the System One JSON editor Long state strings no longer need horizontal scrolling. Each line renders as its own row with its number and block color, so wrapped lines keep their line number and the caret stays aligned * fix(ui): remove horizontal scrolling from the System One tab Long unbroken text in the state, question ids, choice labels and the raw response now wraps instead of widening its box * refactor(ui): replace System One presets with one example and a reset button The tab now starts with a single product review example that uses all three question types, and Reset example restores it after editing * refactor(ui): use an issue triage request as the System One example * fix(ui): type the System One line renderer from exported props rendererProps is not exported by the react-syntax-highlighter types, which broke the dashboard build * fix(ui): address System One review feedback Highlights the score level nearest a fractional calibrated score, keeps extra noul criteria fields in the sent payload, and clears an answer when the request key changes * refactor(ui): parse System One root blocks without mutation and preview all noul criteria The root-block finder is now a tokenizer plus a pure reduce, and the question preview lists every noul criterion that will be sent * refactor(ui): group System One playground files into components and lib Drops the repeated SystemOne prefix from the inner component files and moves the pure logic (schemas, example, payload validation, root block parsing) into lib/. Tests stay colocated with their files, matching the rest of the dashboard. No behavior change * feat(ui): link the decision models discussion from the System One beta notice * feat(ui): ask for decision model feedback in the System One beta notice * feat(ui): make the decision model feedback text the discussion link --------- Co-authored-by: ryan <ryan@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
f932e292c5
commit
c8cd885251
18 changed files with 1826 additions and 1 deletions
|
|
@ -0,0 +1,70 @@
|
|||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import JsonEditor from "./JsonEditor";
|
||||
import { validateSystemOnePayload } from "./lib/validatePayload";
|
||||
|
||||
const validPayload = JSON.stringify(
|
||||
{ state: "Hi", questions: { escalate: { type: "noul", instructions: "Escalate?" } } },
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
describe("JsonEditor", () => {
|
||||
it("marks a valid payload as ready to send and counts its lines", () => {
|
||||
render(<JsonEditor value={validPayload} onChange={vi.fn()} validation={validateSystemOnePayload(validPayload)} />);
|
||||
|
||||
expect(screen.getByText("Valid payload")).toBeInTheDocument();
|
||||
expect(screen.getByRole("status")).toHaveTextContent("Ready to send");
|
||||
expect(screen.getByText(`${validPayload.split("\n").length} lines`)).toBeInTheDocument();
|
||||
expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveAttribute("aria-invalid", "false");
|
||||
});
|
||||
|
||||
it("lists each issue with its path and counts only errors in the status badge", () => {
|
||||
const payload = JSON.stringify({
|
||||
state: "Hi",
|
||||
questions: {
|
||||
category: { type: "choice", instructions: 1, criteria: { support: "Help" } },
|
||||
urgency: { type: "score", instructions: "Rate", criteria: ["Low"] },
|
||||
},
|
||||
});
|
||||
render(<JsonEditor value={payload} onChange={vi.fn()} validation={validateSystemOnePayload(payload)} />);
|
||||
|
||||
expect(screen.getByText("2 issues")).toBeInTheDocument();
|
||||
const issues = screen.getByRole("list", { name: "Payload validation issues" });
|
||||
expect(issues).toHaveTextContent("questions.category.instructionsInstructions must be a string.");
|
||||
expect(issues).toHaveTextContent("questions.urgency.criteriaScore criteria must contain at least 2 levels.");
|
||||
expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveAttribute("aria-invalid", "true");
|
||||
});
|
||||
|
||||
it("shows warnings without counting them as issues", () => {
|
||||
const payload = JSON.stringify({
|
||||
state: "Hi",
|
||||
questions: { urgency: { type: "score", instructions: "Rate", criteria: Array.from({ length: 11 }, () => "L") } },
|
||||
});
|
||||
render(<JsonEditor value={payload} onChange={vi.fn()} validation={validateSystemOnePayload(payload)} />);
|
||||
|
||||
expect(screen.getByText("Valid payload")).toBeInTheDocument();
|
||||
expect(screen.getByRole("list", { name: "Payload validation issues" })).toHaveTextContent(
|
||||
"More than 10 score levels may reduce result quality.",
|
||||
);
|
||||
});
|
||||
|
||||
it("numbers every line, including a trailing empty one, so wrapped lines keep their number", () => {
|
||||
const value = `${validPayload}\n`;
|
||||
render(<JsonEditor value={value} onChange={vi.fn()} validation={validateSystemOnePayload(value)} />);
|
||||
|
||||
const lineCount = value.split("\n").length;
|
||||
expect(screen.getByText(`${lineCount} lines`)).toBeInTheDocument();
|
||||
expect(screen.getByText(String(lineCount))).toBeInTheDocument();
|
||||
expect(screen.queryByText(String(lineCount + 1))).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("reports edits to the caller", () => {
|
||||
const onChange = vi.fn();
|
||||
render(<JsonEditor value={validPayload} onChange={onChange} validation={validateSystemOnePayload(validPayload)} />);
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), { target: { value: "{}" } });
|
||||
|
||||
expect(onChange).toHaveBeenCalledWith("{}");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
import { Badge } from "@/components/ui/badge";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
import { CircleAlert, CircleCheck, TriangleAlert } from "lucide-react";
|
||||
import { useId, useMemo, useRef } from "react";
|
||||
import { createElement, PrismLight as SyntaxHighlighter } from "react-syntax-highlighter";
|
||||
import type { SyntaxHighlighterProps } from "react-syntax-highlighter";
|
||||
import json from "react-syntax-highlighter/dist/esm/languages/prism/json";
|
||||
import { findRootBlocks, ROOT_BLOCK_STYLES, type RootBlock } from "./lib/rootBlocks";
|
||||
import type { SystemOnePayloadValidation } from "./lib/validatePayload";
|
||||
|
||||
SyntaxHighlighter.registerLanguage("json", json);
|
||||
|
||||
const EDITOR_TEXT = "m-0 whitespace-pre-wrap wrap-anywhere py-3 font-mono text-xs leading-5 [scrollbar-gutter:stable]";
|
||||
const GUTTER_WIDTH = "w-11";
|
||||
type LineRendererProps = Parameters<NonNullable<SyntaxHighlighterProps["renderer"]>>[0];
|
||||
const CONTENT_INSET = "pl-14 pr-3";
|
||||
const CODE_TAG_PROPS = { className: "language-json", style: { whiteSpace: "pre-wrap" } } as const;
|
||||
|
||||
const TOKEN_COLORS = [
|
||||
"[&_.token.property]:text-sky-700 dark:[&_.token.property]:text-sky-300",
|
||||
"[&_.token.string]:text-emerald-700 dark:[&_.token.string]:text-emerald-300",
|
||||
"[&_.token.number]:text-amber-700 dark:[&_.token.number]:text-amber-300",
|
||||
"[&_.token.boolean]:text-violet-700 dark:[&_.token.boolean]:text-violet-300",
|
||||
"[&_.token.null]:text-violet-700 dark:[&_.token.null]:text-violet-300",
|
||||
"[&_.token.punctuation]:text-muted-foreground [&_.token.operator]:text-muted-foreground",
|
||||
].join(" ");
|
||||
|
||||
interface JsonEditorProps {
|
||||
value: string;
|
||||
onChange: (value: string) => void;
|
||||
validation: SystemOnePayloadValidation;
|
||||
}
|
||||
|
||||
function ValidationStatus({ validation }: { validation: SystemOnePayloadValidation }) {
|
||||
const errorCount = validation.issues.filter((issue) => issue.severity === "error").length;
|
||||
if (errorCount > 0) {
|
||||
return (
|
||||
<Badge variant="destructive">
|
||||
{errorCount} {errorCount === 1 ? "issue" : "issues"}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
return <Badge variant="secondary">Valid payload</Badge>;
|
||||
}
|
||||
|
||||
function IssueList({ id, validation }: { id: string; validation: SystemOnePayloadValidation }) {
|
||||
if (validation.issues.length === 0) {
|
||||
return (
|
||||
<p id={id} role="status" className="flex items-center gap-1.5 border-t px-3 py-2 text-xs text-muted-foreground">
|
||||
<CircleCheck className="size-3.5 text-emerald-600 dark:text-emerald-400" />
|
||||
Ready to send
|
||||
</p>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<ul id={id} aria-label="Payload validation issues" className="grid max-h-36 gap-1 overflow-auto border-t px-3 py-2">
|
||||
{validation.issues.map((issue, index) => (
|
||||
<li key={`${issue.path}-${index}`} className="flex items-start gap-1.5 text-xs">
|
||||
{issue.severity === "error" ? (
|
||||
<CircleAlert className="mt-0.5 size-3.5 shrink-0 text-destructive" />
|
||||
) : (
|
||||
<TriangleAlert className="mt-0.5 size-3.5 shrink-0 text-amber-600 dark:text-amber-400" />
|
||||
)}
|
||||
<code className="shrink-0 rounded bg-muted px-1 font-mono">{issue.path}</code>
|
||||
<span>{issue.message}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
);
|
||||
}
|
||||
|
||||
function renderLines(rootBlocks: readonly RootBlock[]) {
|
||||
return function LineRows({ rows, stylesheet, useInlineStyles }: LineRendererProps) {
|
||||
return rows.map((row, line) => {
|
||||
const lineElement = { node: row, stylesheet, useInlineStyles, key: line };
|
||||
const block = rootBlocks.find(({ startLine, endLine }) => line >= startLine && line <= endLine);
|
||||
return (
|
||||
<div key={line} className="flex">
|
||||
<span className={cn(GUTTER_WIDTH, "shrink-0 select-none pr-3 text-right text-muted-foreground")}>
|
||||
{line + 1}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"relative min-h-5 min-w-0 flex-1 px-3",
|
||||
block && [
|
||||
"before:absolute before:inset-y-0 before:left-0 before:w-0.5",
|
||||
ROOT_BLOCK_STYLES[block.key].band,
|
||||
line === block.startLine && "rounded-t-sm before:rounded-t-sm",
|
||||
line === block.endLine && "rounded-b-sm before:rounded-b-sm",
|
||||
],
|
||||
)}
|
||||
>
|
||||
{createElement(lineElement)}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
export default function JsonEditor({ value, onChange, validation }: JsonEditorProps) {
|
||||
const issuesId = useId();
|
||||
const highlightRef = useRef<HTMLDivElement>(null);
|
||||
const renderer = useMemo(() => renderLines(findRootBlocks(value)), [value]);
|
||||
const lineCount = value.split("\n").length;
|
||||
const hasErrors = !validation.isValid;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"flex min-h-96 flex-1 flex-col overflow-hidden rounded-md border bg-background",
|
||||
hasErrors && "border-destructive/60",
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 border-b px-3 py-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium">Request JSON</span>
|
||||
<ValidationStatus validation={validation} />
|
||||
</div>
|
||||
<span className="text-xs text-muted-foreground tabular-nums">
|
||||
{lineCount} {lineCount === 1 ? "line" : "lines"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="relative min-h-80 flex-1">
|
||||
<div aria-hidden="true" className={cn(GUTTER_WIDTH, "absolute inset-y-0 left-0 border-r bg-muted/50")} />
|
||||
<div
|
||||
ref={highlightRef}
|
||||
aria-hidden="true"
|
||||
className={cn("absolute inset-0 overflow-hidden", EDITOR_TEXT, TOKEN_COLORS)}
|
||||
>
|
||||
<SyntaxHighlighter
|
||||
language="json"
|
||||
style={{}}
|
||||
useInlineStyles={false}
|
||||
PreTag="div"
|
||||
codeTagProps={CODE_TAG_PROPS}
|
||||
renderer={renderer}
|
||||
>
|
||||
{value}
|
||||
</SyntaxHighlighter>
|
||||
</div>
|
||||
<textarea
|
||||
aria-label="System One JSON payload"
|
||||
aria-invalid={hasErrors}
|
||||
aria-describedby={issuesId}
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onScroll={(event) => {
|
||||
if (highlightRef.current) {
|
||||
highlightRef.current.scrollTop = event.currentTarget.scrollTop;
|
||||
}
|
||||
}}
|
||||
spellCheck={false}
|
||||
autoCapitalize="off"
|
||||
autoComplete="off"
|
||||
placeholder="Paste or write a System One request"
|
||||
className={cn(
|
||||
EDITOR_TEXT,
|
||||
CONTENT_INSET,
|
||||
"absolute inset-0 size-full resize-none overflow-y-auto bg-transparent text-transparent caret-foreground outline-none selection:bg-primary/20 placeholder:text-muted-foreground",
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
<IssueList id={issuesId} validation={validation} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import QuestionBreakdown from "./QuestionBreakdown";
|
||||
import { validateSystemOnePayload } from "./lib/validatePayload";
|
||||
|
||||
const previewOf = (questions: object) => validateSystemOnePayload(JSON.stringify({ state: "Hi", questions })).payload;
|
||||
|
||||
describe("QuestionBreakdown", () => {
|
||||
it("previews every noul criterion that will be sent, not only true and false", () => {
|
||||
render(
|
||||
<QuestionBreakdown
|
||||
payload={previewOf({
|
||||
escalate: {
|
||||
type: "noul",
|
||||
instructions: "Escalate?",
|
||||
criteria: { true: "Asks for a person", unsure: "Maybe" },
|
||||
},
|
||||
})}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Asks for a person")).toBeInTheDocument();
|
||||
expect(screen.getByText("unsure")).toBeInTheDocument();
|
||||
expect(screen.getByText("Maybe")).toBeInTheDocument();
|
||||
expect(screen.queryByText("No criteria defined")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("says when a noul question has no criteria", () => {
|
||||
render(<QuestionBreakdown payload={previewOf({ escalate: { type: "noul", instructions: "Escalate?" } })} />);
|
||||
|
||||
expect(screen.getByText("No criteria defined")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { Badge } from "@/components/ui/badge";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
import { ROOT_BLOCK_STYLES } from "./lib/rootBlocks";
|
||||
import type { SystemOneQuestion, SystemOneRequest } from "./lib/schemas";
|
||||
|
||||
function formatState(state: unknown): string {
|
||||
if (typeof state === "string") {
|
||||
return state;
|
||||
}
|
||||
return JSON.stringify(state, null, 2) ?? String(state);
|
||||
}
|
||||
|
||||
function QuestionCriteria({ question }: { question: SystemOneQuestion }) {
|
||||
if (question.type === "choice") {
|
||||
return (
|
||||
<dl className="grid gap-2">
|
||||
{Object.entries(question.criteria).map(([label, description]) => (
|
||||
<div key={label} className="grid gap-0.5 sm:grid-cols-[minmax(7rem,auto)_1fr] sm:gap-3">
|
||||
<dt className="font-mono text-xs font-medium">{label}</dt>
|
||||
<dd className="text-xs text-muted-foreground">{description}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
);
|
||||
}
|
||||
|
||||
if (question.type === "noul") {
|
||||
const criteria = Object.entries(question.criteria ?? {}).filter(([, description]) => description !== undefined);
|
||||
return criteria.length > 0 ? (
|
||||
<dl className="grid gap-2">
|
||||
{criteria.map(([label, description]) => (
|
||||
<div key={label} className="grid gap-0.5 sm:grid-cols-[minmax(7rem,auto)_1fr] sm:gap-3">
|
||||
<dt className="text-xs font-medium">{label}</dt>
|
||||
<dd className="text-xs text-muted-foreground">
|
||||
{typeof description === "string" ? description : JSON.stringify(description)}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
) : (
|
||||
<p className="text-xs text-muted-foreground">No criteria defined</p>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ol className="grid gap-2">
|
||||
{question.criteria.map((description, index) => (
|
||||
<li key={`${index}-${description}`} className="grid gap-0.5 sm:grid-cols-[minmax(7rem,auto)_1fr] sm:gap-3">
|
||||
<span className="font-mono text-xs font-medium">{index}</span>
|
||||
<span className="text-xs text-muted-foreground">{description}</span>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
);
|
||||
}
|
||||
|
||||
export default function QuestionBreakdown({ payload }: { payload?: SystemOneRequest }) {
|
||||
if (!payload) {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Question breakdown</CardTitle>
|
||||
<CardDescription>Enter a valid request to preview its state and questions.</CardDescription>
|
||||
</CardHeader>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Question breakdown</CardTitle>
|
||||
<CardDescription>Review the state and criteria that will be sent.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-4 wrap-anywhere">
|
||||
<section
|
||||
aria-labelledby="system-one-state-heading"
|
||||
className={cn("grid gap-2 border-l-2 pl-3", ROOT_BLOCK_STYLES.state.accent)}
|
||||
>
|
||||
<h3 id="system-one-state-heading" className="text-sm font-medium">
|
||||
State
|
||||
</h3>
|
||||
<pre className="max-h-48 overflow-y-auto whitespace-pre-wrap rounded-md bg-muted p-3 font-mono text-xs">
|
||||
{formatState(payload.state)}
|
||||
</pre>
|
||||
</section>
|
||||
<div className={cn("grid gap-3 border-l-2 pl-3", ROOT_BLOCK_STYLES.questions.accent)}>
|
||||
{Object.entries(payload.questions).map(([id, question]) => (
|
||||
<section key={id} className="grid gap-3 rounded-md border p-3" aria-labelledby={`question-${id}`}>
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<h3 id={`question-${id}`} className="font-mono text-sm font-medium">
|
||||
{id}
|
||||
</h3>
|
||||
<Badge variant="secondary">{question.type}</Badge>
|
||||
</div>
|
||||
<p className="text-sm">{question.instructions}</p>
|
||||
<div className="grid gap-2">
|
||||
<h4 className="text-xs font-medium text-muted-foreground">Criteria</h4>
|
||||
<QuestionCriteria question={question} />
|
||||
</div>
|
||||
</section>
|
||||
))}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import ResponseView from "./ResponseView";
|
||||
import type { SystemOneResponse } from "./lib/schemas";
|
||||
|
||||
const sampleResponse: SystemOneResponse = {
|
||||
model: "jev-1.13.0",
|
||||
answers: {
|
||||
human: { type: "noul", noul: 0.99 },
|
||||
dept: {
|
||||
type: "choice",
|
||||
choice: "technical",
|
||||
confidence: 0.27,
|
||||
probabilities: { billing: 0.36, technical: 0.64 },
|
||||
},
|
||||
frustration: {
|
||||
type: "score",
|
||||
score: 2,
|
||||
confidence: 0.99,
|
||||
legend: { "0": "calm", "1": "mild", "2": "angry" },
|
||||
probabilities: { "0": 0, "1": 0, "2": 1 },
|
||||
},
|
||||
},
|
||||
usage: { input_tokens: 378, output_tokens: 63 },
|
||||
};
|
||||
|
||||
describe("ResponseView", () => {
|
||||
it("renders calibrated probabilities for choice, noul, and score answers", () => {
|
||||
render(<ResponseView response={sampleResponse} isLoading={false} latencyMs={120} />);
|
||||
|
||||
expect(screen.getAllByText("technical")).toHaveLength(2);
|
||||
const meters = screen.getAllByRole("meter");
|
||||
expect(meters.map((meter) => meter.getAttribute("aria-label"))).toEqual([
|
||||
"Yes probability",
|
||||
"technical probability",
|
||||
"billing probability",
|
||||
"0: calm probability",
|
||||
"1: mild probability",
|
||||
"2: angry probability",
|
||||
]);
|
||||
expect(screen.getByRole("meter", { name: "technical probability" })).toHaveAttribute("aria-valuenow", "64");
|
||||
expect(screen.getByRole("meter", { name: "billing probability" })).toHaveAttribute("aria-valuenow", "36");
|
||||
expect(screen.getByText("99% yes")).toBeInTheDocument();
|
||||
expect(screen.getByRole("meter", { name: "Yes probability" })).toHaveAttribute("aria-valuenow", "99");
|
||||
expect(screen.getByRole("meter", { name: "2: angry probability" })).toBeInTheDocument();
|
||||
expect(screen.getByText("378 input / 63 output tokens")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("selects the score level nearest a fractional calibrated score", () => {
|
||||
const fractional: SystemOneResponse = {
|
||||
answers: {
|
||||
severity: { type: "score", score: 2.39, probabilities: { "1": 0, "2": 0.62, "3": 0.38 } },
|
||||
},
|
||||
};
|
||||
render(<ResponseView response={fractional} isLoading={false} />);
|
||||
|
||||
expect(screen.getByRole("meter", { name: "2 probability" })).toHaveAttribute("aria-valuetext", "62%, selected");
|
||||
expect(screen.getByRole("meter", { name: "3 probability" })).toHaveAttribute("aria-valuetext", "38%");
|
||||
expect(screen.getByRole("meter", { name: "1 probability" })).toHaveAttribute("aria-valuetext", "0%");
|
||||
});
|
||||
|
||||
it("shows an inline error message", () => {
|
||||
render(<ResponseView error="Upstream request failed" isLoading={false} />);
|
||||
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("Upstream request failed");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
import { ChevronDown, LoaderCircle } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import type { SystemOneAnswer, SystemOneResponse } from "./lib/schemas";
|
||||
|
||||
interface ResponseViewProps {
|
||||
response?: SystemOneResponse;
|
||||
fallbackModel?: string;
|
||||
latencyMs?: number;
|
||||
error?: string;
|
||||
isLoading: boolean;
|
||||
}
|
||||
|
||||
function ProbabilityMeter({
|
||||
label,
|
||||
probability,
|
||||
selected = false,
|
||||
}: {
|
||||
label: string;
|
||||
probability: number;
|
||||
selected?: boolean;
|
||||
}) {
|
||||
const percentage = Math.round(probability * 100);
|
||||
return (
|
||||
<div className="grid gap-1.5">
|
||||
<div className="flex items-center justify-between gap-2 text-xs">
|
||||
<span className={selected ? "font-semibold" : "text-muted-foreground"}>{label}</span>
|
||||
<span className={selected ? "font-semibold tabular-nums" : "tabular-nums"}>{percentage}%</span>
|
||||
</div>
|
||||
<div
|
||||
role="meter"
|
||||
aria-label={`${label} probability`}
|
||||
aria-valuenow={percentage}
|
||||
aria-valuetext={selected ? `${percentage}%, selected` : `${percentage}%`}
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
className="h-2 overflow-hidden rounded-full bg-muted"
|
||||
>
|
||||
<div
|
||||
className={`h-full rounded-full ${selected ? "bg-primary" : "bg-primary/40"}`}
|
||||
style={{ width: `${percentage}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
|
||||
if (answer.type === "noul") {
|
||||
const percentage = Math.round(answer.noul * 100);
|
||||
return (
|
||||
<div className="grid gap-3">
|
||||
<p className="text-sm font-medium">{percentage}% yes</p>
|
||||
<ProbabilityMeter label="Yes" probability={answer.noul} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (answer.type === "choice") {
|
||||
const options = Object.entries(answer.probabilities).sort(([, first], [, second]) => second - first);
|
||||
return (
|
||||
<div className="grid gap-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-xs text-muted-foreground">Selected choice</span>
|
||||
<Badge>{answer.choice}</Badge>
|
||||
{answer.confidence !== undefined && (
|
||||
<Badge variant="outline">{Math.round(answer.confidence * 100)}% confidence</Badge>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid gap-3">
|
||||
{options.map(([label, probability]) => (
|
||||
<ProbabilityMeter key={label} label={label} probability={probability} selected={label === answer.choice} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const levels = Object.entries(answer.probabilities).sort(([first], [second]) => Number(first) - Number(second));
|
||||
return (
|
||||
<div className="grid gap-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-xs text-muted-foreground">Score</span>
|
||||
<Badge>{answer.score}</Badge>
|
||||
{answer.confidence !== undefined && (
|
||||
<Badge variant="outline">{Math.round(answer.confidence * 100)}% confidence</Badge>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid gap-3">
|
||||
{levels.map(([level, probability]) => {
|
||||
const label = answer.legend?.[level] ? `${level}: ${answer.legend[level]}` : level;
|
||||
return (
|
||||
<ProbabilityMeter
|
||||
key={level}
|
||||
label={label}
|
||||
probability={probability}
|
||||
selected={Number(level) === Math.round(answer.score)}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function ResponseView({ response, fallbackModel, latencyMs, error, isLoading }: ResponseViewProps) {
|
||||
const [showRaw, setShowRaw] = useState(false);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<Card>
|
||||
<CardContent className="flex items-center justify-center gap-2 py-10 text-sm text-muted-foreground">
|
||||
<LoaderCircle className="size-4 animate-spin" />
|
||||
Calculating calibrated probabilities
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Alert variant="destructive">
|
||||
<AlertTitle>System One request failed</AlertTitle>
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
</Alert>
|
||||
);
|
||||
}
|
||||
|
||||
if (!response) {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Calibrated probabilities</CardTitle>
|
||||
<CardDescription>Send a request to see System One answers and probabilities.</CardDescription>
|
||||
</CardHeader>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
const model = response.model ?? fallbackModel;
|
||||
|
||||
return (
|
||||
<Card className="wrap-anywhere">
|
||||
<CardHeader>
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div className="grid gap-1">
|
||||
<CardTitle>Calibrated probabilities</CardTitle>
|
||||
{model && <CardDescription>{model}</CardDescription>}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2 text-xs">
|
||||
{latencyMs !== undefined && <Badge variant="outline">{Math.round(latencyMs)} ms</Badge>}
|
||||
{response.usage && (
|
||||
<Badge variant="outline">
|
||||
{response.usage.input_tokens} input / {response.usage.output_tokens} output tokens
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-3">
|
||||
{Object.entries(response.answers).map(([id, answer]) => (
|
||||
<Card key={id} size="sm">
|
||||
<CardHeader>
|
||||
<CardTitle className="font-mono">{id}</CardTitle>
|
||||
<CardAction>
|
||||
<Badge variant="secondary">{answer.type}</Badge>
|
||||
</CardAction>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<AnswerDetails answer={answer} />
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
<Collapsible open={showRaw} onOpenChange={setShowRaw}>
|
||||
<CollapsibleTrigger
|
||||
render={
|
||||
<Button variant="ghost" size="sm">
|
||||
<ChevronDown className="size-4" />
|
||||
Raw response
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
<CollapsibleContent>
|
||||
<pre className="whitespace-pre-wrap wrap-anywhere rounded-md bg-muted p-3 font-mono text-xs">
|
||||
{JSON.stringify(response, null, 2)}
|
||||
</pre>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { fireEvent, render as rtlRender, screen, waitForElementToBeRemoved } from "@testing-library/react";
|
||||
import type { ReactElement } from "react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import SystemOneUI from "./SystemOneUI";
|
||||
import type { SystemOneResponse } from "./lib/schemas";
|
||||
|
||||
const responseBody: SystemOneResponse = {
|
||||
model: "jev-1.13.0",
|
||||
answers: {
|
||||
department: {
|
||||
type: "choice",
|
||||
choice: "technical",
|
||||
confidence: 0.9,
|
||||
probabilities: { technical: 0.9, billing: 0.1 },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const render = (ui: ReactElement) =>
|
||||
rtlRender(
|
||||
<QueryClientProvider client={new QueryClient({ defaultOptions: { mutations: { retry: false } } })}>
|
||||
{ui}
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
const createResponse = (body: SystemOneResponse, status = 200, errorText = "") =>
|
||||
({
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
text: async () => (status >= 200 && status < 300 ? JSON.stringify(body) : errorText),
|
||||
}) as Response;
|
||||
|
||||
describe("SystemOneUI integration", () => {
|
||||
const mockFetch = vi.fn<typeof fetch>();
|
||||
|
||||
beforeEach(() => {
|
||||
sessionStorage.clear();
|
||||
mockFetch.mockReset();
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
mockFetch.mockResolvedValue(createResponse(responseBody));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
sessionStorage.clear();
|
||||
});
|
||||
|
||||
it("restores the example after the request is edited", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
const editor = screen.getByRole("textbox", { name: "System One JSON payload" });
|
||||
const example = (editor as HTMLTextAreaElement).value;
|
||||
const resetButton = screen.getByRole("button", { name: "Reset example" });
|
||||
expect(resetButton).toBeDisabled();
|
||||
|
||||
fireEvent.change(editor, { target: { value: "{}" } });
|
||||
expect(resetButton).toBeEnabled();
|
||||
await user.click(resetButton);
|
||||
|
||||
expect(editor).toHaveValue(example);
|
||||
expect(resetButton).toBeDisabled();
|
||||
});
|
||||
|
||||
it("flags the tab as a TypeSafe-only beta without announcing it as an alert", () => {
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
expect(screen.getByRole("note", { name: "System One beta notice" })).toHaveTextContent(
|
||||
"Support for more System One-compatible models is in progress.",
|
||||
);
|
||||
expect(screen.getByRole("link", { name: "Give us feedback on what you want for decision models" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://github.com/BerriAI/litellm/discussions/44231",
|
||||
);
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows invalid JSON and disables Send", async () => {
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), {
|
||||
target: { value: "{" },
|
||||
});
|
||||
|
||||
expect(await screen.findByText(/Invalid JSON syntax/)).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Send" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("posts the request with the session key and renders calibrated answers", async () => {
|
||||
const user = userEvent.setup();
|
||||
sessionStorage.setItem("customProxyBaseUrl", "https://stale.example.com/");
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
sessionStorage.setItem("customProxyBaseUrl", "https://tenant.example.com/");
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
|
||||
expect(await screen.findAllByText("technical")).toHaveLength(2);
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
const [url, options] = mockFetch.mock.calls[0] ?? [];
|
||||
expect(url).toBe("https://tenant.example.com/typesafe/v1/systemone");
|
||||
expect(options?.method).toBe("POST");
|
||||
expect(options?.headers).toMatchObject({ "Content-Type": "application/json" });
|
||||
expect(Object.values(options?.headers as Record<string, string>)).toContain("Bearer session-key");
|
||||
expect(JSON.parse(options?.body as string)).toMatchObject({
|
||||
model: "jev-latest",
|
||||
questions: { has_repro_steps: { type: "noul" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the requested model when the response omits it", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetch.mockResolvedValueOnce(
|
||||
createResponse({
|
||||
answers: responseBody.answers,
|
||||
}),
|
||||
);
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
|
||||
expect(await screen.findByText("jev-latest")).toBeInTheDocument();
|
||||
expect(screen.getByText("Selected choice")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the preview usable when question values have invalid types", () => {
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), {
|
||||
target: {
|
||||
value: JSON.stringify({
|
||||
state: "A new support request",
|
||||
questions: {
|
||||
category: {
|
||||
type: "choice",
|
||||
instructions: { text: "Route the message" },
|
||||
criteria: { support: { text: "Help" } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
expect(screen.getByText("Instructions must be a string.")).toBeInTheDocument();
|
||||
expect(screen.getByText("Choice descriptions must be strings.")).toBeInTheDocument();
|
||||
expect(screen.getByText("Enter a valid request to preview its state and questions.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders upstream errors inline", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetch.mockResolvedValueOnce(createResponse(responseBody, 401, "Virtual key rejected"));
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Virtual key rejected");
|
||||
});
|
||||
|
||||
it("renders malformed success responses inline", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
text: async () =>
|
||||
JSON.stringify({
|
||||
model: "jev-1.13.0",
|
||||
answers: {
|
||||
department: {
|
||||
type: "choice",
|
||||
choice: "technical",
|
||||
probabilities: null,
|
||||
},
|
||||
},
|
||||
}),
|
||||
} as Response);
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("System One response has an invalid shape.");
|
||||
expect(screen.queryByText("jev-1.13.0")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("clears a previous answer when the request editor changes", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), {
|
||||
target: {
|
||||
value: JSON.stringify({
|
||||
state: "A different support request",
|
||||
questions: {
|
||||
department: {
|
||||
type: "choice",
|
||||
instructions: "Where should this go?",
|
||||
criteria: { billing: "Billing questions" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
expect(screen.queryByText("Selected choice")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("jev-1.13.0")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("clears the answer when the UI session key changes", async () => {
|
||||
const user = userEvent.setup();
|
||||
const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } });
|
||||
const view = rtlRender(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<SystemOneUI accessToken="session-key" />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
||||
|
||||
view.rerender(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<SystemOneUI accessToken="new-session-key" />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
await waitForElementToBeRemoved(() => screen.queryByText("Selected choice"));
|
||||
});
|
||||
|
||||
it("aborts an in-flight request when cancelled", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetch.mockImplementation(
|
||||
(_input, init) =>
|
||||
new Promise<Response>((_, reject) => {
|
||||
init?.signal?.addEventListener("abort", () => reject(new DOMException("Request aborted", "AbortError")), {
|
||||
once: true,
|
||||
});
|
||||
}),
|
||||
);
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
await user.click(await screen.findByRole("button", { name: "Cancel request" }));
|
||||
|
||||
expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);
|
||||
expect(screen.getByRole("button", { name: "Send" })).toBeEnabled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
"use client";
|
||||
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { makeSystemOneRequest } from "../../llm_calls/system_one";
|
||||
import { SYSTEM_ONE_EXAMPLE } from "./lib/example";
|
||||
import type { SystemOneRequest } from "./lib/schemas";
|
||||
import JsonEditor from "./JsonEditor";
|
||||
import QuestionBreakdown from "./QuestionBreakdown";
|
||||
import ResponseView from "./ResponseView";
|
||||
import { validateSystemOnePayload } from "./lib/validatePayload";
|
||||
|
||||
interface SystemOneUIProps {
|
||||
accessToken: string | null;
|
||||
disabledPersonalKeyCreation?: boolean;
|
||||
}
|
||||
|
||||
type ApiKeySource = "session" | "custom";
|
||||
|
||||
interface SystemOneSendVariables {
|
||||
payload: SystemOneRequest;
|
||||
apiKey: string;
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
const EXAMPLE_PAYLOAD = JSON.stringify(SYSTEM_ONE_EXAMPLE, null, 2);
|
||||
const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231";
|
||||
|
||||
function getCustomProxyBaseUrl(): string | undefined {
|
||||
return typeof window === "undefined" ? undefined : window.sessionStorage.getItem("customProxyBaseUrl") || undefined;
|
||||
}
|
||||
|
||||
export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = false }: SystemOneUIProps) {
|
||||
const [apiKeySource, setApiKeySource] = useState<ApiKeySource>(disabledPersonalKeyCreation ? "custom" : "session");
|
||||
const [customApiKey, setCustomApiKey] = useState("");
|
||||
const [rawPayload, setRawPayload] = useState(EXAMPLE_PAYLOAD);
|
||||
const activeController = useRef<AbortController | null>(null);
|
||||
const validation = useMemo(() => validateSystemOnePayload(rawPayload), [rawPayload]);
|
||||
const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim();
|
||||
const hasSyntaxError = validation.issues.some((issue) => issue.path === "syntax");
|
||||
|
||||
const systemOne = useMutation({
|
||||
mutationFn: ({ payload, apiKey, signal }: SystemOneSendVariables) =>
|
||||
makeSystemOneRequest(payload, apiKey, getCustomProxyBaseUrl(), signal),
|
||||
});
|
||||
const isLoading = systemOne.isPending;
|
||||
const { reset: resetSystemOne } = systemOne;
|
||||
|
||||
useEffect(() => () => activeController.current?.abort(), []);
|
||||
|
||||
useEffect(() => {
|
||||
activeController.current?.abort();
|
||||
activeController.current = null;
|
||||
resetSystemOne();
|
||||
}, [effectiveApiKey, resetSystemOne]);
|
||||
|
||||
function clearRequestState() {
|
||||
activeController.current?.abort();
|
||||
activeController.current = null;
|
||||
systemOne.reset();
|
||||
}
|
||||
|
||||
function handlePayloadChange(value: string) {
|
||||
if (value !== rawPayload) {
|
||||
clearRequestState();
|
||||
setRawPayload(value);
|
||||
}
|
||||
}
|
||||
|
||||
function handleFormatJson() {
|
||||
if (rawPayload.trim() && !hasSyntaxError) {
|
||||
handlePayloadChange(JSON.stringify(JSON.parse(rawPayload), null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
function handleSend() {
|
||||
if (!validation.payload || !effectiveApiKey) {
|
||||
return;
|
||||
}
|
||||
clearRequestState();
|
||||
const controller = new AbortController();
|
||||
activeController.current = controller;
|
||||
systemOne.mutate({ payload: validation.payload, apiKey: effectiveApiKey, signal: controller.signal });
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col gap-4 overflow-auto p-4 xl:overflow-hidden">
|
||||
<section className="grid gap-3">
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-sm font-medium text-muted-foreground">Virtual Key Source</span>
|
||||
<Select
|
||||
value={apiKeySource}
|
||||
onValueChange={(value) => {
|
||||
if (value === "session" || value === "custom") {
|
||||
setApiKeySource(value);
|
||||
}
|
||||
}}
|
||||
disabled={disabledPersonalKeyCreation}
|
||||
>
|
||||
<SelectTrigger className="w-48" aria-label="Virtual Key Source">
|
||||
<SelectValue>{apiKeySource === "custom" ? "Virtual Key" : "Current UI Session"}</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="session" disabled={!accessToken}>
|
||||
Current UI Session
|
||||
</SelectItem>
|
||||
<SelectItem value="custom">Virtual Key</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{apiKeySource === "custom" && (
|
||||
<Input
|
||||
type="password"
|
||||
aria-label="Virtual Key"
|
||||
value={customApiKey}
|
||||
onChange={(event) => setCustomApiKey(event.target.value)}
|
||||
placeholder="Enter Virtual Key"
|
||||
className="w-56"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => handlePayloadChange(EXAMPLE_PAYLOAD)}
|
||||
disabled={rawPayload === EXAMPLE_PAYLOAD}
|
||||
>
|
||||
<RotateCcw />
|
||||
Reset example
|
||||
</Button>
|
||||
<Button variant="outline" onClick={handleFormatJson} disabled={!rawPayload.trim() || hasSyntaxError}>
|
||||
<Code />
|
||||
Format JSON
|
||||
</Button>
|
||||
{isLoading && (
|
||||
<Button variant="outline" onClick={clearRequestState}>
|
||||
Cancel request
|
||||
</Button>
|
||||
)}
|
||||
<Button onClick={handleSend} disabled={!validation.isValid || isLoading || !effectiveApiKey}>
|
||||
{isLoading ? <LoaderCircle className="animate-spin" /> : <Send />}
|
||||
Send
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<Alert role="note" aria-label="System One beta notice">
|
||||
<Info />
|
||||
<AlertTitle>Beta: TypeSafe Jev only for now</AlertTitle>
|
||||
<AlertDescription>
|
||||
Sends System One requests (choice, noul, score) through /typesafe/v1/systemone and requires TYPESAFE_API_KEY
|
||||
on the proxy. Support for more System One-compatible models is in progress.{" "}
|
||||
<a href={DECISION_MODELS_DISCUSSION_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
Give us feedback on what you want for decision models
|
||||
</a>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</section>
|
||||
|
||||
<div className="grid gap-4 xl:min-h-0 xl:flex-1 xl:grid-cols-2">
|
||||
<section className="flex min-h-96 flex-col xl:min-h-0" aria-label="System One request editor">
|
||||
<JsonEditor value={rawPayload} onChange={handlePayloadChange} validation={validation} />
|
||||
</section>
|
||||
<section className="grid content-start gap-4 xl:min-h-0 xl:overflow-auto" aria-label="System One results">
|
||||
<ResponseView
|
||||
response={systemOne.data?.response}
|
||||
fallbackModel={validation.payload?.model}
|
||||
latencyMs={systemOne.data?.latencyMs}
|
||||
error={systemOne.error?.message}
|
||||
isLoading={isLoading}
|
||||
/>
|
||||
<QuestionBreakdown payload={validation.payload} />
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import type { SystemOneRequest } from "./schemas";
|
||||
|
||||
export const SYSTEM_ONE_EXAMPLE: SystemOneRequest = {
|
||||
model: "jev-latest",
|
||||
state:
|
||||
"Since upgrading to the latest release, streaming responses stop halfway through whenever a fallback model takes over. Non-streaming requests still work. I haven't narrowed down which change caused it, but it happens on most long prompts.",
|
||||
questions: {
|
||||
area: {
|
||||
type: "choice",
|
||||
instructions: "Which part of the project does this issue belong to?",
|
||||
criteria: {
|
||||
backend: "Server, API routing, streaming, and fallbacks",
|
||||
sdk: "The client library and its helpers",
|
||||
ui: "The web dashboard",
|
||||
docs: "Documentation and examples",
|
||||
},
|
||||
},
|
||||
has_repro_steps: {
|
||||
type: "noul",
|
||||
instructions: "Does the issue include steps someone could follow to reproduce it?",
|
||||
criteria: {
|
||||
true: "It gives concrete steps, a config, or a request that triggers the bug",
|
||||
false: "It only describes the symptom without a way to trigger it",
|
||||
},
|
||||
},
|
||||
severity: {
|
||||
type: "score",
|
||||
instructions: "How severe is this issue?",
|
||||
criteria: [
|
||||
"Cosmetic or a typo",
|
||||
"Minor bug with an easy workaround",
|
||||
"Broken feature with a workaround",
|
||||
"Broken feature with no workaround",
|
||||
"Outage or data loss",
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { findRootBlocks } from "./rootBlocks";
|
||||
|
||||
describe("findRootBlocks", () => {
|
||||
it("maps each top-level request key to the lines its value spans", () => {
|
||||
const text = JSON.stringify(
|
||||
{
|
||||
model: "jev-latest",
|
||||
state: { message: "Hi" },
|
||||
questions: { escalate: { type: "noul", instructions: "Escalate?" } },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
expect(findRootBlocks(text)).toEqual([
|
||||
{ key: "model", startLine: 1, endLine: 1 },
|
||||
{ key: "state", startLine: 2, endLine: 4 },
|
||||
{ key: "questions", startLine: 5, endLine: 10 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("ignores nested keys, unrelated root keys and braces or quotes inside strings", () => {
|
||||
const text = [
|
||||
"{",
|
||||
' "temperature": 0,',
|
||||
' "state": "a } tricky \\" { string",',
|
||||
' "questions": {',
|
||||
' "model": { "state": "nested" }',
|
||||
" }",
|
||||
"}",
|
||||
].join("\n");
|
||||
|
||||
expect(findRootBlocks(text)).toEqual([
|
||||
{ key: "state", startLine: 2, endLine: 2 },
|
||||
{ key: "questions", startLine: 3, endLine: 5 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("finds every block on a single minified line", () => {
|
||||
expect(findRootBlocks('{"state":"Hi","questions":{"a":{"type":"noul"}}}')).toEqual([
|
||||
{ key: "state", startLine: 0, endLine: 0 },
|
||||
{ key: "questions", startLine: 0, endLine: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps highlighting an unfinished block while the user is typing", () => {
|
||||
expect(findRootBlocks('{\n "state": "Hi",\n "questions": {\n "a": {')).toEqual([
|
||||
{ key: "state", startLine: 1, endLine: 1 },
|
||||
{ key: "questions", startLine: 2, endLine: 3 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
export type RootBlockKey = "model" | "state" | "questions";
|
||||
|
||||
export interface RootBlock {
|
||||
readonly key: RootBlockKey;
|
||||
readonly startLine: number;
|
||||
readonly endLine: number;
|
||||
}
|
||||
|
||||
export const ROOT_BLOCK_STYLES: Readonly<Record<RootBlockKey, { band: string; accent: string }>> = {
|
||||
model: { band: "bg-blue-500/10 before:bg-blue-500/60", accent: "border-l-blue-500/60" },
|
||||
state: { band: "bg-green-500/10 before:bg-green-500/60", accent: "border-l-green-500/60" },
|
||||
questions: { band: "bg-orange-500/10 before:bg-orange-500/60", accent: "border-l-orange-500/60" },
|
||||
};
|
||||
|
||||
interface Token {
|
||||
readonly text: string;
|
||||
readonly line: number;
|
||||
}
|
||||
|
||||
interface ScanState {
|
||||
readonly depth: number;
|
||||
readonly previous?: Token;
|
||||
readonly open?: { readonly key: RootBlockKey; readonly startLine: number };
|
||||
readonly blocks: readonly RootBlock[];
|
||||
}
|
||||
|
||||
const JSON_TOKEN = /"(?:[^"\\\n]|\\.)*"?|[{}[\],:]|[^\s{}[\],:"]+/g;
|
||||
|
||||
const isRootBlockKey = (key: string): key is RootBlockKey => Object.hasOwn(ROOT_BLOCK_STYLES, key);
|
||||
|
||||
const tokenize = (text: string): Token[] =>
|
||||
text
|
||||
.split("\n")
|
||||
.flatMap((lineText, line) => Array.from(lineText.matchAll(JSON_TOKEN), ([match]) => ({ text: match, line })));
|
||||
|
||||
const closeOpenBlock = ({ open, previous, blocks }: ScanState): readonly RootBlock[] =>
|
||||
open && previous ? [...blocks, { ...open, endLine: previous.line }] : blocks;
|
||||
|
||||
function step(state: ScanState, token: Token): ScanState {
|
||||
const next = { ...state, previous: token };
|
||||
switch (token.text) {
|
||||
case "{":
|
||||
case "[":
|
||||
return { ...next, depth: state.depth + 1 };
|
||||
case "}":
|
||||
case "]":
|
||||
return state.depth === 1
|
||||
? { ...next, depth: 0, open: undefined, blocks: closeOpenBlock(state) }
|
||||
: { ...next, depth: state.depth - 1 };
|
||||
case ",":
|
||||
return state.depth === 1 ? { ...next, open: undefined, blocks: closeOpenBlock(state) } : next;
|
||||
case ":": {
|
||||
const key = state.previous?.text.slice(1, -1) ?? "";
|
||||
return state.depth === 1 && state.previous?.text.startsWith('"') && isRootBlockKey(key)
|
||||
? { ...next, open: { key, startLine: state.previous.line } }
|
||||
: next;
|
||||
}
|
||||
default:
|
||||
return next;
|
||||
}
|
||||
}
|
||||
|
||||
export function findRootBlocks(text: string): readonly RootBlock[] {
|
||||
return closeOpenBlock(tokenize(text).reduce(step, { depth: 0, blocks: [] }));
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
import { z } from "zod";
|
||||
|
||||
const MAX_CHOICE_OPTIONS = 255;
|
||||
|
||||
const nonEmptyString = (message: string) =>
|
||||
z.string({ invalid_type_error: message }).refine((value) => value.trim().length > 0, message);
|
||||
|
||||
const instructions = z.string({
|
||||
required_error: "Required property 'instructions' is missing.",
|
||||
invalid_type_error: "Instructions must be a string.",
|
||||
});
|
||||
|
||||
const choiceQuestionSchema = z
|
||||
.object({
|
||||
type: z.literal("choice"),
|
||||
instructions,
|
||||
criteria: z
|
||||
.record(z.string({ invalid_type_error: "Choice descriptions must be strings." }), {
|
||||
required_error: "Choice criteria must be an object mapping labels to descriptions.",
|
||||
invalid_type_error: "Choice criteria must be an object mapping labels to descriptions.",
|
||||
})
|
||||
.refine((criteria) => {
|
||||
const count = Object.keys(criteria).length;
|
||||
return count >= 1 && count <= MAX_CHOICE_OPTIONS;
|
||||
}, `Choice criteria must contain between 1 and ${MAX_CHOICE_OPTIONS} options.`),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
const noulQuestionSchema = z
|
||||
.object({
|
||||
type: z.literal("noul"),
|
||||
instructions,
|
||||
criteria: z
|
||||
.object(
|
||||
{
|
||||
true: z.string({ invalid_type_error: "Noul true criteria must be a string." }).optional(),
|
||||
false: z.string({ invalid_type_error: "Noul false criteria must be a string." }).optional(),
|
||||
},
|
||||
{ invalid_type_error: "Noul criteria, when provided, must be an object." },
|
||||
)
|
||||
.passthrough()
|
||||
.optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
const scoreQuestionSchema = z
|
||||
.object({
|
||||
type: z.literal("score"),
|
||||
instructions,
|
||||
criteria: z
|
||||
.array(z.string({ invalid_type_error: "Score levels must be strings." }), {
|
||||
required_error: "Score criteria must be an array of levels.",
|
||||
invalid_type_error: "Score criteria must be an array of levels.",
|
||||
})
|
||||
.min(2, "Score criteria must contain at least 2 levels."),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
const QUESTION_ERROR_MESSAGES: Partial<Record<z.ZodIssueCode, string>> = {
|
||||
[z.ZodIssueCode.invalid_union_discriminator]: "Question type must be choice, noul, or score.",
|
||||
[z.ZodIssueCode.invalid_type]: "Question must be an object.",
|
||||
};
|
||||
|
||||
const questionSchema = z.discriminatedUnion("type", [choiceQuestionSchema, noulQuestionSchema, scoreQuestionSchema], {
|
||||
errorMap: (issue, context) => ({ message: QUESTION_ERROR_MESSAGES[issue.code] ?? context.defaultError }),
|
||||
});
|
||||
|
||||
export const systemOneRequestSchema = z
|
||||
.object(
|
||||
{
|
||||
model: nonEmptyString("Model must be a non-empty string.").optional(),
|
||||
state: z.unknown(),
|
||||
questions: z
|
||||
.record(questionSchema, {
|
||||
required_error: "Required property 'questions' is missing.",
|
||||
invalid_type_error: "Questions must be a non-empty object.",
|
||||
})
|
||||
.refine((questions) => Object.keys(questions).length > 0, "At least one question is required."),
|
||||
},
|
||||
{ invalid_type_error: "Payload must be a JSON object." },
|
||||
)
|
||||
.passthrough()
|
||||
.refine((request) => "state" in request, { message: "Required property 'state' is missing.", path: ["state"] });
|
||||
|
||||
const probability = z.number().finite().min(0).max(1);
|
||||
const probabilities = z.record(probability);
|
||||
|
||||
const noulAnswerSchema = z.object({ type: z.literal("noul"), noul: probability }).passthrough();
|
||||
|
||||
const choiceAnswerShape = {
|
||||
type: z.literal("choice"),
|
||||
choice: z.string(),
|
||||
confidence: probability.optional(),
|
||||
probabilities,
|
||||
};
|
||||
const choiceAnswerSchema = z.object(choiceAnswerShape).passthrough();
|
||||
|
||||
const scoreAnswerShape = {
|
||||
type: z.literal("score"),
|
||||
score: z.number().finite(),
|
||||
confidence: probability.optional(),
|
||||
legend: z.record(z.string()).optional(),
|
||||
probabilities,
|
||||
};
|
||||
const scoreAnswerSchema = z.object(scoreAnswerShape).passthrough();
|
||||
|
||||
const tokenCount = z.number().finite().nonnegative();
|
||||
|
||||
export const systemOneResponseSchema = z
|
||||
.object({
|
||||
model: nonEmptyString("Model must be a non-empty string.").nullish(),
|
||||
answers: z.record(z.discriminatedUnion("type", [noulAnswerSchema, choiceAnswerSchema, scoreAnswerSchema])),
|
||||
usage: z.object({ input_tokens: tokenCount, output_tokens: tokenCount }).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
export type SystemOneRequest = z.infer<typeof systemOneRequestSchema>;
|
||||
export type SystemOneQuestion = z.infer<typeof questionSchema>;
|
||||
export type SystemOneResponse = z.infer<typeof systemOneResponseSchema>;
|
||||
export type SystemOneAnswer = SystemOneResponse["answers"][string];
|
||||
|
|
@ -0,0 +1,250 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { SYSTEM_ONE_EXAMPLE } from "./example";
|
||||
import { validateSystemOnePayload } from "./validatePayload";
|
||||
|
||||
const requestWith = (fields: Record<string, unknown> = {}) =>
|
||||
JSON.stringify({
|
||||
state: "A support request",
|
||||
questions: { category: { type: "choice", instructions: "Route the message", criteria: { support: "Help" } } },
|
||||
...fields,
|
||||
});
|
||||
|
||||
describe("validateSystemOnePayload", () => {
|
||||
it("rejects empty input", () => {
|
||||
expect(validateSystemOnePayload(" ").issues).toContainEqual({
|
||||
path: "root",
|
||||
message: "Payload cannot be empty.",
|
||||
severity: "error",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports JSON syntax errors", () => {
|
||||
expect(validateSystemOnePayload("{").issues[0]?.path).toBe("syntax");
|
||||
});
|
||||
|
||||
it("requires an object root", () => {
|
||||
expect(validateSystemOnePayload("[]").issues[0]?.path).toBe("root");
|
||||
});
|
||||
|
||||
it("requires the state key while accepting null state", () => {
|
||||
expect(validateSystemOnePayload(requestWith({ state: undefined })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "state", severity: "error" }),
|
||||
);
|
||||
expect(validateSystemOnePayload(requestWith({ state: null })).isValid).toBe(true);
|
||||
});
|
||||
|
||||
it.each([null, "", 123])("requires a non-empty string model when present: %s", (model) => {
|
||||
expect(validateSystemOnePayload(requestWith({ model })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "model", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires questions", () => {
|
||||
expect(validateSystemOnePayload(JSON.stringify({ state: null })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([null, [], "questions", {}])("requires a non-empty questions object: %s", (questions) => {
|
||||
expect(validateSystemOnePayload(requestWith({ questions })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([null, [], "question"])("requires each question to be an object: %s", (question) => {
|
||||
expect(validateSystemOnePayload(requestWith({ questions: { invalid: question } })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.invalid", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires a supported question type", () => {
|
||||
expect(
|
||||
validateSystemOnePayload(requestWith({ questions: { invalid: { type: "other", instructions: "Do this" } } }))
|
||||
.issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.invalid.type", severity: "error" }));
|
||||
});
|
||||
|
||||
it("requires instructions", () => {
|
||||
expect(validateSystemOnePayload(requestWith({ questions: { category: { type: "noul" } } })).issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.category.instructions", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires instructions to be a string", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: { category: { type: "choice", instructions: { text: "Route" }, criteria: { support: "Help" } } },
|
||||
}),
|
||||
);
|
||||
expect(result.isValid).toBe(false);
|
||||
expect(result.issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.category.instructions", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([undefined, null, [], "criteria"])("requires choice criteria as an object: %s", (criteria) => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { category: { type: "choice", instructions: "Route", criteria } } }),
|
||||
).issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.category.criteria", severity: "error" }));
|
||||
});
|
||||
|
||||
it("requires choice descriptions to be strings", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: { category: { type: "choice", instructions: "Route", criteria: { support: { text: "Help" } } } },
|
||||
}),
|
||||
);
|
||||
expect(result.isValid).toBe(false);
|
||||
expect(result.issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.category.criteria.support", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([{}, Object.fromEntries(Array.from({ length: 256 }, (_, index) => [`option-${index}`, "Description"]))])(
|
||||
"requires between 1 and 255 choice criteria",
|
||||
(criteria) => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { category: { type: "choice", instructions: "Route", criteria } } }),
|
||||
).issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.category.criteria", severity: "error" }));
|
||||
},
|
||||
);
|
||||
|
||||
it("accepts 1 and 255 choice criteria", () => {
|
||||
const one = { first: "First option" };
|
||||
const twoHundredFiftyFiveOptions = Object.fromEntries(
|
||||
Array.from({ length: 255 }, (_, index) => [`option-${index}`, "Option"]),
|
||||
);
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { category: { type: "choice", instructions: "Route", criteria: one } } }),
|
||||
).isValid,
|
||||
).toBe(true);
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: { category: { type: "choice", instructions: "Route", criteria: twoHundredFiftyFiveOptions } },
|
||||
}),
|
||||
).isValid,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([null, {}, "criteria"])("requires score criteria as an array: %s", (criteria) => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { urgency: { type: "score", instructions: "Rate urgency", criteria } } }),
|
||||
).issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.urgency.criteria", severity: "error" }));
|
||||
});
|
||||
|
||||
it("requires at least two score levels", () => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { urgency: { type: "score", instructions: "Rate urgency", criteria: ["Low"] } } }),
|
||||
).issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.urgency.criteria", severity: "error" }));
|
||||
});
|
||||
|
||||
it("requires score levels to be strings", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: {
|
||||
urgency: { type: "score", instructions: "Rate urgency", criteria: ["Low", { level: "High" }] },
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.isValid).toBe(false);
|
||||
expect(result.issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.urgency.criteria.1", severity: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("warns about more than ten score levels without invalidating the request", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: {
|
||||
urgency: {
|
||||
type: "score",
|
||||
instructions: "Rate urgency",
|
||||
criteria: Array.from({ length: 11 }, () => "Level"),
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.isValid).toBe(true);
|
||||
expect(result.issues).toContainEqual(
|
||||
expect.objectContaining({ path: "questions.urgency.criteria", severity: "warning" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts a score question with two levels", () => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: { urgency: { type: "score", instructions: "Rate urgency", criteria: ["Low", "High"] } },
|
||||
}),
|
||||
).isValid,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([null, [], "criteria"])("requires optional noul criteria to be an object: %s", (criteria) => {
|
||||
expect(
|
||||
validateSystemOnePayload(
|
||||
requestWith({ questions: { escalation: { type: "noul", instructions: "Escalate?", criteria } } }),
|
||||
).issues,
|
||||
).toContainEqual(expect.objectContaining({ path: "questions.escalation.criteria", severity: "error" }));
|
||||
});
|
||||
|
||||
it("allows omitted noul criteria", () => {
|
||||
expect(
|
||||
validateSystemOnePayload(requestWith({ questions: { escalation: { type: "noul", instructions: "Escalate?" } } }))
|
||||
.isValid,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires noul criteria definitions to be strings", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
questions: {
|
||||
escalation: {
|
||||
type: "noul",
|
||||
instructions: "Escalate?",
|
||||
criteria: { true: { meaning: "yes" }, false: 1 },
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.isValid).toBe(false);
|
||||
expect(result.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ path: "questions.escalation.criteria.true", severity: "error" }),
|
||||
expect.objectContaining({ path: "questions.escalation.criteria.false", severity: "error" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps fields outside the known schema so they reach the upstream model", () => {
|
||||
const result = validateSystemOnePayload(
|
||||
requestWith({
|
||||
temperature: 0,
|
||||
questions: {
|
||||
category: { type: "choice", instructions: "Route", criteria: { support: "Help" }, weight: 2 },
|
||||
escalate: { type: "noul", instructions: "Escalate?", criteria: { true: "Yes", unsure: "Maybe" } },
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.payload).toMatchObject({ temperature: 0, questions: { category: { weight: 2 } } });
|
||||
expect(result.payload?.questions.escalate).toEqual({
|
||||
type: "noul",
|
||||
instructions: "Escalate?",
|
||||
criteria: { true: "Yes", unsure: "Maybe" },
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts the built-in example without issues", () => {
|
||||
expect(validateSystemOnePayload(JSON.stringify(SYSTEM_ONE_EXAMPLE))).toMatchObject({ isValid: true, issues: [] });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
import { systemOneRequestSchema, type SystemOneRequest } from "./schemas";
|
||||
|
||||
const RECOMMENDED_MAX_SCORE_LEVELS = 10;
|
||||
|
||||
export interface SystemOnePayloadIssue {
|
||||
path: string;
|
||||
message: string;
|
||||
severity: "error" | "warning";
|
||||
}
|
||||
|
||||
export interface SystemOnePayloadValidation {
|
||||
isValid: boolean;
|
||||
payload?: SystemOneRequest;
|
||||
issues: SystemOnePayloadIssue[];
|
||||
}
|
||||
|
||||
const invalid = (path: string, message: string): SystemOnePayloadValidation => ({
|
||||
isValid: false,
|
||||
issues: [{ path, message, severity: "error" }],
|
||||
});
|
||||
|
||||
function parseJson(raw: string): { ok: true; value: unknown } | { ok: false; message: string } {
|
||||
try {
|
||||
return { ok: true, value: JSON.parse(raw) };
|
||||
} catch (error: unknown) {
|
||||
return { ok: false, message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
const scoreLevelWarnings = (payload: SystemOneRequest): SystemOnePayloadIssue[] =>
|
||||
Object.entries(payload.questions)
|
||||
.filter(([, question]) => question.type === "score" && question.criteria.length > RECOMMENDED_MAX_SCORE_LEVELS)
|
||||
.map(([id]) => ({
|
||||
path: `questions.${id}.criteria`,
|
||||
message: `More than ${RECOMMENDED_MAX_SCORE_LEVELS} score levels may reduce result quality.`,
|
||||
severity: "warning",
|
||||
}));
|
||||
|
||||
export function validateSystemOnePayload(raw: string): SystemOnePayloadValidation {
|
||||
if (!raw.trim()) {
|
||||
return invalid("root", "Payload cannot be empty.");
|
||||
}
|
||||
|
||||
const json = parseJson(raw);
|
||||
if (!json.ok) {
|
||||
return invalid("syntax", `Invalid JSON syntax: ${json.message}`);
|
||||
}
|
||||
|
||||
const result = systemOneRequestSchema.safeParse(json.value);
|
||||
if (!result.success) {
|
||||
return {
|
||||
isValid: false,
|
||||
issues: result.error.issues.map((issue) => ({
|
||||
path: issue.path.length > 0 ? issue.path.join(".") : "root",
|
||||
message: issue.message,
|
||||
severity: "error",
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
return { isValid: true, payload: result.data, issues: scoreLevelWarnings(result.data) };
|
||||
}
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { makeSystemOneRequest } from "./system_one";
|
||||
import type { SystemOneRequest, SystemOneResponse } from "../components/systemOneUI/lib/schemas";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getProxyBaseUrl: vi.fn(() => "https://proxy.example.com"),
|
||||
getGlobalLitellmHeaderName: vi.fn(() => "Authorization"),
|
||||
}));
|
||||
|
||||
const payload: SystemOneRequest = {
|
||||
model: "jev-latest",
|
||||
state: "The app will not open",
|
||||
questions: {
|
||||
category: {
|
||||
type: "choice",
|
||||
instructions: "Which team should respond?",
|
||||
criteria: { technical: "Application issues" },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const responseBody: SystemOneResponse = {
|
||||
model: "jev-1.13.0",
|
||||
answers: { category: { type: "choice", choice: "technical", probabilities: { technical: 1 } } },
|
||||
};
|
||||
|
||||
const malformedResponses: { name: string; body: unknown }[] = [
|
||||
{ name: "model", body: { model: 1, answers: {} } },
|
||||
{ name: "answers", body: { model: "jev-1.13.0", answers: [] } },
|
||||
{ name: "answer type", body: { model: "jev-1.13.0", answers: { category: { type: "unknown" } } } },
|
||||
{
|
||||
name: "probability",
|
||||
body: {
|
||||
model: "jev-1.13.0",
|
||||
answers: { category: { type: "choice", choice: "technical", probabilities: { technical: 1.5 } } },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "usage",
|
||||
body: { model: "jev-1.13.0", answers: {}, usage: { input_tokens: "1", output_tokens: 1 } },
|
||||
},
|
||||
];
|
||||
|
||||
describe("makeSystemOneRequest", () => {
|
||||
const mockFetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>();
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
text: async () => JSON.stringify(responseBody),
|
||||
} as Response);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("sends the typed request to the proxy endpoint with required headers and parses the response", async () => {
|
||||
const result = await makeSystemOneRequest(payload, "session-key", "https://custom.example.com/");
|
||||
const expectedRequest: Partial<RequestInit> = {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: "Bearer session-key",
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: undefined,
|
||||
};
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
"https://custom.example.com/typesafe/v1/systemone",
|
||||
expect.objectContaining(expectedRequest),
|
||||
);
|
||||
expect(result.response).toEqual(responseBody);
|
||||
expect(result.latencyMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("uses the proxy base URL when no custom base is provided", async () => {
|
||||
await makeSystemOneRequest(payload, "session-key");
|
||||
|
||||
expect(mockFetch.mock.calls[0]?.[0]).toBe("https://proxy.example.com/typesafe/v1/systemone");
|
||||
});
|
||||
|
||||
it("throws the upstream error text for failed requests", async () => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: async () => "Invalid virtual key",
|
||||
} as Response);
|
||||
|
||||
await expect(makeSystemOneRequest(payload, "session-key")).rejects.toThrow("Invalid virtual key");
|
||||
});
|
||||
|
||||
it.each(malformedResponses)("returns an error for a malformed System One $name response", async ({ body }) => {
|
||||
mockFetch.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
text: async () => JSON.stringify(body) ?? "",
|
||||
} as Response);
|
||||
|
||||
await expect(makeSystemOneRequest(payload, "session-key")).rejects.toThrow(
|
||||
"System One response has an invalid shape.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
|
||||
import { withRequiredHeaders } from "@/components/llm_calls/request_headers";
|
||||
import { createApiClient } from "@/lib/http/client";
|
||||
import {
|
||||
systemOneResponseSchema,
|
||||
type SystemOneRequest,
|
||||
type SystemOneResponse,
|
||||
} from "../components/systemOneUI/lib/schemas";
|
||||
|
||||
export interface SystemOneResult {
|
||||
response: SystemOneResponse;
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
export async function makeSystemOneRequest(
|
||||
payload: SystemOneRequest,
|
||||
accessToken: string,
|
||||
customBaseUrl?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<SystemOneResult> {
|
||||
const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
|
||||
const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl;
|
||||
const startedAt = performance.now();
|
||||
const headers = withRequiredHeaders(
|
||||
{},
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
[getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
|
||||
},
|
||||
);
|
||||
const client = createApiClient({ getBaseUrl: () => normalizedBaseUrl });
|
||||
const body = await client.post<unknown>("/typesafe/v1/systemone", { body: payload, headers, signal });
|
||||
const parsed = systemOneResponseSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
throw new Error("System One response has an invalid shape.");
|
||||
}
|
||||
return { response: parsed.data, latencyMs: performance.now() - startedAt };
|
||||
}
|
||||
|
|
@ -34,6 +34,10 @@ vi.mock("@/app/(dashboard)/playground/components/complianceUI/ComplianceUI", ()
|
|||
default: () => <div data-testid="compliance-ui" />,
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/playground/components/systemOneUI/SystemOneUI", () => ({
|
||||
default: () => <div data-testid="system-one-ui" />,
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/playground/components/chat_ui/AgentBuilderView", () => ({
|
||||
default: () => <div data-testid="agent-builder" />,
|
||||
}));
|
||||
|
|
@ -55,6 +59,7 @@ describe("PlaygroundPage role guard", () => {
|
|||
expect(screen.queryByTestId("chat-ui")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("compare-ui")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("compliance-ui")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("system-one-ui")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("agent-builder")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -82,6 +87,15 @@ describe("PlaygroundPage ?tab= deep link", () => {
|
|||
expect(screen.getByRole("tab", { name: "Chat" })).toHaveAttribute("aria-selected", "false");
|
||||
});
|
||||
|
||||
it("activates System One from ?tab=system-one", () => {
|
||||
renderWithProviders(<PlaygroundPage />, { searchParams: { tab: "system-one" } });
|
||||
|
||||
const tab = screen.getByRole("tab", { name: /^System One/ });
|
||||
expect(tab).toHaveAttribute("aria-selected", "true");
|
||||
expect(tab).toHaveTextContent("Beta");
|
||||
expect(screen.getByTestId("system-one-ui")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("falls back to Chat when ?tab= is not a playground tab", () => {
|
||||
renderWithProviders(<PlaygroundPage />, { searchParams: { tab: "settings" } });
|
||||
|
||||
|
|
|
|||
|
|
@ -5,13 +5,15 @@ import AgentBuilderView from "@/app/(dashboard)/playground/components/chat_ui/Ag
|
|||
import ChatUI from "@/app/(dashboard)/playground/components/chat_ui/ChatUI";
|
||||
import CompareUI from "@/app/(dashboard)/playground/components/compareUI/CompareUI";
|
||||
import ComplianceUI from "@/app/(dashboard)/playground/components/complianceUI/ComplianceUI";
|
||||
import SystemOneUI from "@/app/(dashboard)/playground/components/systemOneUI/SystemOneUI";
|
||||
import BetaBadge from "@/components/BetaBadge";
|
||||
import { DeprecationBanner } from "@/components/DeprecationBanner";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { fetchProxySettings } from "@/utils/proxyUtils";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { useUrlTab } from "@/hooks/useUrlTab";
|
||||
|
||||
const PLAYGROUND_TABS = ["chat", "compare", "compliance", "agent-builder"] as const;
|
||||
const PLAYGROUND_TABS = ["chat", "compare", "compliance", "system-one", "agent-builder"] as const;
|
||||
|
||||
interface ProxySettings {
|
||||
PROXY_BASE_URL?: string;
|
||||
|
|
@ -67,6 +69,9 @@ export default function PlaygroundPage() {
|
|||
<TabsTrigger value="compliance" className="flex-none">
|
||||
Compliance
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="system-one" className="flex-none">
|
||||
<BetaBadge>System One</BetaBadge>
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="agent-builder" className="flex-none">
|
||||
Agent Builder (Experimental)
|
||||
</TabsTrigger>
|
||||
|
|
@ -91,6 +96,9 @@ export default function PlaygroundPage() {
|
|||
<TabsContent value="compliance" className="mt-0 h-full data-hidden:hidden" keepMounted>
|
||||
<ComplianceUI accessToken={accessToken} disabledPersonalKeyCreation={disabledPersonalKeyCreation} />
|
||||
</TabsContent>
|
||||
<TabsContent value="system-one" className="mt-0 min-h-0 data-hidden:hidden" keepMounted>
|
||||
<SystemOneUI accessToken={accessToken} disabledPersonalKeyCreation={disabledPersonalKeyCreation} />
|
||||
</TabsContent>
|
||||
<TabsContent value="agent-builder" className="mt-0 h-full data-hidden:hidden" keepMounted>
|
||||
<DeprecationBanner featureName="The Playground's Agent Builder" />
|
||||
<AgentBuilderView
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue